├── .github ├── FUNDING.yml ├── dependabot.yml └── workflows │ ├── lint.yml │ ├── release.yml │ └── test.yml ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── buffer ├── buffer.go ├── buffer_arc.go ├── buffer_lfu.go ├── buffer_lru.go └── buffer_unlimited.go ├── counters.go ├── example └── example.go ├── go.mod ├── go.sum ├── hash.go ├── hook.go ├── main_test.go ├── matchers.go ├── middleware_absolute.go ├── middleware_custom.go ├── middleware_threshold.go ├── middleware_uniform.go ├── middleware_uniform_test.go ├── random.go └── random_test.go /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: [samber] 2 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: github-actions 4 | directory: / 5 | schedule: 6 | interval: weekly 7 | - package-ecosystem: gomod 8 | directory: / 9 | schedule: 10 | interval: weekly 11 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: Lint 2 | 3 | on: 4 | push: 5 | pull_request: 6 | 7 | jobs: 8 | golangci: 9 | name: lint 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/setup-go@v5 13 | with: 14 | go-version: 1.21 15 | stable: false 16 | - uses: actions/checkout@v4 17 | - name: golangci-lint 18 | uses: golangci/golangci-lint-action@v8 19 | with: 20 | args: --timeout 120s --max-same-issues 50 21 | 22 | - name: Bearer 23 | uses: bearer/bearer-action@v2 24 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | semver: 7 | type: string 8 | description: 'Semver (eg: v1.2.3)' 9 | required: true 10 | 11 | jobs: 12 | release: 13 | if: github.triggering_actor == 'samber' 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v4 17 | 18 | - name: Set up Go 19 | uses: actions/setup-go@v5 20 | with: 21 | go-version: 1.21 22 | stable: false 23 | 24 | - name: Test 25 | run: make test 26 | 27 | # remove tests in order to clean dependencies 28 | - name: Remove xxx_test.go files 29 | run: rm -rf *_test.go ./example ./images 30 | 31 | # cleanup test dependencies 32 | - name: Cleanup dependencies 33 | run: go mod tidy 34 | 35 | - name: List files 36 | run: tree -Cfi 37 | - name: Write new go.mod into logs 38 | run: cat go.mod 39 | - name: Write new go.sum into logs 40 | run: cat go.sum 41 | 42 | - name: Create tag 43 | run: | 44 | git config --global user.name '${{ github.triggering_actor }}' 45 | git config --global user.email "${{ github.triggering_actor}}@users.noreply.github.com" 46 | 47 | git add . 48 | git commit --allow-empty -m 'bump ${{ inputs.semver }}' 49 | git tag ${{ inputs.semver }} 50 | git push origin ${{ inputs.semver }} 51 | 52 | - name: Release 53 | uses: softprops/action-gh-release@v2 54 | with: 55 | name: ${{ inputs.semver }} 56 | tag_name: ${{ inputs.semver }} 57 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | push: 5 | tags: 6 | branches: 7 | pull_request: 8 | 9 | jobs: 10 | 11 | test: 12 | runs-on: ubuntu-latest 13 | strategy: 14 | matrix: 15 | go: 16 | - '1.21' 17 | - '1.22' 18 | - '1.23' 19 | - '1.24' 20 | - '1.x' 21 | steps: 22 | - uses: actions/checkout@v4 23 | 24 | - name: Set up Go 25 | uses: actions/setup-go@v5 26 | with: 27 | go-version: ${{ matrix.go }} 28 | stable: false 29 | 30 | - name: Build 31 | run: make build 32 | 33 | - name: Test 34 | run: make test 35 | 36 | - name: Test 37 | run: make coverage 38 | 39 | - name: Codecov 40 | uses: codecov/codecov-action@v5 41 | with: 42 | token: ${{ secrets.CODECOV_TOKEN }} 43 | file: ./cover.out 44 | flags: unittests 45 | verbose: true 46 | if: matrix.go == '1.21' 47 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.toptal.com/developers/gitignore/api/go 3 | # Edit at https://www.toptal.com/developers/gitignore?templates=go 4 | 5 | ### Go ### 6 | # If you prefer the allow list template instead of the deny list, see community template: 7 | # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore 8 | # 9 | # Binaries for programs and plugins 10 | *.exe 11 | *.exe~ 12 | *.dll 13 | *.so 14 | *.dylib 15 | 16 | # Test binary, built with `go test -c` 17 | *.test 18 | 19 | # Output of the go coverage tool, specifically when used with LiteIDE 20 | *.out 21 | 22 | # Dependency directories (remove the comment below to include it) 23 | # vendor/ 24 | 25 | # Go workspace file 26 | go.work 27 | 28 | ### Go Patch ### 29 | /vendor/ 30 | /Godeps/ 31 | 32 | # End of https://www.toptal.com/developers/gitignore/api/go 33 | 34 | cover.out 35 | cover.html 36 | .vscode 37 | 38 | .idea/ 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Samuel Berthe 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | build: 3 | go build -v ./... 4 | 5 | test: 6 | go test -race -v ./... 7 | watch-test: 8 | reflex -t 50ms -s -- sh -c 'gotest -race -v ./...' 9 | 10 | bench: 11 | go test -benchmem -count 3 -bench ./... 12 | watch-bench: 13 | reflex -t 50ms -s -- sh -c 'go test -benchmem -count 3 -bench ./...' 14 | 15 | coverage: 16 | go test -v -coverprofile=cover.out -covermode=atomic ./... 17 | go tool cover -html=cover.out -o cover.html 18 | 19 | tools: 20 | go install github.com/cespare/reflex@latest 21 | go install github.com/rakyll/gotest@latest 22 | go install github.com/psampaz/go-mod-outdated@latest 23 | go install github.com/jondot/goweight@latest 24 | go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest 25 | go get -t -u golang.org/x/tools/cmd/cover 26 | go install github.com/sonatype-nexus-community/nancy@latest 27 | go mod tidy 28 | 29 | lint: 30 | golangci-lint run --timeout 60s --max-same-issues 50 ./... 31 | lint-fix: 32 | golangci-lint run --timeout 60s --max-same-issues 50 --fix ./... 33 | 34 | audit: 35 | go list -json -m all | nancy sleuth 36 | 37 | outdated: 38 | go list -u -m -json all | go-mod-outdated -update -direct 39 | 40 | weight: 41 | goweight 42 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # Slog sampling policy 3 | 4 | [![tag](https://img.shields.io/github/tag/samber/slog-sampling.svg)](https://github.com/samber/slog-sampling/releases) 5 | ![Go Version](https://img.shields.io/badge/Go-%3E%3D%201.21-%23007d9c) 6 | [![GoDoc](https://godoc.org/github.com/samber/slog-sampling?status.svg)](https://pkg.go.dev/github.com/samber/slog-sampling) 7 | ![Build Status](https://github.com/samber/slog-sampling/actions/workflows/test.yml/badge.svg) 8 | [![Go report](https://goreportcard.com/badge/github.com/samber/slog-sampling)](https://goreportcard.com/report/github.com/samber/slog-sampling) 9 | [![Coverage](https://img.shields.io/codecov/c/github/samber/slog-sampling)](https://codecov.io/gh/samber/slog-sampling) 10 | [![Contributors](https://img.shields.io/github/contributors/samber/slog-sampling)](https://github.com/samber/slog-sampling/graphs/contributors) 11 | [![License](https://img.shields.io/github/license/samber/slog-sampling)](./LICENSE) 12 | 13 | A middleware that samples incoming records which caps the CPU and I/O load of logging while attempting to preserve a representative subset of your logs. 14 | 15 | Sampling fixes throughput by dropping repetitive log entries. 16 | 17 |
18 |
19 | Sponsored by: 20 |
21 | 22 |
23 | Quickwit 24 |
25 |
26 | Cloud-native search engine for observability - An OSS alternative to Splunk, Elasticsearch, Loki, and Tempo. 27 |
28 | 29 |
30 |
31 | 32 | **See also:** 33 | 34 | - [slog-multi](https://github.com/samber/slog-multi): `slog.Handler` chaining, fanout, routing, failover, load balancing... 35 | - [slog-formatter](https://github.com/samber/slog-formatter): `slog` attribute formatting 36 | - [slog-sampling](https://github.com/samber/slog-sampling): `slog` sampling policy 37 | - [slog-mock](https://github.com/samber/slog-mock): `slog.Handler` for test purposes 38 | 39 | **HTTP middlewares:** 40 | 41 | - [slog-gin](https://github.com/samber/slog-gin): Gin middleware for `slog` logger 42 | - [slog-echo](https://github.com/samber/slog-echo): Echo middleware for `slog` logger 43 | - [slog-fiber](https://github.com/samber/slog-fiber): Fiber middleware for `slog` logger 44 | - [slog-chi](https://github.com/samber/slog-chi): Chi middleware for `slog` logger 45 | - [slog-http](https://github.com/samber/slog-http): `net/http` middleware for `slog` logger 46 | 47 | **Loggers:** 48 | 49 | - [slog-zap](https://github.com/samber/slog-zap): A `slog` handler for `Zap` 50 | - [slog-zerolog](https://github.com/samber/slog-zerolog): A `slog` handler for `Zerolog` 51 | - [slog-logrus](https://github.com/samber/slog-logrus): A `slog` handler for `Logrus` 52 | 53 | **Log sinks:** 54 | 55 | - [slog-datadog](https://github.com/samber/slog-datadog): A `slog` handler for `Datadog` 56 | - [slog-betterstack](https://github.com/samber/slog-betterstack): A `slog` handler for `Betterstack` 57 | - [slog-rollbar](https://github.com/samber/slog-rollbar): A `slog` handler for `Rollbar` 58 | - [slog-loki](https://github.com/samber/slog-loki): A `slog` handler for `Loki` 59 | - [slog-sentry](https://github.com/samber/slog-sentry): A `slog` handler for `Sentry` 60 | - [slog-syslog](https://github.com/samber/slog-syslog): A `slog` handler for `Syslog` 61 | - [slog-logstash](https://github.com/samber/slog-logstash): A `slog` handler for `Logstash` 62 | - [slog-fluentd](https://github.com/samber/slog-fluentd): A `slog` handler for `Fluentd` 63 | - [slog-graylog](https://github.com/samber/slog-graylog): A `slog` handler for `Graylog` 64 | - [slog-quickwit](https://github.com/samber/slog-quickwit): A `slog` handler for `Quickwit` 65 | - [slog-slack](https://github.com/samber/slog-slack): A `slog` handler for `Slack` 66 | - [slog-telegram](https://github.com/samber/slog-telegram): A `slog` handler for `Telegram` 67 | - [slog-mattermost](https://github.com/samber/slog-mattermost): A `slog` handler for `Mattermost` 68 | - [slog-microsoft-teams](https://github.com/samber/slog-microsoft-teams): A `slog` handler for `Microsoft Teams` 69 | - [slog-webhook](https://github.com/samber/slog-webhook): A `slog` handler for `Webhook` 70 | - [slog-kafka](https://github.com/samber/slog-kafka): A `slog` handler for `Kafka` 71 | - [slog-nats](https://github.com/samber/slog-nats): A `slog` handler for `NATS` 72 | - [slog-parquet](https://github.com/samber/slog-parquet): A `slog` handler for `Parquet` + `Object Storage` 73 | - [slog-channel](https://github.com/samber/slog-channel): A `slog` handler for Go channels 74 | 75 | ## 🚀 Install 76 | 77 | ```sh 78 | go get github.com/samber/slog-sampling 79 | ``` 80 | 81 | **Compatibility**: go >= 1.21 82 | 83 | No breaking changes will be made to exported APIs before v2.0.0. 84 | 85 | ## 💡 Usage 86 | 87 | GoDoc: [https://pkg.go.dev/github.com/samber/slog-sampling](https://pkg.go.dev/github.com/samber/slog-sampling) 88 | 89 | ### Middlewares 90 | 91 | 3 strategies are available: 92 | - [Uniform sampling](#uniform-sampling): drop % of logs 93 | - [Threshold sampling](#threshold-sampling): drop % of logs after a threshold 94 | - [Absolute sampling](#absolute-sampling): limit logs throughput to a fixed number of records 95 | - [Custom sampler](#custom-sampler) 96 | 97 | The sampling middleware can be used standalone or with the `slog-multi` helpers. 98 | 99 | A combination of multiple sampling strategies can be chained. Eg: 100 | - drop when a single log message is produced more than 100 times per second 101 | - drop above 1000 log records per second (globally) 102 | 103 | ### Matchers 104 | 105 | Similar log records can be deduplicated and rate-limited using the `Matcher` API. 106 | 107 | Available `Matcher`: 108 | - `slogsampling.MatchByLevelAndMessage` (default) 109 | - `slogsampling.MatchAll` 110 | - `slogsampling.MatchByLevel` 111 | - `slogsampling.MatchByMessage` 112 | - `slogsampling.MatchBySource` 113 | - `slogsampling.MatchByAttribute` 114 | - `slogsampling.MatchByContextValue` 115 | 116 | ### Uniform sampling 117 | 118 | ```go 119 | type UniformSamplingOption struct { 120 | // The sample rate for sampling traces in the range [0.0, 1.0]. 121 | Rate float64 122 | 123 | // Optional hooks 124 | OnAccepted func(context.Context, slog.Record) 125 | OnDropped func(context.Context, slog.Record) 126 | } 127 | ``` 128 | 129 | Example using `slog-multi`: 130 | 131 | ```go 132 | import ( 133 | slogmulti "github.com/samber/slog-multi" 134 | slogsampling "github.com/samber/slog-sampling" 135 | "log/slog" 136 | ) 137 | 138 | // Will print 33% of entries. 139 | option := slogsampling.UniformSamplingOption{ 140 | // The sample rate for sampling traces in the range [0.0, 1.0]. 141 | Rate: 0.33, 142 | } 143 | 144 | logger := slog.New( 145 | slogmulti. 146 | Pipe(option.NewMiddleware()). 147 | Handler(slog.NewJSONHandler(os.Stdout, nil)), 148 | ) 149 | ``` 150 | 151 | ### Threshold sampling 152 | 153 | ```go 154 | type ThresholdSamplingOption struct { 155 | // This will log the first `Threshold` log entries with the same hash, 156 | // in a `Tick` interval as-is. Following that, it will allow `Rate` in the range [0.0, 1.0]. 157 | Tick time.Duration 158 | Threshold uint64 159 | Rate float64 160 | 161 | // Group similar logs (default: by level and message) 162 | Matcher func(ctx context.Context, record *slog.Record) string 163 | 164 | // Optional hooks 165 | OnAccepted func(context.Context, slog.Record) 166 | OnDropped func(context.Context, slog.Record) 167 | } 168 | ``` 169 | 170 | If `Rate` is zero, the middleware will drop all log entries after the first `Threshold` records in that interval. 171 | 172 | Example using `slog-multi`: 173 | 174 | ```go 175 | import ( 176 | slogmulti "github.com/samber/slog-multi" 177 | slogsampling "github.com/samber/slog-sampling" 178 | "log/slog" 179 | ) 180 | 181 | // Will print the first 10 entries having the same level+message, then every 10th messages until next interval. 182 | option := slogsampling.ThresholdSamplingOption{ 183 | Tick: 5 * time.Second, 184 | Threshold: 10, 185 | Rate: 0.1, 186 | } 187 | 188 | logger := slog.New( 189 | slogmulti. 190 | Pipe(option.NewMiddleware()). 191 | Handler(slog.NewJSONHandler(os.Stdout, nil)), 192 | ) 193 | ``` 194 | 195 | ### Absolute sampling 196 | 197 | ```go 198 | type AbsoluteSamplingOption struct { 199 | // This will log all entries with the same hash until max is reached, 200 | // in a `Tick` interval as-is. Following that, it will reduce log throughput 201 | // depending on previous interval. 202 | Tick time.Duration 203 | Max uint64 204 | 205 | // Group similar logs (default: by level and message) 206 | Matcher Matcher 207 | 208 | // Optional hooks 209 | OnAccepted func(context.Context, slog.Record) 210 | OnDropped func(context.Context, slog.Record) 211 | } 212 | ``` 213 | 214 | Example using `slog-multi`: 215 | 216 | ```go 217 | import ( 218 | slogmulti "github.com/samber/slog-multi" 219 | slogsampling "github.com/samber/slog-sampling" 220 | "log/slog" 221 | ) 222 | 223 | // Will print the first 10 entries during the first 5s, then a fraction of messages during the following intervals. 224 | option := slogsampling.AbsoluteSamplingOption{ 225 | Tick: 5 * time.Second, 226 | Max: 10, 227 | 228 | Matcher: slogsampling.MatchAll(), 229 | } 230 | 231 | logger := slog.New( 232 | slogmulti. 233 | Pipe(option.NewMiddleware()). 234 | Handler(slog.NewJSONHandler(os.Stdout, nil)), 235 | ) 236 | ``` 237 | 238 | ### Custom sampler 239 | 240 | ```go 241 | type CustomSamplingOption struct { 242 | // The sample rate for sampling traces in the range [0.0, 1.0]. 243 | Sampler func(context.Context, slog.Record) float64 244 | 245 | // Optional hooks 246 | OnAccepted func(context.Context, slog.Record) 247 | OnDropped func(context.Context, slog.Record) 248 | } 249 | ``` 250 | 251 | Example using `slog-multi`: 252 | 253 | ```go 254 | import ( 255 | slogmulti "github.com/samber/slog-multi" 256 | slogsampling "github.com/samber/slog-sampling" 257 | "log/slog" 258 | ) 259 | 260 | // Will print 100% of log entries during the night, or 50% of errors, 20% of warnings and 1% of lower levels. 261 | option := slogsampling.CustomSamplingOption{ 262 | Sampler: func(ctx context.Context, record slog.Record) float64 { 263 | if record.Time.Hour() < 6 || record.Time.Hour() > 22 { 264 | return 1 265 | } 266 | 267 | switch record.Level { 268 | case slog.LevelError: 269 | return 0.5 270 | case slog.LevelWarn: 271 | return 0.2 272 | default: 273 | return 0.01 274 | } 275 | }, 276 | } 277 | 278 | logger := slog.New( 279 | slogmulti. 280 | Pipe(option.NewMiddleware()). 281 | Handler(slog.NewJSONHandler(os.Stdout, nil)), 282 | ) 283 | ``` 284 | 285 | ## 🤝 Contributing 286 | 287 | - Ping me on twitter [@samuelberthe](https://twitter.com/samuelberthe) (DMs, mentions, whatever :)) 288 | - Fork the [project](https://github.com/samber/slog-sampling) 289 | - Fix [open issues](https://github.com/samber/slog-sampling/issues) or request new features 290 | 291 | Don't hesitate ;) 292 | 293 | ```bash 294 | # Install some dev dependencies 295 | make tools 296 | 297 | # Run tests 298 | make test 299 | # or 300 | make watch-test 301 | ``` 302 | 303 | ## 👤 Contributors 304 | 305 | ![Contributors](https://contrib.rocks/image?repo=samber/slog-sampling) 306 | 307 | ## 💫 Show your support 308 | 309 | Give a ⭐️ if this project helped you! 310 | 311 | [![GitHub Sponsors](https://img.shields.io/github/sponsors/samber?style=for-the-badge)](https://github.com/sponsors/samber) 312 | 313 | ## 📝 License 314 | 315 | Copyright © 2023 [Samuel Berthe](https://github.com/samber). 316 | 317 | This project is [MIT](./LICENSE) licensed. 318 | -------------------------------------------------------------------------------- /buffer/buffer.go: -------------------------------------------------------------------------------- 1 | package buffer 2 | 3 | type BufferKey interface { 4 | ~string | ~int | ~int8 | ~int16 | ~int32 | ~int64 | ~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 | ~float32 | ~float64 5 | } 6 | 7 | type Buffer[K BufferKey] interface { 8 | GetOrInsert(K) (any, bool) 9 | } 10 | -------------------------------------------------------------------------------- /buffer/buffer_arc.go: -------------------------------------------------------------------------------- 1 | package buffer 2 | 3 | import ( 4 | "github.com/bluele/gcache" 5 | ) 6 | 7 | var _ Buffer[string] = (*ARCBuffer[string])(nil) 8 | 9 | func NewARCBuffer[K BufferKey](size int) func(generator func(K) any) Buffer[K] { 10 | return func(generator func(K) any) Buffer[K] { 11 | return &ARCBuffer[K]{ 12 | generator: generator, 13 | items: gcache.New(size). 14 | ARC(). 15 | LoaderFunc(func(k interface{}) (interface{}, error) { 16 | return generator(k.(K)), nil 17 | }). 18 | Build(), 19 | } 20 | } 21 | } 22 | 23 | type ARCBuffer[K BufferKey] struct { 24 | generator func(K) any 25 | items gcache.Cache 26 | } 27 | 28 | func (b ARCBuffer[K]) GetOrInsert(key K) (any, bool) { 29 | item, err := b.items.Get(key) 30 | return item, err == nil 31 | } 32 | -------------------------------------------------------------------------------- /buffer/buffer_lfu.go: -------------------------------------------------------------------------------- 1 | package buffer 2 | 3 | import ( 4 | "github.com/bluele/gcache" 5 | ) 6 | 7 | var _ Buffer[string] = (*LFUBuffer[string])(nil) 8 | 9 | func NewLFUBuffer[K BufferKey](size int) func(generator func(K) any) Buffer[K] { 10 | return func(generator func(K) any) Buffer[K] { 11 | return &LFUBuffer[K]{ 12 | generator: generator, 13 | items: gcache.New(size). 14 | LFU(). 15 | LoaderFunc(func(k interface{}) (interface{}, error) { 16 | return generator(k.(K)), nil 17 | }). 18 | Build(), 19 | } 20 | } 21 | } 22 | 23 | type LFUBuffer[K BufferKey] struct { 24 | generator func(K) any 25 | items gcache.Cache 26 | } 27 | 28 | func (b LFUBuffer[K]) GetOrInsert(key K) (any, bool) { 29 | item, err := b.items.Get(key) 30 | return item, err == nil 31 | } 32 | -------------------------------------------------------------------------------- /buffer/buffer_lru.go: -------------------------------------------------------------------------------- 1 | package buffer 2 | 3 | import ( 4 | "github.com/bluele/gcache" 5 | ) 6 | 7 | var _ Buffer[string] = (*LRUBuffer[string])(nil) 8 | 9 | func NewLRUBuffer[K BufferKey](size int) func(generator func(K) any) Buffer[K] { 10 | return func(generator func(K) any) Buffer[K] { 11 | return &LRUBuffer[K]{ 12 | generator: generator, 13 | items: gcache.New(size). 14 | LRU(). 15 | LoaderFunc(func(k interface{}) (interface{}, error) { 16 | return generator(k.(K)), nil 17 | }). 18 | Build(), 19 | } 20 | } 21 | } 22 | 23 | type LRUBuffer[K BufferKey] struct { 24 | generator func(K) any 25 | items gcache.Cache 26 | } 27 | 28 | func (b LRUBuffer[K]) GetOrInsert(key K) (any, bool) { 29 | item, err := b.items.Get(key) 30 | return item, err == nil 31 | } 32 | -------------------------------------------------------------------------------- /buffer/buffer_unlimited.go: -------------------------------------------------------------------------------- 1 | package buffer 2 | 3 | import ( 4 | "github.com/cornelk/hashmap" 5 | ) 6 | 7 | var _ Buffer[string] = (*UnlimitedBuffer[string])(nil) 8 | 9 | func NewUnlimitedBuffer[K BufferKey]() func(generator func(K) any) Buffer[K] { 10 | return func(generator func(K) any) Buffer[K] { 11 | return &UnlimitedBuffer[K]{ 12 | generator: generator, 13 | items: hashmap.New[K, any](), 14 | } 15 | } 16 | } 17 | 18 | type UnlimitedBuffer[K BufferKey] struct { 19 | generator func(K) any 20 | items *hashmap.Map[K, any] 21 | } 22 | 23 | func (b UnlimitedBuffer[K]) GetOrInsert(key K) (any, bool) { 24 | return b.items.GetOrInsert(key, b.generator(key)) 25 | } 26 | -------------------------------------------------------------------------------- /counters.go: -------------------------------------------------------------------------------- 1 | package slogsampling 2 | 3 | import ( 4 | "sync/atomic" 5 | "time" 6 | 7 | "github.com/samber/lo" 8 | ) 9 | 10 | func newCounter() *counter { 11 | return &counter{ 12 | resetAt: atomic.Int64{}, 13 | counter: atomic.Uint64{}, 14 | } 15 | } 16 | 17 | type counter struct { 18 | resetAt atomic.Int64 19 | counter atomic.Uint64 20 | } 21 | 22 | func (c *counter) Inc(tick time.Duration) uint64 { 23 | // i prefer not using record.Time, because only the sampling middleware time is relevant 24 | tn := time.Now().UnixNano() 25 | resetAfter := c.resetAt.Load() 26 | if resetAfter > tn { 27 | return c.counter.Add(1) 28 | } 29 | 30 | c.counter.Store(1) 31 | 32 | newResetAfter := tn + tick.Nanoseconds() 33 | if !c.resetAt.CompareAndSwap(resetAfter, newResetAfter) { 34 | // We raced with another goroutine trying to reset, and it also reset 35 | // the counter to 1, so we need to reincrement the counter. 36 | return c.counter.Add(1) 37 | } 38 | 39 | return 1 40 | } 41 | 42 | func newCounterWithMemory() *counterWithMemory { 43 | c := &counterWithMemory{ 44 | resetAtAndPreviousCounter: atomic.Pointer[lo.Tuple2[int64, uint64]]{}, 45 | counter: atomic.Uint64{}, 46 | } 47 | c.resetAtAndPreviousCounter.Store(lo.ToPtr(lo.T2(int64(0), uint64(0)))) 48 | return c 49 | } 50 | 51 | type counterWithMemory struct { 52 | resetAtAndPreviousCounter atomic.Pointer[lo.Tuple2[int64, uint64]] // it would be more memory-efficient with a dedicated struct, but i'm lazy 53 | counter atomic.Uint64 54 | } 55 | 56 | func (c *counterWithMemory) Inc(tick time.Duration) (n uint64, previousCycle uint64) { 57 | // i prefer not using record.Time, because only the sampling middleware time is relevant 58 | tn := time.Now().UnixNano() 59 | resetAtAndPreviousCounter := c.resetAtAndPreviousCounter.Load() 60 | if resetAtAndPreviousCounter.A > tn { 61 | return c.counter.Add(1), resetAtAndPreviousCounter.B 62 | } 63 | 64 | old := c.counter.Swap(1) 65 | 66 | newResetAfter := lo.T2(tn+tick.Nanoseconds(), old) 67 | if !c.resetAtAndPreviousCounter.CompareAndSwap(resetAtAndPreviousCounter, lo.ToPtr(newResetAfter)) { 68 | // We raced with another goroutine trying to reset, and it also reset 69 | // the counter to 1, so we need to reincrement the counter. 70 | return c.counter.Add(1), resetAtAndPreviousCounter.B // we should load again instead of returning this outdated value, but it's not a big deal 71 | } 72 | 73 | return 1, resetAtAndPreviousCounter.B 74 | } 75 | -------------------------------------------------------------------------------- /example/example.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "sync/atomic" 8 | "time" 9 | 10 | "log/slog" 11 | 12 | slogmulti "github.com/samber/slog-multi" 13 | slogsampling "github.com/samber/slog-sampling" 14 | ) 15 | 16 | func main() { 17 | var accepted atomic.Int64 18 | var dropped atomic.Int64 19 | 20 | option := slogsampling.ThresholdSamplingOption{ 21 | Tick: 5 * time.Second, 22 | Threshold: 10, 23 | Rate: 0.1, 24 | 25 | Matcher: func(ctx context.Context, record *slog.Record) string { 26 | return record.Level.String() 27 | }, 28 | 29 | OnAccepted: func(context.Context, slog.Record) { 30 | accepted.Add(1) 31 | }, 32 | OnDropped: func(context.Context, slog.Record) { 33 | dropped.Add(1) 34 | }, 35 | } 36 | 37 | // option := slogsampling.CustomSamplingOption{ 38 | // Sampler: func(ctx context.Context, record slog.Record) float64 { 39 | // switch record.Level { 40 | // case slog.LevelError: 41 | // return 0.5 42 | // case slog.LevelWarn: 43 | // return 0.2 44 | // default: 45 | // return 0.01 46 | // } 47 | // }, 48 | // OnAccepted: func(context.Context, slog.Record) { 49 | // accepted.Add(1) 50 | // }, 51 | // OnDropped: func(context.Context, slog.Record) { 52 | // dropped.Add(1) 53 | // }, 54 | // } 55 | 56 | // option := slogsampling.UniformSamplingOption{ 57 | // Rate: 0.33, 58 | // OnAccepted: func(context.Context, slog.Record) { 59 | // accepted.Add(1) 60 | // }, 61 | // OnDropped: func(context.Context, slog.Record) { 62 | // dropped.Add(1) 63 | // }, 64 | // } 65 | 66 | // option := slogsampling.AbsoluteSamplingOption{ 67 | // Tick: 5 * time.Second, 68 | // Max: 10, 69 | 70 | // Matcher: func(ctx context.Context, record *slog.Record) string { 71 | // return record.Level.String() 72 | // }, 73 | 74 | // OnAccepted: func(context.Context, slog.Record) { 75 | // accepted.Add(1) 76 | // }, 77 | // OnDropped: func(context.Context, slog.Record) { 78 | // dropped.Add(1) 79 | // }, 80 | // } 81 | 82 | logger := slog.New( 83 | slogmulti. 84 | Pipe(option.NewMiddleware()). 85 | Handler(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{})), 86 | ) 87 | 88 | l := logger. 89 | With("email", "samuel@acme.org"). 90 | With("environment", "dev"). 91 | With("hello", "world") 92 | 93 | for i := 0; i < 100; i++ { 94 | l.Error("Message 1") 95 | l.Error("Message 2") 96 | l.Info("Message 1") 97 | time.Sleep(100 * time.Millisecond) 98 | } 99 | 100 | fmt.Printf("\n\nResults:\n") 101 | fmt.Printf("Accepted: %d\n", accepted.Load()) 102 | fmt.Printf("Dropped: %d\n", dropped.Load()) 103 | } 104 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/samber/slog-sampling 2 | 3 | go 1.21 4 | 5 | require ( 6 | github.com/bluele/gcache v0.0.2 7 | github.com/cornelk/hashmap v1.0.8 8 | github.com/samber/lo v1.51.0 9 | github.com/samber/slog-common v0.18.1 10 | github.com/samber/slog-multi v1.4.0 11 | github.com/stretchr/testify v1.10.0 12 | go.uber.org/goleak v1.3.0 13 | ) 14 | 15 | require ( 16 | github.com/davecgh/go-spew v1.1.1 // indirect 17 | github.com/kr/text v0.2.0 // indirect 18 | github.com/pmezard/go-difflib v1.0.0 // indirect 19 | golang.org/x/text v0.22.0 // indirect 20 | gopkg.in/yaml.v3 v3.0.1 // indirect 21 | ) 22 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= 2 | github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= 3 | github.com/cornelk/hashmap v1.0.8 h1:nv0AWgw02n+iDcawr5It4CjQIAcdMMKRrs10HOJYlrc= 4 | github.com/cornelk/hashmap v1.0.8/go.mod h1:RfZb7JO3RviW/rT6emczVuC/oxpdz4UsSB2LJSclR1k= 5 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 6 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 7 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 9 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 10 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 11 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 12 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 13 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 14 | github.com/samber/lo v1.51.0 h1:kysRYLbHy/MB7kQZf5DSN50JHmMsNEdeY24VzJFu7wI= 15 | github.com/samber/lo v1.51.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0= 16 | github.com/samber/slog-common v0.18.1 h1:c0EipD/nVY9HG5shgm/XAs67mgpWDMF+MmtptdJNCkQ= 17 | github.com/samber/slog-common v0.18.1/go.mod h1:QNZiNGKakvrfbJ2YglQXLCZauzkI9xZBjOhWFKS3IKk= 18 | github.com/samber/slog-multi v1.4.0 h1:pwlPMIE7PrbTHQyKWDU+RIoxP1+HKTNOujk3/kdkbdg= 19 | github.com/samber/slog-multi v1.4.0/go.mod h1:FsQ4Uv2L+E/8TZt+/BVgYZ1LoDWCbfCU21wVIoMMrO8= 20 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 21 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 22 | go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= 23 | go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= 24 | golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= 25 | golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= 26 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 27 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= 28 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 29 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 30 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 31 | -------------------------------------------------------------------------------- /hash.go: -------------------------------------------------------------------------------- 1 | package slogsampling 2 | 3 | // // fnv32a, adapted from "hash/fnv", but without a []byte(string) alloc 4 | // func fnv32a(s string) uint32 { 5 | // const ( 6 | // offset32 = 2166136261 7 | // prime32 = 16777619 8 | // ) 9 | // hash := uint32(offset32) 10 | // for i := 0; i < len(s); i++ { 11 | // hash ^= uint32(s[i]) 12 | // hash *= prime32 13 | // } 14 | // return hash 15 | // } 16 | -------------------------------------------------------------------------------- /hook.go: -------------------------------------------------------------------------------- 1 | package slogsampling 2 | 3 | import ( 4 | "context" 5 | 6 | "log/slog" 7 | ) 8 | 9 | func hook(hook func(context.Context, slog.Record), ctx context.Context, record slog.Record) { 10 | if hook != nil { 11 | hook(ctx, record) 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /main_test.go: -------------------------------------------------------------------------------- 1 | package slogsampling 2 | 3 | import ( 4 | "testing" 5 | 6 | "go.uber.org/goleak" 7 | ) 8 | 9 | func TestMain(m *testing.M) { 10 | goleak.VerifyTestMain(m) 11 | } 12 | -------------------------------------------------------------------------------- /matchers.go: -------------------------------------------------------------------------------- 1 | package slogsampling 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/gob" 7 | "fmt" 8 | "hash/fnv" 9 | "log/slog" 10 | "runtime" 11 | "strconv" 12 | 13 | slogcommon "github.com/samber/slog-common" 14 | ) 15 | 16 | var DefaultMatcher = MatchByLevelAndMessage() 17 | 18 | // Matcher is a function that returns a string hash for a given record. 19 | // Returning []byte would have been much much better, but go's hashmap doesn't support it. 🤬 20 | type Matcher func(context.Context, *slog.Record) string 21 | 22 | func MatchAll() func(context.Context, *slog.Record) string { 23 | return func(ctx context.Context, r *slog.Record) string { 24 | return "" 25 | } 26 | } 27 | 28 | func MatchByLevel() func(context.Context, *slog.Record) string { 29 | return func(ctx context.Context, r *slog.Record) string { 30 | return r.Level.String() 31 | } 32 | } 33 | 34 | func MatchByMessage() func(context.Context, *slog.Record) string { 35 | return func(ctx context.Context, r *slog.Record) string { 36 | return r.Message 37 | } 38 | } 39 | 40 | func MatchByLevelAndMessage() func(context.Context, *slog.Record) string { 41 | return func(ctx context.Context, r *slog.Record) string { 42 | // separator is used to avoid collisions 43 | return r.Level.String() + "@" + r.Message 44 | } 45 | } 46 | 47 | func MatchBySource() func(context.Context, *slog.Record) string { 48 | return func(ctx context.Context, r *slog.Record) string { 49 | fs := runtime.CallersFrames([]uintptr{r.PC}) 50 | f, _ := fs.Next() 51 | 52 | // separator is used to avoid collisions 53 | return fmt.Sprintf("%s@%d@%s", f.File, f.Line, f.Function) 54 | } 55 | } 56 | 57 | func MatchByAttribute(groups []string, key string) func(context.Context, *slog.Record) string { 58 | return func(ctx context.Context, r *slog.Record) string { 59 | var output string 60 | 61 | r.Attrs(func(attr slog.Attr) bool { 62 | attr, found := slogcommon.FindAttribute([]slog.Attr{attr}, groups, key) 63 | if found { 64 | value := attr.Value.Resolve().Any() 65 | output = anyToString(value) 66 | 67 | // if value is nil or empty, we keep looking for a matching attribute 68 | return len(output) > 0 69 | } 70 | 71 | return true 72 | }) 73 | 74 | return output 75 | } 76 | } 77 | 78 | func MatchByContextValue(key any) func(context.Context, *slog.Record) string { 79 | return func(ctx context.Context, r *slog.Record) string { 80 | return anyToString(ctx.Value(key)) 81 | } 82 | } 83 | 84 | func anyToString(value any) string { 85 | // no value 86 | if value == nil { 87 | return "" 88 | } 89 | 90 | // primitive types 91 | switch v := value.(type) { 92 | case []byte: 93 | return string(v) 94 | case string: 95 | return v 96 | case int, int64, int32, int16, int8: 97 | return strconv.FormatInt(value.(int64), 10) 98 | case uint, uint64, uint32, uint16, uint8: 99 | return strconv.FormatUint(value.(uint64), 10) 100 | case float64, float32: 101 | return strconv.FormatFloat(value.(float64), 'f', -1, 64) 102 | case bool: 103 | return strconv.FormatBool(value.(bool)) 104 | case complex128, complex64: 105 | return strconv.FormatComplex(value.(complex128), 'f', -1, 64) 106 | } 107 | 108 | // gob-encodable types 109 | var buf bytes.Buffer 110 | enc := gob.NewEncoder(&buf) 111 | 112 | // bearer:disable go_lang_deserialization_of_user_input 113 | err := enc.Encode(value) 114 | if err != nil { 115 | return fmt.Sprintf("%#v", value) 116 | } 117 | 118 | return buf.String() 119 | } 120 | 121 | func CompactionFNV32a(input string) string { 122 | hash := fnv.New32a() 123 | hash.Write([]byte(input)) 124 | return strconv.FormatInt(int64(hash.Sum32()), 10) 125 | } 126 | 127 | func CompactionFNV64a(input string) string { 128 | hash := fnv.New64a() 129 | hash.Write([]byte(input)) 130 | return strconv.FormatInt(int64(hash.Sum64()), 10) 131 | } 132 | 133 | func CompactionFNV128a(input string) string { 134 | return string(fnv.New128a().Sum([]byte(input))) 135 | } 136 | -------------------------------------------------------------------------------- /middleware_absolute.go: -------------------------------------------------------------------------------- 1 | package slogsampling 2 | 3 | import ( 4 | "context" 5 | "log/slog" 6 | "time" 7 | 8 | slogmulti "github.com/samber/slog-multi" 9 | "github.com/samber/slog-sampling/buffer" 10 | ) 11 | 12 | type AbsoluteSamplingOption struct { 13 | // This will log all entries with the same hash until max is reached, 14 | // in a `Tick` interval as-is. Following that, it will reduce log throughput 15 | // depending on previous interval. 16 | Tick time.Duration 17 | Max uint64 18 | 19 | // Group similar logs (default: by level and message) 20 | Matcher Matcher 21 | Buffer func(generator func(string) any) buffer.Buffer[string] 22 | buffer buffer.Buffer[string] 23 | 24 | // Optional hooks 25 | OnAccepted func(context.Context, slog.Record) 26 | OnDropped func(context.Context, slog.Record) 27 | } 28 | 29 | // NewMiddleware returns a slog-multi middleware. 30 | func (o AbsoluteSamplingOption) NewMiddleware() slogmulti.Middleware { 31 | if o.Max == 0 { 32 | panic("unexpected Max: must be greater than 0") 33 | } 34 | 35 | if o.Matcher == nil { 36 | o.Matcher = DefaultMatcher 37 | } 38 | 39 | if o.Buffer == nil { 40 | o.Buffer = buffer.NewUnlimitedBuffer[string]() 41 | } 42 | 43 | o.buffer = o.Buffer(func(k string) any { 44 | return newCounterWithMemory() 45 | }) 46 | 47 | return slogmulti.NewInlineMiddleware( 48 | func(ctx context.Context, level slog.Level, next func(context.Context, slog.Level) bool) bool { 49 | return next(ctx, level) 50 | }, 51 | func(ctx context.Context, record slog.Record, next func(context.Context, slog.Record) error) error { 52 | key := o.Matcher(ctx, &record) 53 | c, _ := o.buffer.GetOrInsert(key) 54 | n, p := c.(*counterWithMemory).Inc(o.Tick) 55 | 56 | random, err := randomPercentage(1000) // 0.001 precision 57 | if err != nil { 58 | return err 59 | } 60 | 61 | // 3 cases: 62 | // - current interval is over threshold but not previous -> drop 63 | // - previous interval is over threshold -> apply rate limit 64 | // - none of current and previous intervals are over threshold -> accept 65 | 66 | if (n > o.Max && p <= o.Max) || (p > o.Max && random >= float64(o.Max)/float64(p)) { 67 | hook(o.OnDropped, ctx, record) 68 | return nil 69 | } 70 | 71 | hook(o.OnAccepted, ctx, record) 72 | return next(ctx, record) 73 | }, 74 | func(attrs []slog.Attr, next func([]slog.Attr) slog.Handler) slog.Handler { 75 | return next(attrs) 76 | }, 77 | func(name string, next func(string) slog.Handler) slog.Handler { 78 | return next(name) 79 | }, 80 | ) 81 | } 82 | -------------------------------------------------------------------------------- /middleware_custom.go: -------------------------------------------------------------------------------- 1 | package slogsampling 2 | 3 | import ( 4 | "context" 5 | 6 | "log/slog" 7 | 8 | slogmulti "github.com/samber/slog-multi" 9 | ) 10 | 11 | type CustomSamplingOption struct { 12 | // The sample rate for sampling traces in the range [0.0, 1.0]. 13 | Sampler func(context.Context, slog.Record) float64 14 | 15 | // Optional hooks 16 | OnAccepted func(context.Context, slog.Record) 17 | OnDropped func(context.Context, slog.Record) 18 | } 19 | 20 | // NewMiddleware returns a slog-multi middleware. 21 | func (o CustomSamplingOption) NewMiddleware() slogmulti.Middleware { 22 | return slogmulti.NewInlineMiddleware( 23 | func(ctx context.Context, level slog.Level, next func(context.Context, slog.Level) bool) bool { 24 | return next(ctx, level) 25 | }, 26 | func(ctx context.Context, record slog.Record, next func(context.Context, slog.Record) error) error { 27 | rate := o.Sampler(ctx, record) 28 | if rate < 0.0 || rate > 1.0 { 29 | // unexpected rate: we just drop 30 | hook(o.OnDropped, ctx, record) 31 | return nil 32 | } 33 | 34 | random, err := randomPercentage(1000) // 0.001 precision 35 | if err != nil { 36 | return err 37 | } 38 | 39 | if random >= rate { 40 | hook(o.OnDropped, ctx, record) 41 | return nil 42 | } 43 | 44 | hook(o.OnAccepted, ctx, record) 45 | return next(ctx, record) 46 | }, 47 | func(attrs []slog.Attr, next func([]slog.Attr) slog.Handler) slog.Handler { 48 | return next(attrs) 49 | }, 50 | func(name string, next func(string) slog.Handler) slog.Handler { 51 | return next(name) 52 | }, 53 | ) 54 | } 55 | -------------------------------------------------------------------------------- /middleware_threshold.go: -------------------------------------------------------------------------------- 1 | package slogsampling 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "log/slog" 8 | 9 | slogmulti "github.com/samber/slog-multi" 10 | "github.com/samber/slog-sampling/buffer" 11 | ) 12 | 13 | type ThresholdSamplingOption struct { 14 | // This will log the first `Threshold` log entries with the same hash, 15 | // in a `Tick` interval as-is. Following that, it will allow `Rate` in the range [0.0, 1.0]. 16 | Tick time.Duration 17 | Threshold uint64 18 | Rate float64 19 | 20 | // Group similar logs (default: by level and message) 21 | Matcher Matcher 22 | Buffer func(generator func(string) any) buffer.Buffer[string] 23 | buffer buffer.Buffer[string] 24 | 25 | // Optional hooks 26 | OnAccepted func(context.Context, slog.Record) 27 | OnDropped func(context.Context, slog.Record) 28 | } 29 | 30 | // NewMiddleware returns a slog-multi middleware. 31 | func (o ThresholdSamplingOption) NewMiddleware() slogmulti.Middleware { 32 | if o.Rate < 0.0 || o.Rate > 1.0 { 33 | panic("unexpected Rate: must be between 0.0 and 1.0") 34 | } 35 | 36 | if o.Matcher == nil { 37 | o.Matcher = DefaultMatcher 38 | } 39 | 40 | if o.Buffer == nil { 41 | o.Buffer = buffer.NewUnlimitedBuffer[string]() 42 | } 43 | 44 | o.buffer = o.Buffer(func(k string) any { 45 | return newCounter() 46 | }) 47 | 48 | return slogmulti.NewInlineMiddleware( 49 | func(ctx context.Context, level slog.Level, next func(context.Context, slog.Level) bool) bool { 50 | return next(ctx, level) 51 | }, 52 | func(ctx context.Context, record slog.Record, next func(context.Context, slog.Record) error) error { 53 | key := o.Matcher(ctx, &record) 54 | c, _ := o.buffer.GetOrInsert(key) 55 | n := c.(*counter).Inc(o.Tick) 56 | 57 | random, err := randomPercentage(1000) // 0.001 precision 58 | if err != nil { 59 | return err 60 | } 61 | 62 | if n > o.Threshold && random >= o.Rate { 63 | hook(o.OnDropped, ctx, record) 64 | return nil 65 | } 66 | 67 | hook(o.OnAccepted, ctx, record) 68 | return next(ctx, record) 69 | }, 70 | func(attrs []slog.Attr, next func([]slog.Attr) slog.Handler) slog.Handler { 71 | return next(attrs) 72 | }, 73 | func(name string, next func(string) slog.Handler) slog.Handler { 74 | return next(name) 75 | }, 76 | ) 77 | } 78 | -------------------------------------------------------------------------------- /middleware_uniform.go: -------------------------------------------------------------------------------- 1 | package slogsampling 2 | 3 | import ( 4 | "context" 5 | 6 | "log/slog" 7 | 8 | slogmulti "github.com/samber/slog-multi" 9 | ) 10 | 11 | type UniformSamplingOption struct { 12 | // The sample rate for sampling traces in the range [0.0, 1.0]. 13 | Rate float64 14 | 15 | // Optional hooks 16 | OnAccepted func(context.Context, slog.Record) 17 | OnDropped func(context.Context, slog.Record) 18 | } 19 | 20 | // NewMiddleware returns a slog-multi middleware. 21 | func (o UniformSamplingOption) NewMiddleware() slogmulti.Middleware { 22 | if o.Rate < 0.0 || o.Rate > 1.0 { 23 | panic("unexpected Rate: must be between 0.0 and 1.0") 24 | } 25 | 26 | return slogmulti.NewInlineMiddleware( 27 | func(ctx context.Context, level slog.Level, next func(context.Context, slog.Level) bool) bool { 28 | return next(ctx, level) 29 | }, 30 | func(ctx context.Context, record slog.Record, next func(context.Context, slog.Record) error) error { 31 | random, err := randomPercentage(1000) // 0.001 precision 32 | if err != nil { 33 | return err 34 | } 35 | 36 | if random >= o.Rate { 37 | hook(o.OnDropped, ctx, record) 38 | return nil 39 | } 40 | 41 | hook(o.OnAccepted, ctx, record) 42 | return next(ctx, record) 43 | }, 44 | func(attrs []slog.Attr, next func([]slog.Attr) slog.Handler) slog.Handler { 45 | return next(attrs) 46 | }, 47 | func(name string, next func(string) slog.Handler) slog.Handler { 48 | return next(name) 49 | }, 50 | ) 51 | } 52 | -------------------------------------------------------------------------------- /middleware_uniform_test.go: -------------------------------------------------------------------------------- 1 | package slogsampling 2 | 3 | import ( 4 | "bytes" 5 | "log/slog" 6 | "sync" 7 | "testing" 8 | 9 | slogmulti "github.com/samber/slog-multi" 10 | ) 11 | 12 | // This test previously failed with the race detector 13 | func TestUniformRace(t *testing.T) { 14 | const numGoroutines = 100 15 | 16 | buf := &bytes.Buffer{} 17 | textLogHandler := slog.NewTextHandler(buf, nil) 18 | sampleMiddleware := UniformSamplingOption{Rate: 0.2}.NewMiddleware() 19 | sampledLogger := slog.New(slogmulti.Pipe(sampleMiddleware).Handler(textLogHandler)) 20 | 21 | wg := &sync.WaitGroup{} 22 | wg.Add(numGoroutines) 23 | for i := 0; i < numGoroutines; i++ { 24 | goroutineIndex := i 25 | go func() { 26 | defer wg.Done() 27 | sampledLogger.Info("mesage from goroutine", "goroutineIndex", goroutineIndex) 28 | }() 29 | } 30 | wg.Wait() 31 | 32 | // numLines should be in exclusive range (0, numGoroutines) 33 | // this is probabilistic so it might fail but is pretty unlikely 34 | numLines := bytes.Count(buf.Bytes(), []byte("\n")) 35 | if 0 >= numLines || numLines >= numGoroutines { 36 | t.Errorf("numLines=%d; should be in exclusive range (0, %d)", numLines, numGoroutines) 37 | t.Error("raw output:") 38 | t.Error(buf.String()) 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /random.go: -------------------------------------------------------------------------------- 1 | package slogsampling 2 | 3 | import ( 4 | "crypto/rand" 5 | "math/big" 6 | ) 7 | 8 | func randomPercentage(precision int64) (float64, error) { 9 | random, err := rand.Int(rand.Reader, big.NewInt(precision)) 10 | if err != nil { 11 | return 0, err 12 | } 13 | 14 | return float64(random.Int64()) / float64(precision), nil 15 | } 16 | -------------------------------------------------------------------------------- /random_test.go: -------------------------------------------------------------------------------- 1 | package slogsampling 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestRandomPercentage(t *testing.T) { 10 | is := assert.New(t) 11 | 12 | for i := 1; i < 10000; i++ { 13 | r, err := randomPercentage(int64(i)) 14 | is.NoError(err) 15 | is.True(r >= 0 && r < 1) 16 | } 17 | } 18 | --------------------------------------------------------------------------------