├── .github └── workflows │ └── golangci-lint.yml ├── .golangci.yml ├── LICENSE ├── README.md ├── api_test.go ├── circonusllhist.go ├── circonusllhist_test.go └── go.mod /.github/workflows/golangci-lint.yml: -------------------------------------------------------------------------------- 1 | name: golangci-lint 2 | on: 3 | push: 4 | tags: [ "v*" ] 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ "*" ] 8 | 9 | jobs: 10 | golangci: 11 | name: lint 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/setup-go@v2 15 | with: 16 | stable: true 17 | go-version: 1.16.x 18 | - uses: actions/checkout@v2 19 | - name: golangci-lint 20 | uses: golangci/golangci-lint-action@v2 21 | with: 22 | version: v1.40 23 | skip-go-installation: true 24 | args: --timeout=5m 25 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | run: 2 | concurrency: 4 3 | issues-exit-code: 1 4 | tests: true 5 | skip-dirs-use-default: true 6 | skip-files: 7 | - ".*_mock_test.go$" 8 | allow-parallel-runners: true 9 | 10 | # all available settings of specific linters 11 | linters-settings: 12 | govet: 13 | check-shadowing: true 14 | enable-all: true 15 | gofmt: 16 | simplify: true 17 | gosec: 18 | excludes: 19 | - G404 20 | goimports: 21 | local-prefixes: github.com/circonus-labs,github.com/openhistogram,github.com/circonus 22 | misspell: 23 | locale: US 24 | unused: 25 | check-exported: false 26 | unparam: 27 | check-exported: false 28 | staticcheck: 29 | go: "1.16" 30 | # https://staticcheck.io/docs/options#checks 31 | checks: [ "all", "-ST1017" ] 32 | stylecheck: 33 | go: "1.16" 34 | # https://staticcheck.io/docs/options#checks 35 | checks: [ "all", "-ST1017" ] 36 | 37 | linters: 38 | enable: 39 | - deadcode 40 | - errcheck 41 | - gocritic 42 | - gofmt 43 | - gosec 44 | - gosimple 45 | - govet 46 | - ineffassign 47 | - megacheck 48 | - misspell 49 | - prealloc 50 | - staticcheck 51 | - structcheck 52 | - typecheck 53 | - unparam 54 | - unused 55 | - varcheck 56 | - gci 57 | - godot 58 | - godox 59 | - goerr113 60 | - predeclared 61 | - unconvert 62 | - wrapcheck 63 | - revive 64 | - exportloopref 65 | - asciicheck 66 | - errorlint 67 | - wrapcheck 68 | - goconst 69 | #- stylecheck 70 | - forcetypeassert 71 | - goimports 72 | disable: 73 | - scopelint # deprecated 74 | - golint # deprecated 75 | - maligned # deprecated 76 | disable-all: false 77 | presets: 78 | - bugs 79 | - unused 80 | fast: false 81 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2012-2022 Circonus, Inc. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # circonusllhist 2 | 3 | A golang implementation of the OpenHistogram [libcircllhist](https://github.com/openhistogram/libcircllhist) library. 4 | 5 | [![godocs.io](http://godocs.io/github.com/openhistogram/circonusllhist?status.svg)](http://godocs.io/github.com/openhistogram/circonusllhist) 6 | 7 | 8 | ## Overview 9 | 10 | Package `circllhist` provides an implementation of OpenHistogram's fixed log-linear histogram data structure. This allows tracking of histograms in a composable way such that accurate error can be reasoned about. 11 | 12 | ## License 13 | 14 | [Apache 2.0](LICENSE) 15 | 16 | ## Documentation 17 | 18 | More complete docs can be found at [godoc](https://godocs.io/github.com/openhistogram/circonusllhist) or [pkg.go.dev](https://pkg.go.dev/github.com/openhistogram/circonusllhist) 19 | 20 | ## Usage Example 21 | 22 | ```go 23 | package main 24 | 25 | import ( 26 | "fmt" 27 | 28 | "github.com/openhistogram/circonusllhist" 29 | ) 30 | 31 | func main() { 32 | //Create a new histogram 33 | h := circonusllhist.New() 34 | 35 | //Insert value 123, three times 36 | if err := h.RecordValues(123, 3); err != nil { 37 | panic(err) 38 | } 39 | 40 | //Insert 1x10^1 41 | if err := h.RecordIntScale(1, 1); err != nil { 42 | panic(err) 43 | } 44 | 45 | //Print the count of samples stored in the histogram 46 | fmt.Printf("%d\n", h.Count()) 47 | 48 | //Print the sum of all samples 49 | fmt.Printf("%f\n", h.ApproxSum()) 50 | } 51 | ``` 52 | 53 | ### Usage Without Lookup Tables 54 | 55 | By default, bi-level sparse lookup tables are used in this OpenHistogram implementation to improve insertion time by about 20%. However, the size of these tables ranges from a minimum of ~0.5KiB to a maximum of ~130KiB. While usage nearing the theoretical maximum is unlikely, as the lookup tables are kept as sparse tables, normal usage will be above the minimum. For applications where insertion time is not the most important factor and memory efficiency is, especially when datasets contain large numbers of individual histograms, opting out of the lookup tables is an appropriate choice. Generate new histograms without lookup tables like: 56 | 57 | ```go 58 | package main 59 | 60 | import "github.com/openhistogram/circonusllhist" 61 | 62 | func main() { 63 | //Create a new histogram without lookup tables 64 | h := circonusllhist.New(circonusllhist.NoLookup()) 65 | // ... 66 | } 67 | ``` 68 | 69 | #### Notes on Serialization 70 | 71 | When intentionally working without lookup tables, care must be taken to correctly serialize and deserialize the histogram data. The following example creates a histogram without lookup tables, serializes and deserializes it manually while never allocating any excess memory: 72 | 73 | ```go 74 | package main 75 | 76 | import ( 77 | "bytes" 78 | "fmt" 79 | 80 | "github.com/openhistogram/circonusllhist" 81 | ) 82 | 83 | func main() { 84 | // create a new histogram without lookup tables 85 | h := circonusllhist.New(circonusllhist.NoLookup()) 86 | if err := h.RecordValue(1.2); err != nil { 87 | panic(err) 88 | } 89 | 90 | // serialize the histogram 91 | var buf bytes.Buffer 92 | if err := h.Serialize(&buf); err != nil { 93 | panic(err) 94 | } 95 | 96 | // deserialize into a new histogram 97 | h2, err := circonusllhist.DeserializeWithOptions(&buf, circonusllhist.NoLookup()) 98 | if err != nil { 99 | panic(err) 100 | } 101 | 102 | // the two histograms are equal 103 | fmt.Println(h.Equals(h2)) 104 | } 105 | ``` 106 | 107 | While the example above works cleanly when manual (de)serialization is required, a different approach is needed when implicitly (de)serializing histograms into a JSON format. The following example creates a histogram without lookup tables, serializes and deserializes it implicitly using Go's JSON library, ensuring no excess memory allocations occur: 108 | 109 | ```go 110 | package main 111 | 112 | import ( 113 | "encoding/json" 114 | "fmt" 115 | 116 | "github.com/openhistogram/circonusllhist" 117 | ) 118 | 119 | func main() { 120 | // create a new histogram without lookup tables 121 | h := circonusllhist.New(circonusllhist.NoLookup()) 122 | if err := h.RecordValue(1.2); err != nil { 123 | panic(err) 124 | } 125 | 126 | // serialize the histogram 127 | data, err := json.Marshal(h) 128 | if err != nil { 129 | panic(err) 130 | } 131 | 132 | // deserialize into a new histogram 133 | var wrapper2 circonusllhist.HistogramWithoutLookups 134 | if err := json.Unmarshal(data, &wrapper2); err != nil { 135 | panic(err) 136 | } 137 | h2 := wrapper2.Histogram() 138 | 139 | // the two histograms are equal 140 | fmt.Println(h.Equals(h2)) 141 | } 142 | ``` 143 | 144 | Once the `circonusllhist.HistogramWithoutLookups` wrapper has been used as a deserialization target, the underlying histogram may be extracted with the `Histogram()` method. It is also possible to extract the histogram while allocating memory for lookup tables if necessary with the `HistogramWithLookups()` method. 145 | -------------------------------------------------------------------------------- /api_test.go: -------------------------------------------------------------------------------- 1 | package circonusllhist_test 2 | 3 | import ( 4 | "math" 5 | "math/rand" 6 | "testing" 7 | "time" 8 | 9 | hist "github.com/openhistogram/circonusllhist" 10 | ) 11 | 12 | func fuzzyEquals(expected, actual float64) bool { 13 | delta := math.Abs(expected / 100000.0) 14 | if actual >= expected-delta && actual <= expected+delta { 15 | return true 16 | } 17 | return false 18 | } 19 | 20 | var s1 = []float64{0.123, 0, 0.43, 0.41, 0.415, 0.2201, 0.3201, 0.125, 0.13} 21 | 22 | func TestDecStrings(t *testing.T) { 23 | h := hist.New() 24 | for _, sample := range s1 { 25 | _ = h.RecordValue(sample) 26 | } 27 | out := h.DecStrings() 28 | expect := []string{"H[0.0e+00]=1", "H[1.2e-01]=2", "H[1.3e-01]=1", 29 | "H[2.2e-01]=1", "H[3.2e-01]=1", "H[4.1e-01]=2", 30 | "H[4.3e-01]=1"} 31 | for i, str := range expect { 32 | if str != out[i] { 33 | t.Errorf("DecString '%v' != '%v'", out[i], str) 34 | } 35 | } 36 | } 37 | 38 | func TestNewFromStrings(t *testing.T) { 39 | strings := []string{"H[0.0e+00]=1", "H[1.2e-01]=2", "H[1.3e-01]=1", 40 | "H[2.2e-01]=1", "H[3.2e-01]=1", "H[4.1e-01]=2", "H[4.3e-01]=1"} 41 | 42 | // hist of single set of strings 43 | singleHist, err := hist.NewFromStrings(strings, false) 44 | if err != nil { 45 | t.Errorf("error creating hist from strings '%v'", err) 46 | } 47 | 48 | // hist of multiple sets of strings 49 | strings = append(strings, strings...) 50 | doubleHist, err := hist.NewFromStrings(strings, false) 51 | if err != nil { 52 | t.Errorf("error creating hist from strings '%v'", err) 53 | } 54 | 55 | // sanity check the sums are doubled 56 | if singleHist.ApproxSum()*2 != doubleHist.ApproxSum() { 57 | t.Error("aggregate histogram approxSum failure") 58 | } 59 | 60 | if singleHist.Equals(doubleHist) { 61 | t.Error("histograms should not be equal") 62 | } 63 | } 64 | 65 | func TestMean(t *testing.T) { 66 | h := hist.New() 67 | for _, sample := range s1 { 68 | _ = h.RecordValue(sample) 69 | } 70 | mean := h.ApproxMean() 71 | if !fuzzyEquals(0.2444444444, mean) { 72 | t.Errorf("mean() -> %v != %v", mean, 0.24444) 73 | } 74 | } 75 | 76 | func helpQTest(t *testing.T, vals, qin, qexpect []float64) { 77 | h := hist.New() 78 | for _, sample := range vals { 79 | _ = h.RecordValue(sample) 80 | } 81 | qout, _ := h.ApproxQuantile(qin) 82 | if len(qout) != len(qexpect) { 83 | t.Errorf("wrong number of quantiles") 84 | } 85 | for i, q := range qout { 86 | if !fuzzyEquals(qexpect[i], q) { 87 | t.Errorf("q(%v) -> %v != %v", qin[i], q, qexpect[i]) 88 | } 89 | } 90 | } 91 | 92 | func TestQuantiles(t *testing.T) { 93 | helpQTest(t, []float64{1}, []float64{0, 0.25, 0.5, 1}, []float64{1, 1.025, 1.05, 1.1}) 94 | helpQTest(t, s1, []float64{0, 0.95, 0.99, 1.0}, []float64{0, 0.4355, 0.4391, 0.44}) 95 | helpQTest(t, []float64{1.0, 2.0}, []float64{0.5}, []float64{1.1}) 96 | helpQTest(t, []float64{1.0, 1e200}, []float64{0, 1}, []float64{1.0, 1.1}) 97 | helpQTest(t, []float64{1e200, 1e200, 1e200, 0, 0, 1e-20, 1e-20, 1e-20, 1e-10}, []float64{0, 1}, 98 | []float64{0, 1.1e-10}) 99 | helpQTest(t, []float64{0, 1}, []float64{0, 0.1}, []float64{0, 0}) 100 | } 101 | 102 | func BenchmarkHistogramRecordValue(b *testing.B) { 103 | h := hist.New(hist.NoLocks()) 104 | for i := 0; i < b.N; i++ { 105 | _ = h.RecordValue(float64(i % 1000)) 106 | } 107 | b.ReportAllocs() 108 | } 109 | 110 | func BenchmarkHistogramTypical(b *testing.B) { 111 | h := hist.New(hist.NoLocks()) 112 | for i := 0; i < b.N; i++ { 113 | _ = h.RecordValue(float64(i % 1000)) 114 | } 115 | b.ReportAllocs() 116 | } 117 | 118 | func BenchmarkHistogramRecordIntScale(b *testing.B) { 119 | h := hist.New(hist.NoLocks()) 120 | for i := 0; i < b.N; i++ { 121 | _ = h.RecordIntScale(int64(i%90+10), (i/1000)%3) 122 | } 123 | b.ReportAllocs() 124 | } 125 | 126 | func BenchmarkHistogramTypicalIntScale(b *testing.B) { 127 | h := hist.New(hist.NoLocks()) 128 | for i := 0; i < b.N; i++ { 129 | _ = h.RecordIntScale(int64(i%90+10), (i/1000)%3) 130 | } 131 | b.ReportAllocs() 132 | } 133 | 134 | func BenchmarkNew(b *testing.B) { 135 | b.ReportAllocs() 136 | 137 | for i := 0; i < b.N; i++ { 138 | hist.New() 139 | } 140 | } 141 | 142 | func TestCompare(t *testing.T) { 143 | // var h1, h2 *Bin 144 | } 145 | 146 | func TestConcurrent(t *testing.T) { 147 | h := hist.New() 148 | for r := 0; r < 100; r++ { 149 | go func(t *testing.T) { 150 | for j := 0; j < 100; j++ { 151 | for i := 50; i < 100; i++ { 152 | if err := h.RecordValue(float64(i)); err != nil { 153 | t.Error(err) 154 | return 155 | } 156 | } 157 | } 158 | }(t) 159 | } 160 | } 161 | 162 | func TestRang(t *testing.T) { 163 | h1 := hist.New() 164 | rnd := rand.New(rand.NewSource(time.Now().UnixNano())) 165 | for i := 0; i < 1000000; i++ { 166 | _ = h1.RecordValue(rnd.Float64() * 10) 167 | } 168 | } 169 | 170 | func TestEquals(t *testing.T) { 171 | h1 := hist.New() 172 | for i := 0; i < 1000000; i++ { 173 | if err := h1.RecordValue(float64(i)); err != nil { 174 | t.Fatal(err) 175 | } 176 | } 177 | 178 | h2 := hist.New() 179 | for i := 0; i < 10000; i++ { 180 | if err := h1.RecordValue(float64(i)); err != nil { 181 | t.Fatal(err) 182 | } 183 | } 184 | 185 | if h1.Equals(h2) { 186 | t.Error("Expected Histograms to not be equivalent") 187 | } 188 | 189 | h1.Reset() 190 | h2.Reset() 191 | 192 | if !h1.Equals(h2) { 193 | t.Error("Expected Histograms to be equivalent") 194 | } 195 | } 196 | 197 | func TestMinMaxMean(t *testing.T) { 198 | const ( 199 | minVal = 0 200 | maxVal = 1000000 201 | ) 202 | 203 | h := hist.New() 204 | for i := minVal; i < maxVal; i++ { 205 | if err := h.RecordValue(float64(i)); err != nil { 206 | t.Fatal(err) 207 | } 208 | } 209 | 210 | if h.Min() > minVal { 211 | t.Error("incorrect min value") 212 | } 213 | 214 | if h.Max() < maxVal { 215 | t.Error("incorrect max value") 216 | } 217 | 218 | round := func(val float64) int { 219 | if val < 0 { 220 | return int(val - 0.5) 221 | } 222 | return int(val + 0.5) 223 | } 224 | 225 | if round(h.Mean()) != round(maxVal/2) { 226 | t.Errorf("incorrect mean value") 227 | } 228 | } 229 | 230 | func TestCopy(t *testing.T) { 231 | h1 := hist.New() 232 | for i := 0; i < 1000000; i++ { 233 | if err := h1.RecordValue(float64(i)); err != nil { 234 | t.Fatal(err) 235 | } 236 | } 237 | 238 | h2 := h1.Copy() 239 | if !h2.Equals(h1) { 240 | t.Errorf("expected copy: %v to equal original: %v", h2, h1) 241 | } 242 | } 243 | 244 | func TestFullReset(t *testing.T) { 245 | h1 := hist.New() 246 | for i := 0; i < 1000000; i++ { 247 | if err := h1.RecordValue(float64(i)); err != nil { 248 | t.Fatal(err) 249 | } 250 | } 251 | 252 | h1.Reset() 253 | h2 := hist.New() 254 | if !h2.Equals(h1) { 255 | t.Errorf("expected reset value: %v to equal new value: %v", h1, h2) 256 | } 257 | } 258 | 259 | func TestMerge(t *testing.T) { 260 | h1 := hist.New() 261 | h2 := hist.New() 262 | expect := hist.New() 263 | 264 | // record 0-100 values in both h1 and h2. 265 | for i := 0; i < 100; i++ { 266 | if err := h1.RecordValues(float64(i), 1); err != nil { 267 | t.Fatal(err) 268 | } 269 | if err := h2.RecordValues(float64(i), 2); err != nil { 270 | t.Fatal(err) 271 | } 272 | if err := expect.RecordValues(float64(i), 3); err != nil { 273 | t.Fatal(err) 274 | } 275 | } 276 | // record 100-200 values in h1. 277 | for i := 100; i < 200; i++ { 278 | if err := h1.RecordValues(float64(i), 1); err != nil { 279 | t.Fatal(err) 280 | } 281 | if err := expect.RecordValues(float64(i), 1); err != nil { 282 | t.Fatal(err) 283 | } 284 | } 285 | // record 400-600 values in h2. 286 | for i := 400; i < 600; i++ { 287 | if err := h2.RecordValues(float64(i), 1); err != nil { 288 | t.Fatal(err) 289 | } 290 | if err := expect.RecordValues(float64(i), 1); err != nil { 291 | t.Fatal(err) 292 | } 293 | } 294 | 295 | h1.Merge(h2) 296 | if !h1.Equals(expect) { 297 | t.Error("Expected histograms to be equivalent") 298 | } 299 | } 300 | 301 | func BenchmarkHistogramMerge(b *testing.B) { 302 | b.Run("random", func(b *testing.B) { 303 | rand.New(rand.NewSource(time.Now().UnixNano())) 304 | b.ReportAllocs() 305 | for i := 0; i < b.N; i++ { 306 | h1 := hist.New() 307 | for i := 0; i < 500; i++ { 308 | _ = h1.RecordIntScale(rand.Int63n(1000), 0) 309 | } 310 | h2 := hist.New() 311 | for i := 0; i < 500; i++ { 312 | _ = h2.RecordIntScale(rand.Int63n(1000), 0) 313 | } 314 | h1.Merge(h2) 315 | } 316 | }) 317 | 318 | b.Run("large insert", func(b *testing.B) { 319 | b.ReportAllocs() 320 | for i := 0; i < b.N; i++ { 321 | h1 := hist.New() 322 | _ = h1.RecordIntScale(1, 0) 323 | _ = h1.RecordIntScale(1000, 0) 324 | h2 := hist.New() 325 | for i := 10; i < 1000; i++ { 326 | _ = h2.RecordIntScale(int64(i), 0) 327 | } 328 | h1.Merge(h2) 329 | } 330 | }) 331 | } 332 | -------------------------------------------------------------------------------- /circonusllhist.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016, Circonus, Inc. All rights reserved. 2 | // See the LICENSE file. 3 | 4 | // Package circllhist provides an implementation of Circonus' fixed log-linear 5 | // histogram data structure. This allows tracking of histograms in a 6 | // composable way such that accurate error can be reasoned about. 7 | package circonusllhist 8 | 9 | import ( 10 | "bytes" 11 | "encoding/base64" 12 | "encoding/binary" 13 | "encoding/json" 14 | "fmt" 15 | "io" 16 | "math" 17 | "strconv" 18 | "strings" 19 | "sync" 20 | "time" 21 | ) 22 | 23 | const ( 24 | defaultHistSize = uint16(100) 25 | ) 26 | 27 | var powerOfTen = [...]float64{ 28 | 1, 10, 100, 1000, 10000, 100000, 1e+06, 1e+07, 1e+08, 1e+09, 1e+10, 29 | 1e+11, 1e+12, 1e+13, 1e+14, 1e+15, 1e+16, 1e+17, 1e+18, 1e+19, 1e+20, 30 | 1e+21, 1e+22, 1e+23, 1e+24, 1e+25, 1e+26, 1e+27, 1e+28, 1e+29, 1e+30, 31 | 1e+31, 1e+32, 1e+33, 1e+34, 1e+35, 1e+36, 1e+37, 1e+38, 1e+39, 1e+40, 32 | 1e+41, 1e+42, 1e+43, 1e+44, 1e+45, 1e+46, 1e+47, 1e+48, 1e+49, 1e+50, 33 | 1e+51, 1e+52, 1e+53, 1e+54, 1e+55, 1e+56, 1e+57, 1e+58, 1e+59, 1e+60, 34 | 1e+61, 1e+62, 1e+63, 1e+64, 1e+65, 1e+66, 1e+67, 1e+68, 1e+69, 1e+70, 35 | 1e+71, 1e+72, 1e+73, 1e+74, 1e+75, 1e+76, 1e+77, 1e+78, 1e+79, 1e+80, 36 | 1e+81, 1e+82, 1e+83, 1e+84, 1e+85, 1e+86, 1e+87, 1e+88, 1e+89, 1e+90, 37 | 1e+91, 1e+92, 1e+93, 1e+94, 1e+95, 1e+96, 1e+97, 1e+98, 1e+99, 1e+100, 38 | 1e+101, 1e+102, 1e+103, 1e+104, 1e+105, 1e+106, 1e+107, 1e+108, 1e+109, 39 | 1e+110, 1e+111, 1e+112, 1e+113, 1e+114, 1e+115, 1e+116, 1e+117, 1e+118, 40 | 1e+119, 1e+120, 1e+121, 1e+122, 1e+123, 1e+124, 1e+125, 1e+126, 1e+127, 41 | 1e-128, 1e-127, 1e-126, 1e-125, 1e-124, 1e-123, 1e-122, 1e-121, 1e-120, 42 | 1e-119, 1e-118, 1e-117, 1e-116, 1e-115, 1e-114, 1e-113, 1e-112, 1e-111, 43 | 1e-110, 1e-109, 1e-108, 1e-107, 1e-106, 1e-105, 1e-104, 1e-103, 1e-102, 44 | 1e-101, 1e-100, 1e-99, 1e-98, 1e-97, 1e-96, 45 | 1e-95, 1e-94, 1e-93, 1e-92, 1e-91, 1e-90, 1e-89, 1e-88, 1e-87, 1e-86, 46 | 1e-85, 1e-84, 1e-83, 1e-82, 1e-81, 1e-80, 1e-79, 1e-78, 1e-77, 1e-76, 47 | 1e-75, 1e-74, 1e-73, 1e-72, 1e-71, 1e-70, 1e-69, 1e-68, 1e-67, 1e-66, 48 | 1e-65, 1e-64, 1e-63, 1e-62, 1e-61, 1e-60, 1e-59, 1e-58, 1e-57, 1e-56, 49 | 1e-55, 1e-54, 1e-53, 1e-52, 1e-51, 1e-50, 1e-49, 1e-48, 1e-47, 1e-46, 50 | 1e-45, 1e-44, 1e-43, 1e-42, 1e-41, 1e-40, 1e-39, 1e-38, 1e-37, 1e-36, 51 | 1e-35, 1e-34, 1e-33, 1e-32, 1e-31, 1e-30, 1e-29, 1e-28, 1e-27, 1e-26, 52 | 1e-25, 1e-24, 1e-23, 1e-22, 1e-21, 1e-20, 1e-19, 1e-18, 1e-17, 1e-16, 53 | 1e-15, 1e-14, 1e-13, 1e-12, 1e-11, 1e-10, 1e-09, 1e-08, 1e-07, 1e-06, 54 | 1e-05, 0.0001, 0.001, 0.01, 0.1, 55 | } 56 | 57 | // A Bracket is a part of a cumulative distribution. 58 | type bin struct { 59 | count uint64 60 | val int8 61 | exp int8 62 | } 63 | 64 | func newBinRaw(val int8, exp int8, count uint64) *bin { 65 | return &bin{ 66 | count: count, 67 | val: val, 68 | exp: exp, 69 | } 70 | } 71 | 72 | // func newBin() *bin { 73 | // return newBinRaw(0, 0, 0) 74 | // } 75 | 76 | func newBinFromFloat64(d float64) *bin { 77 | hb := newBinRaw(0, 0, 0) 78 | hb.setFromFloat64(d) 79 | return hb 80 | } 81 | 82 | type fastL2 struct { 83 | l1, l2 int 84 | } 85 | 86 | func (hb *bin) newFastL2() fastL2 { 87 | return fastL2{l1: int(uint8(hb.exp)), l2: int(uint8(hb.val))} 88 | } 89 | 90 | func (hb *bin) setFromFloat64(d float64) *bin { //nolint:unparam 91 | hb.val = -1 92 | if math.IsInf(d, 0) || math.IsNaN(d) { 93 | return hb 94 | } 95 | if d == 0.0 { 96 | hb.val = 0 97 | return hb 98 | } 99 | sign := 1 100 | if math.Signbit(d) { 101 | sign = -1 102 | } 103 | d = math.Abs(d) 104 | bigExp := int(math.Floor(math.Log10(d))) 105 | hb.exp = int8(bigExp) 106 | if int(hb.exp) != bigExp { // rolled 107 | hb.exp = 0 108 | if bigExp < 0 { 109 | hb.val = 0 110 | } 111 | return hb 112 | } 113 | d /= hb.powerOfTen() 114 | d *= 10 115 | hb.val = int8(sign * int(math.Floor(d+1e-13))) 116 | if hb.val == 100 || hb.val == -100 { 117 | if hb.exp < 127 { 118 | hb.val /= 10 119 | hb.exp++ 120 | } else { 121 | hb.val = 0 122 | hb.exp = 0 123 | } 124 | } 125 | if hb.val == 0 { 126 | hb.exp = 0 127 | return hb 128 | } 129 | if !((hb.val >= 10 && hb.val < 100) || 130 | (hb.val <= -10 && hb.val > -100)) { 131 | hb.val = -1 132 | hb.exp = 0 133 | } 134 | return hb 135 | } 136 | 137 | func (hb *bin) powerOfTen() float64 { 138 | idx := int(uint8(hb.exp)) 139 | return powerOfTen[idx] 140 | } 141 | 142 | func (hb *bin) isNaN() bool { 143 | // aval := abs(hb.val) 144 | aval := hb.val 145 | if aval < 0 { 146 | aval = -aval 147 | } 148 | if 99 < aval { // in [100... ]: nan 149 | return true 150 | } 151 | if 9 < aval { // in [10 - 99]: valid range 152 | return false 153 | } 154 | if 0 < aval { // in [1 - 9 ]: nan 155 | return true 156 | } 157 | if 0 == aval { // in [0] : zero bucket 158 | return false 159 | } 160 | return false 161 | } 162 | 163 | func (hb *bin) value() float64 { 164 | if hb.isNaN() { 165 | return math.NaN() 166 | } 167 | if hb.val < 10 && hb.val > -10 { 168 | return 0.0 169 | } 170 | return (float64(hb.val) / 10.0) * hb.powerOfTen() 171 | } 172 | 173 | func (hb *bin) binWidth() float64 { 174 | if hb.isNaN() { 175 | return math.NaN() 176 | } 177 | if hb.val < 10 && hb.val > -10 { 178 | return 0.0 179 | } 180 | return hb.powerOfTen() / 10.0 181 | } 182 | 183 | func (hb *bin) midpoint() float64 { 184 | if hb.isNaN() { 185 | return math.NaN() 186 | } 187 | out := hb.value() 188 | if out == 0 { 189 | return 0 190 | } 191 | interval := hb.binWidth() 192 | if out < 0 { 193 | interval *= -1 194 | } 195 | return out + interval/2.0 196 | } 197 | 198 | func (hb *bin) left() float64 { 199 | if hb.isNaN() { 200 | return math.NaN() 201 | } 202 | out := hb.value() 203 | if out >= 0 { 204 | return out 205 | } 206 | return out - hb.binWidth() 207 | } 208 | 209 | func (hb *bin) compare(h2 *bin) int { 210 | var v1, v2 int 211 | 212 | // 1) slide exp positive 213 | // 2) shift by size of val multiple by (val != 0) 214 | // 3) then add or subtract val accordingly 215 | 216 | if hb.val >= 0 { 217 | v1 = ((int(hb.exp)+256)<<8)*(((int(hb.val)|(^int(hb.val)+1))>>8)&1) + int(hb.val) 218 | } else { 219 | v1 = ((int(hb.exp)+256)<<8)*(((int(hb.val)|(^int(hb.val)+1))>>8)&1) - int(hb.val) 220 | } 221 | 222 | if h2.val >= 0 { 223 | v2 = ((int(h2.exp)+256)<<8)*(((int(h2.val)|(^int(h2.val)+1))>>8)&1) + int(h2.val) 224 | } else { 225 | v2 = ((int(h2.exp)+256)<<8)*(((int(h2.val)|(^int(h2.val)+1))>>8)&1) - int(h2.val) 226 | } 227 | 228 | // return the difference 229 | return v2 - v1 230 | } 231 | 232 | // Histogram tracks values are two decimal digits of precision 233 | // with a bounded error that remains bounded upon composition. 234 | type Histogram struct { 235 | bvs []bin 236 | lookup [][]uint16 237 | mutex sync.RWMutex 238 | used uint16 239 | useLookup bool 240 | useLocks bool 241 | } 242 | 243 | //nolint:golint,revive 244 | const ( 245 | BVL1, BVL1MASK uint64 = iota, 0xff << (8 * iota) 246 | BVL2, BVL2MASK 247 | BVL3, BVL3MASK 248 | BVL4, BVL4MASK 249 | BVL5, BVL5MASK 250 | BVL6, BVL6MASK 251 | BVL7, BVL7MASK 252 | BVL8, BVL8MASK 253 | ) 254 | 255 | func getBytesRequired(val uint64) int8 { 256 | if 0 != (BVL8MASK|BVL7MASK|BVL6MASK|BVL5MASK)&val { 257 | if 0 != BVL8MASK&val { 258 | return int8(BVL8) 259 | } 260 | if 0 != BVL7MASK&val { 261 | return int8(BVL7) 262 | } 263 | if 0 != BVL6MASK&val { 264 | return int8(BVL6) 265 | } 266 | if 0 != BVL5MASK&val { 267 | return int8(BVL5) 268 | } 269 | } else { 270 | if 0 != BVL4MASK&val { 271 | return int8(BVL4) 272 | } 273 | if 0 != BVL3MASK&val { 274 | return int8(BVL3) 275 | } 276 | if 0 != BVL2MASK&val { 277 | return int8(BVL2) 278 | } 279 | } 280 | return int8(BVL1) 281 | } 282 | 283 | func writeBin(out io.Writer, in bin) (err error) { 284 | 285 | err = binary.Write(out, binary.BigEndian, in.val) 286 | if err != nil { 287 | return 288 | } 289 | 290 | err = binary.Write(out, binary.BigEndian, in.exp) 291 | if err != nil { 292 | return 293 | } 294 | 295 | var tgtType = getBytesRequired(in.count) 296 | 297 | err = binary.Write(out, binary.BigEndian, tgtType) 298 | if err != nil { 299 | return 300 | } 301 | 302 | var bcount = make([]uint8, 8) 303 | b := bcount[0 : tgtType+1] 304 | for i := tgtType; i >= 0; i-- { 305 | b[i] = uint8(uint64(in.count>>(uint8(i)*8)) & 0xff) //nolint:unconvert 306 | } 307 | 308 | err = binary.Write(out, binary.BigEndian, b) 309 | if err != nil { 310 | return 311 | } 312 | return 313 | } 314 | 315 | func readBin(in io.Reader) (bin, error) { 316 | var out bin 317 | 318 | err := binary.Read(in, binary.BigEndian, &out.val) 319 | if err != nil { 320 | return out, fmt.Errorf("read: %w", err) 321 | } 322 | 323 | err = binary.Read(in, binary.BigEndian, &out.exp) 324 | if err != nil { 325 | return out, fmt.Errorf("read: %w", err) 326 | } 327 | var bvl uint8 328 | err = binary.Read(in, binary.BigEndian, &bvl) 329 | if err != nil { 330 | return out, fmt.Errorf("read: %w", err) 331 | } 332 | if bvl > uint8(BVL8) { 333 | return out, fmt.Errorf("encoding error: bvl value is greater than max allowable") //nolint:goerr113 334 | } 335 | 336 | bcount := make([]byte, 8) 337 | b := bcount[0 : bvl+1] 338 | err = binary.Read(in, binary.BigEndian, b) 339 | if err != nil { 340 | return out, fmt.Errorf("read: %w", err) 341 | } 342 | 343 | count := uint64(0) 344 | for i := int(bvl + 1); i >= 0; i-- { 345 | count |= uint64(bcount[i]) << (uint8(i) * 8) 346 | } 347 | 348 | out.count = count 349 | return out, nil 350 | } 351 | 352 | func Deserialize(in io.Reader) (h *Histogram, err error) { 353 | return DeserializeWithOptions(in) 354 | } 355 | 356 | func DeserializeWithOptions(in io.Reader, options ...Option) (h *Histogram, err error) { 357 | var nbin int16 358 | err = binary.Read(in, binary.BigEndian, &nbin) 359 | if err != nil { 360 | return 361 | } 362 | 363 | options = append(options, Size(uint16(nbin))) 364 | h = New(options...) 365 | for ii := int16(0); ii < nbin; ii++ { 366 | bb, err := readBin(in) 367 | if err != nil { 368 | return h, err 369 | } 370 | h.insertBin(&bb, int64(bb.count)) 371 | } 372 | return h, nil 373 | } 374 | 375 | func (h *Histogram) Serialize(w io.Writer) error { 376 | var nbin int16 377 | for i := range h.bvs { 378 | if h.bvs[i].count != 0 { 379 | nbin++ 380 | } 381 | } 382 | 383 | if err := binary.Write(w, binary.BigEndian, nbin); err != nil { 384 | return fmt.Errorf("write: %w", err) 385 | } 386 | 387 | for _, bv := range h.bvs { 388 | if bv.count != 0 { 389 | if err := writeBin(w, bv); err != nil { 390 | return err 391 | } 392 | } 393 | } 394 | return nil 395 | } 396 | 397 | func (h *Histogram) SerializeB64(w io.Writer) error { 398 | buf := bytes.NewBuffer([]byte{}) 399 | if err := h.Serialize(buf); err != nil { 400 | return err 401 | } 402 | 403 | encoder := base64.NewEncoder(base64.StdEncoding, w) 404 | if _, err := encoder.Write(buf.Bytes()); err != nil { 405 | return fmt.Errorf("b64 encode write: %w", err) 406 | } 407 | if err := encoder.Close(); err != nil { 408 | return fmt.Errorf("b64 encoder close: %w", err) 409 | } 410 | 411 | return nil 412 | } 413 | 414 | // Options are exposed options for initializing a histogram. 415 | type Options struct { 416 | // Size is the number of bins. 417 | Size uint16 418 | 419 | // UseLocks determines if the histogram should use locks 420 | UseLocks bool 421 | 422 | // UseLookup determines if the histogram should use a lookup table for bins 423 | UseLookup bool 424 | } 425 | 426 | // Option knows how to mutate the Options to change initialization. 427 | type Option func(*Options) 428 | 429 | // NoLocks configures a histogram to not use locks. 430 | func NoLocks() Option { 431 | return func(options *Options) { 432 | options.UseLocks = false 433 | } 434 | } 435 | 436 | // NoLookup configures a histogram to not use a lookup table for bins. 437 | // This is an appropriate option to use when the data set being operated 438 | // over contains a large number of individual histograms and the insert 439 | // speed into any histogram is not of the utmost importance. This option 440 | // reduces the baseline memory consumption of one Histogram by at least 441 | // 0.5kB and up to 130kB while increasing the insertion time by ~20%. 442 | func NoLookup() Option { 443 | return func(options *Options) { 444 | options.UseLookup = false 445 | } 446 | } 447 | 448 | // Size configures a histogram to initialize a specific number of bins. 449 | // When more bins are required, allocations increase linearly by the default 450 | // size (100). 451 | func Size(size uint16) Option { 452 | return func(options *Options) { 453 | options.Size = size 454 | } 455 | } 456 | 457 | // New returns a new Histogram, respecting the passed Options. 458 | func New(options ...Option) *Histogram { 459 | o := Options{ 460 | Size: defaultHistSize, 461 | UseLocks: true, 462 | UseLookup: true, 463 | } 464 | for _, opt := range options { 465 | opt(&o) 466 | } 467 | h := &Histogram{ 468 | used: 0, 469 | bvs: make([]bin, o.Size), 470 | useLocks: o.UseLocks, 471 | useLookup: o.UseLookup, 472 | } 473 | if h.useLookup { 474 | h.lookup = make([][]uint16, 256) 475 | } 476 | return h 477 | } 478 | 479 | // NewNoLocks returns a new histogram not using locks. 480 | // Deprecated: use New(NoLocks()) instead. 481 | func NewNoLocks() *Histogram { 482 | return New(NoLocks()) 483 | } 484 | 485 | // NewFromStrings returns a Histogram created from DecStrings strings. 486 | func NewFromStrings(strs []string, locks bool) (*Histogram, error) { 487 | 488 | bin, err := stringsToBin(strs) 489 | if err != nil { 490 | return nil, err 491 | } 492 | 493 | return newFromBins(bin, locks), nil 494 | } 495 | 496 | // NewFromBins returns a Histogram created from a bins struct slice. 497 | func newFromBins(bins []bin, locks bool) *Histogram { 498 | return &Histogram{ 499 | used: uint16(len(bins)), 500 | bvs: bins, 501 | useLocks: locks, 502 | lookup: make([][]uint16, 256), 503 | useLookup: true, 504 | } 505 | } 506 | 507 | // Max returns the approximate maximum recorded value. 508 | func (h *Histogram) Max() float64 { 509 | return h.ValueAtQuantile(1.0) 510 | } 511 | 512 | // Min returns the approximate minimum recorded value. 513 | func (h *Histogram) Min() float64 { 514 | return h.ValueAtQuantile(0.0) 515 | } 516 | 517 | // Mean returns the approximate arithmetic mean of the recorded values. 518 | func (h *Histogram) Mean() float64 { 519 | return h.ApproxMean() 520 | } 521 | 522 | // Count returns the number of recorded values. 523 | func (h *Histogram) Count() uint64 { 524 | if h.useLocks { 525 | h.mutex.RLock() 526 | defer h.mutex.RUnlock() 527 | } 528 | var count uint64 529 | for _, bin := range h.bvs[0:h.used] { 530 | if bin.isNaN() { 531 | continue 532 | } 533 | count += bin.count 534 | } 535 | return count 536 | } 537 | 538 | // BinCount returns the number of used bins. 539 | func (h *Histogram) BinCount() uint64 { 540 | if h.useLocks { 541 | h.mutex.RLock() 542 | defer h.mutex.RUnlock() 543 | } 544 | binCount := h.used 545 | return uint64(binCount) 546 | } 547 | 548 | // Reset forgets all bins in the histogram (they remain allocated). 549 | func (h *Histogram) Reset() { 550 | if h.useLocks { 551 | h.mutex.Lock() 552 | defer h.mutex.Unlock() 553 | } 554 | h.used = 0 555 | 556 | if !h.useLookup { 557 | return 558 | } 559 | for i := 0; i < 256; i++ { 560 | if h.lookup[i] != nil { 561 | for j := range h.lookup[i] { 562 | h.lookup[i][j] = 0 563 | } 564 | } 565 | } 566 | } 567 | 568 | // RecordIntScale records an integer scaler value, returning an error if the 569 | // value is out of range. 570 | func (h *Histogram) RecordIntScale(val int64, scale int) error { 571 | return h.RecordIntScales(val, scale, 1) 572 | } 573 | 574 | // RecordValue records the given value, returning an error if the value is out 575 | // of range. 576 | func (h *Histogram) RecordValue(v float64) error { 577 | return h.RecordValues(v, 1) 578 | } 579 | 580 | // RecordDuration records the given time.Duration in seconds, returning an error 581 | // if the value is out of range. 582 | func (h *Histogram) RecordDuration(v time.Duration) error { 583 | return h.RecordIntScale(int64(v), -9) 584 | } 585 | 586 | // RecordCorrectedValue records the given value, correcting for stalls in the 587 | // recording process. This only works for processes which are recording values 588 | // at an expected interval (e.g., doing jitter analysis). Processes which are 589 | // recording ad-hoc values (e.g., latency for incoming requests) can't take 590 | // advantage of this. 591 | // CH Compat. 592 | func (h *Histogram) RecordCorrectedValue(v, expectedInterval int64) error { 593 | if err := h.RecordValue(float64(v)); err != nil { 594 | return err 595 | } 596 | 597 | if expectedInterval <= 0 || v <= expectedInterval { 598 | return nil 599 | } 600 | 601 | missingValue := v - expectedInterval 602 | for missingValue >= expectedInterval { 603 | if err := h.RecordValue(float64(missingValue)); err != nil { 604 | return err 605 | } 606 | missingValue -= expectedInterval 607 | } 608 | 609 | return nil 610 | } 611 | 612 | // find where a new bin should go. 613 | func (h *Histogram) internalFind(hb *bin) (bool, uint16) { 614 | if h.used == 0 { 615 | return false, 0 616 | } 617 | if h.useLookup { 618 | f2 := hb.newFastL2() 619 | if h.lookup[f2.l1] != nil { 620 | if idx := h.lookup[f2.l1][f2.l2]; idx != 0 { 621 | return true, idx - 1 622 | } 623 | } 624 | } 625 | rv := -1 626 | idx := uint16(0) 627 | l := int(0) 628 | r := int(h.used - 1) 629 | for l < r { 630 | check := (r + l) / 2 631 | rv = h.bvs[check].compare(hb) 632 | switch { 633 | case rv == 0: 634 | l = check 635 | r = check 636 | case rv > 0: 637 | l = check + 1 638 | default: 639 | r = check - 1 640 | } 641 | } 642 | if rv != 0 { 643 | rv = h.bvs[l].compare(hb) 644 | } 645 | idx = uint16(l) 646 | if rv == 0 { 647 | return true, idx 648 | } 649 | if rv < 0 { 650 | return false, idx 651 | } 652 | idx++ 653 | return false, idx 654 | } 655 | 656 | func (h *Histogram) insertBin(hb *bin, count int64) uint64 { //nolint:unparam 657 | if h.useLocks { 658 | h.mutex.Lock() 659 | defer h.mutex.Unlock() 660 | } 661 | found, idx := h.internalFind(hb) 662 | if !found { 663 | count := h.insertNewBinAt(idx, hb, count) 664 | // update the fast lookup table data after the index 665 | h.updateFast(idx) 666 | return count 667 | } 668 | return h.updateOldBinAt(idx, count) 669 | } 670 | 671 | func (h *Histogram) insertNewBinAt(idx uint16, hb *bin, count int64) uint64 { 672 | h.bvs = append(h.bvs, bin{}) 673 | copy(h.bvs[idx+1:], h.bvs[idx:]) 674 | h.bvs[idx].val = hb.val 675 | h.bvs[idx].exp = hb.exp 676 | h.bvs[idx].count = uint64(count) 677 | h.used++ 678 | return h.bvs[idx].count 679 | } 680 | 681 | func (h *Histogram) updateFast(start uint16) { 682 | if !h.useLookup { 683 | return 684 | } 685 | for i := start; i < h.used; i++ { 686 | f2 := h.bvs[i].newFastL2() 687 | if h.lookup[f2.l1] == nil { 688 | h.lookup[f2.l1] = make([]uint16, 256) 689 | } 690 | h.lookup[f2.l1][f2.l2] = i + 1 691 | } 692 | } 693 | 694 | func (h *Histogram) updateOldBinAt(idx uint16, count int64) uint64 { 695 | var newval uint64 696 | if count >= 0 { 697 | newval = h.bvs[idx].count + uint64(count) 698 | } else { 699 | newval = h.bvs[idx].count - uint64(-count) 700 | } 701 | if newval < h.bvs[idx].count { // rolled 702 | newval = ^uint64(0) 703 | } 704 | h.bvs[idx].count = newval 705 | return newval - h.bvs[idx].count 706 | } 707 | 708 | // RecordIntScales records n occurrences of the given value, returning an error if 709 | // the value is out of range. 710 | func (h *Histogram) RecordIntScales(val int64, scale int, n int64) error { 711 | sign := int64(1) 712 | if val == 0 { 713 | scale = 0 714 | } else { 715 | scale++ 716 | if val < 0 { 717 | val = 0 - val 718 | sign = -1 719 | } 720 | if val < 10 { 721 | val *= 10 722 | scale-- 723 | } 724 | for val >= 100 { 725 | val /= 10 726 | scale++ 727 | } 728 | } 729 | if scale < -128 { 730 | val = 0 731 | scale = 0 732 | } else if scale > 127 { 733 | val = 0xff 734 | scale = 0 735 | } 736 | val *= sign 737 | hb := bin{val: int8(val), exp: int8(scale), count: 0} 738 | h.insertBin(&hb, n) 739 | return nil 740 | } 741 | 742 | // RecordValues records n occurrences of the given value, returning an error if 743 | // the value is out of range. 744 | func (h *Histogram) RecordValues(v float64, n int64) error { 745 | var hb bin 746 | hb.setFromFloat64(v) 747 | h.insertBin(&hb, n) 748 | return nil 749 | } 750 | 751 | // ApproxMean returns an approximate mean. 752 | func (h *Histogram) ApproxMean() float64 { 753 | if h.useLocks { 754 | h.mutex.RLock() 755 | defer h.mutex.RUnlock() 756 | } 757 | divisor := 0.0 758 | sum := 0.0 759 | for i := uint16(0); i < h.used; i++ { 760 | midpoint := h.bvs[i].midpoint() 761 | cardinality := float64(h.bvs[i].count) 762 | divisor += cardinality 763 | sum += midpoint * cardinality 764 | } 765 | if divisor == 0.0 { 766 | return math.NaN() 767 | } 768 | return sum / divisor 769 | } 770 | 771 | // ApproxSum returns an approximate sum. 772 | func (h *Histogram) ApproxSum() float64 { 773 | if h.useLocks { 774 | h.mutex.RLock() 775 | defer h.mutex.RUnlock() 776 | } 777 | sum := 0.0 778 | for i := uint16(0); i < h.used; i++ { 779 | midpoint := h.bvs[i].midpoint() 780 | cardinality := float64(h.bvs[i].count) 781 | sum += midpoint * cardinality 782 | } 783 | return sum 784 | } 785 | 786 | func (h *Histogram) ApproxQuantile(qIn []float64) ([]float64, error) { 787 | if h.useLocks { 788 | h.mutex.RLock() 789 | defer h.mutex.RUnlock() 790 | } 791 | qOut := make([]float64, len(qIn)) 792 | iq, ib := 0, uint16(0) 793 | totalCnt, binWidth, binLeft, lowerCnt, upperCnt := 0.0, 0.0, 0.0, 0.0, 0.0 794 | if len(qIn) == 0 { 795 | return qOut, nil 796 | } 797 | // Make sure the requested quantiles are in order 798 | for iq = 1; iq < len(qIn); iq++ { 799 | if qIn[iq-1] > qIn[iq] { 800 | return nil, fmt.Errorf("out of order") //nolint:goerr113 801 | } 802 | } 803 | // Add up the bins 804 | for ib = 0; ib < h.used; ib++ { 805 | if !h.bvs[ib].isNaN() { 806 | totalCnt += float64(h.bvs[ib].count) 807 | } 808 | } 809 | if totalCnt == 0.0 { 810 | return nil, fmt.Errorf("empty_histogram") //nolint:goerr113 811 | } 812 | 813 | for iq = 0; iq < len(qIn); iq++ { 814 | if qIn[iq] < 0.0 || qIn[iq] > 1.0 { 815 | return nil, fmt.Errorf("out of bound quantile") //nolint:goerr113 816 | } 817 | qOut[iq] = totalCnt * qIn[iq] 818 | } 819 | 820 | for ib = 0; ib < h.used; ib++ { 821 | if h.bvs[ib].isNaN() { 822 | continue 823 | } 824 | binWidth = h.bvs[ib].binWidth() 825 | binLeft = h.bvs[ib].left() 826 | lowerCnt = upperCnt 827 | upperCnt = lowerCnt + float64(h.bvs[ib].count) 828 | break 829 | } 830 | for iq = 0; iq < len(qIn); iq++ { 831 | for ib < (h.used-1) && upperCnt < qOut[iq] { 832 | ib++ 833 | binWidth = h.bvs[ib].binWidth() 834 | binLeft = h.bvs[ib].left() 835 | lowerCnt = upperCnt 836 | upperCnt = lowerCnt + float64(h.bvs[ib].count) 837 | } 838 | switch { 839 | case lowerCnt == qOut[iq]: 840 | qOut[iq] = binLeft 841 | case upperCnt == qOut[iq]: 842 | qOut[iq] = binLeft + binWidth 843 | default: 844 | if binWidth == 0 { 845 | qOut[iq] = binLeft 846 | } else { 847 | qOut[iq] = binLeft + (qOut[iq]-lowerCnt)/(upperCnt-lowerCnt)*binWidth 848 | } 849 | } 850 | } 851 | return qOut, nil 852 | } 853 | 854 | // ValueAtQuantile returns the recorded value at the given quantile (0..1). 855 | func (h *Histogram) ValueAtQuantile(q float64) float64 { 856 | if h.useLocks { 857 | h.mutex.RLock() 858 | defer h.mutex.RUnlock() 859 | } 860 | qIn := make([]float64, 1) 861 | qIn[0] = q 862 | qOut, err := h.ApproxQuantile(qIn) 863 | if err == nil && len(qOut) == 1 { 864 | return qOut[0] 865 | } 866 | return math.NaN() 867 | } 868 | 869 | // SignificantFigures returns the significant figures used to create the 870 | // histogram 871 | // CH Compat. 872 | func (h *Histogram) SignificantFigures() int64 { 873 | return 2 874 | } 875 | 876 | // Equals returns true if the two Histograms are equivalent, false if not. 877 | func (h *Histogram) Equals(other *Histogram) bool { 878 | if h.useLocks { 879 | h.mutex.RLock() 880 | defer h.mutex.RUnlock() 881 | } 882 | if other.useLocks { 883 | other.mutex.RLock() 884 | defer other.mutex.RUnlock() 885 | } 886 | switch { 887 | case 888 | h.used != other.used: 889 | return false 890 | default: 891 | for i := uint16(0); i < h.used; i++ { 892 | if h.bvs[i].compare(&other.bvs[i]) != 0 { 893 | return false 894 | } 895 | if h.bvs[i].count != other.bvs[i].count { 896 | return false 897 | } 898 | } 899 | } 900 | return true 901 | } 902 | 903 | // Copy creates and returns an exact copy of a histogram. 904 | func (h *Histogram) Copy() *Histogram { 905 | if h.useLocks { 906 | h.mutex.Lock() 907 | defer h.mutex.Unlock() 908 | } 909 | 910 | newhist := New() 911 | newhist.used = h.used 912 | newhist.useLocks = h.useLocks 913 | 914 | newhist.bvs = make([]bin, len(h.bvs)) 915 | copy(newhist.bvs, h.bvs) 916 | 917 | newhist.useLookup = h.useLookup 918 | if h.useLookup { 919 | newhist.lookup = make([][]uint16, 256) 920 | for i, u := range h.lookup { 921 | newhist.lookup[i] = append(newhist.lookup[i], u...) 922 | } 923 | } 924 | 925 | return newhist 926 | } 927 | 928 | // FullReset resets a histogram to default empty values. 929 | func (h *Histogram) FullReset() { 930 | if h.useLocks { 931 | h.mutex.Lock() 932 | defer h.mutex.Unlock() 933 | } 934 | 935 | h.bvs = []bin{} 936 | h.used = 0 937 | if h.useLookup { 938 | h.lookup = make([][]uint16, 256) 939 | } 940 | } 941 | 942 | // CopyAndReset creates and returns an exact copy of a histogram, 943 | // and resets it to default empty values. 944 | func (h *Histogram) CopyAndReset() *Histogram { 945 | newhist := h.Copy() 946 | h.FullReset() 947 | return newhist 948 | } 949 | 950 | func (h *Histogram) DecStrings() []string { 951 | if h.useLocks { 952 | h.mutex.Lock() 953 | defer h.mutex.Unlock() 954 | } 955 | out := make([]string, h.used) 956 | for i, bin := range h.bvs[0:h.used] { 957 | var buffer bytes.Buffer 958 | buffer.WriteString("H[") 959 | buffer.WriteString(fmt.Sprintf("%3.1e", bin.value())) 960 | buffer.WriteString("]=") 961 | buffer.WriteString(fmt.Sprintf("%v", bin.count)) 962 | out[i] = buffer.String() 963 | } 964 | return out 965 | } 966 | 967 | // takes the output of DecStrings and deserializes it into a Bin struct slice. 968 | func stringsToBin(strs []string) ([]bin, error) { 969 | 970 | bins := make([]bin, len(strs)) 971 | for i, str := range strs { 972 | 973 | // H[0.0e+00]=1 974 | 975 | // H[0.0e+00]= <1> 976 | countString := strings.Split(str, "=")[1] 977 | countInt, err := strconv.ParseInt(countString, 10, 64) 978 | if err != nil { 979 | return nil, fmt.Errorf("parse int: %w", err) 980 | } 981 | 982 | // H[ <0.0> e+00]=1 983 | valString := strings.Split(strings.Split(strings.Split(str, "=")[0], "e")[0], "[")[1] 984 | valInt, err := strconv.ParseFloat(valString, 64) 985 | if err != nil { 986 | return nil, fmt.Errorf("parse float: %w", err) 987 | } 988 | 989 | // H[0.0e <+00> ]=1 990 | expString := strings.Split(strings.Split(strings.Split(str, "=")[0], "e")[1], "]")[0] 991 | expInt, err := strconv.ParseInt(expString, 10, 8) 992 | if err != nil { 993 | return nil, fmt.Errorf("parse int: %w", err) 994 | } 995 | bins[i] = *newBinRaw(int8(valInt*10), int8(expInt), uint64(countInt)) 996 | } 997 | 998 | return bins, nil 999 | } 1000 | 1001 | // UnmarshalJSON - histogram will come in a base64 encoded serialized form. 1002 | func (h *Histogram) UnmarshalJSON(b []byte) error { 1003 | return UnmarshalJSONWithOptions(h, b) 1004 | } 1005 | 1006 | // UnmarshalJSONWithOptions unmarshals the byte data into the parent histogram, 1007 | // using the provided Options to create the output Histogram. 1008 | func UnmarshalJSONWithOptions(parent *Histogram, b []byte, options ...Option) error { 1009 | var s string 1010 | if err := json.Unmarshal(b, &s); err != nil { 1011 | return fmt.Errorf("json unmarshal: %w", err) 1012 | } 1013 | 1014 | data, err := base64.StdEncoding.DecodeString(s) 1015 | if err != nil { 1016 | return fmt.Errorf("b64 decode: %w", err) 1017 | } 1018 | 1019 | hNew, err := DeserializeWithOptions(bytes.NewBuffer(data), options...) 1020 | if err != nil { 1021 | return err 1022 | } 1023 | 1024 | // Go's JSON package will create a new Histogram to deserialize into by 1025 | // reflection, so all fields will have their zero values. Some of the 1026 | // default Histogram fields are not the zero values, so we can set them 1027 | // by proxy from the new histogram that's been created from deserialization. 1028 | parent.useLocks = hNew.useLocks 1029 | parent.useLookup = hNew.useLookup 1030 | if parent.useLookup { 1031 | parent.lookup = make([][]uint16, 256) 1032 | } 1033 | 1034 | parent.Merge(hNew) 1035 | return nil 1036 | } 1037 | 1038 | func (h *Histogram) MarshalJSON() ([]byte, error) { 1039 | return MarshalJSON(h) 1040 | } 1041 | 1042 | func MarshalJSON(h *Histogram) ([]byte, error) { 1043 | buf := bytes.NewBuffer([]byte{}) 1044 | err := h.SerializeB64(buf) 1045 | if err != nil { 1046 | return buf.Bytes(), err 1047 | } 1048 | data, err := json.Marshal(buf.String()) 1049 | if err != nil { 1050 | return nil, fmt.Errorf("json marshal: %w", err) 1051 | } 1052 | return data, nil 1053 | } 1054 | 1055 | // Merge merges all bins from another histogram. 1056 | func (h *Histogram) Merge(o *Histogram) { 1057 | if o == nil { 1058 | return 1059 | } 1060 | 1061 | if o.useLocks { 1062 | o.mutex.Lock() 1063 | defer o.mutex.Unlock() 1064 | } 1065 | if h.useLocks { 1066 | h.mutex.Lock() 1067 | defer h.mutex.Unlock() 1068 | } 1069 | 1070 | var i, j uint16 1071 | for ; i < h.used && j < o.used; i++ { 1072 | diff := h.bvs[i].compare(&o.bvs[j]) 1073 | // o.bvs[j] > h.bvs[i], do nothing. 1074 | if diff > 0 { 1075 | continue 1076 | } 1077 | 1078 | b := &o.bvs[j] 1079 | j++ 1080 | switch { 1081 | case diff == 0: 1082 | h.updateOldBinAt(i, int64(b.count)) 1083 | case diff < 0: 1084 | h.insertNewBinAt(i, b, int64(b.count)) 1085 | } 1086 | } 1087 | 1088 | // append the rest bins 1089 | for ; j < o.used; j++ { 1090 | h.insertNewBinAt(h.used, &o.bvs[j], int64(o.bvs[j].count)) 1091 | } 1092 | 1093 | // rebuild all the fast lookup table 1094 | h.updateFast(0) 1095 | } 1096 | 1097 | // HistogramWithoutLookups holds a Histogram that's not configured to use 1098 | // a lookup table. This type is useful to round-trip serialize the underlying 1099 | // data while never allocating memory for the lookup table. 1100 | // The main Histogram type must use lookups by default to be compatible with 1101 | // the circllhist implementation of other languages. Furthermore, it is not 1102 | // possible to encode the lookup table preference into the serialized form, 1103 | // as that's again defined across languages. Therefore, the most straightforward 1104 | // manner by which a user can deserialize histogram data while not allocating 1105 | // lookup tables is by using a dedicated type in their structures describing 1106 | // on-disk forms. 1107 | // This structure can divulge the underlying Histogram, optionally allocating 1108 | // the lookup tables first. 1109 | type HistogramWithoutLookups struct { 1110 | histogram *Histogram 1111 | } 1112 | 1113 | // NewHistogramWithoutLookups creates a new container for a Histogram without 1114 | // lookup tables. 1115 | func NewHistogramWithoutLookups(histogram *Histogram) *HistogramWithoutLookups { 1116 | histogram.useLookup = false 1117 | histogram.lookup = nil 1118 | return &HistogramWithoutLookups{ 1119 | histogram: histogram, 1120 | } 1121 | } 1122 | 1123 | // Histogram divulges the underlying Histogram that was deserialized. This 1124 | // Histogram will not have lookup tables allocated. 1125 | func (h *HistogramWithoutLookups) Histogram() *Histogram { 1126 | return h.histogram 1127 | } 1128 | 1129 | // HistogramWithLookups allocates lookup tables in the underlying Histogram that was 1130 | // deserialized, then divulges it. 1131 | func (h *HistogramWithoutLookups) HistogramWithLookups() *Histogram { 1132 | h.histogram.useLookup = true 1133 | h.histogram.lookup = make([][]uint16, 256) 1134 | return h.histogram 1135 | } 1136 | 1137 | // UnmarshalJSON unmarshals a histogram from a base64 encoded serialized form. 1138 | func (h *HistogramWithoutLookups) UnmarshalJSON(b []byte) error { 1139 | var histogram Histogram 1140 | if err := UnmarshalJSONWithOptions(&histogram, b, NoLookup()); err != nil { 1141 | return err 1142 | } 1143 | h.histogram = &histogram 1144 | return nil 1145 | } 1146 | 1147 | // MarshalJSON marshals a histogram to a base64 encoded serialized form. 1148 | func (h *HistogramWithoutLookups) MarshalJSON() ([]byte, error) { 1149 | return MarshalJSON(h.histogram) 1150 | } 1151 | -------------------------------------------------------------------------------- /circonusllhist_test.go: -------------------------------------------------------------------------------- 1 | package circonusllhist 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "math" 8 | "math/rand" 9 | "reflect" 10 | "strings" 11 | "testing" 12 | "time" 13 | ) 14 | 15 | func TestCreate(t *testing.T) { 16 | h := New() 17 | /* 18 | for j := 0; j < 100000; j++ { 19 | h.RecordIntScale(rand.Intn(1000), 0) 20 | } 21 | */ 22 | _ = h.RecordIntScales(99, 0, int64(rand.Intn(2))+1) 23 | buf := bytes.NewBuffer([]byte{}) 24 | if err := h.Serialize(buf); err != nil { 25 | t.Error(err) 26 | } 27 | h2, err := Deserialize(buf) 28 | if err != nil { 29 | t.Error(err) 30 | } 31 | for j := uint16(0); j < h2.used; j++ { 32 | if h2.bvs[j].exp < 1 && (h2.bvs[j].val%10) != 0 { 33 | t.Errorf("bad bin[%v] %ve%v", j, float64(h2.bvs[j].val)/10.0, h2.bvs[j].exp) 34 | } 35 | } 36 | } 37 | 38 | func TestSerialize(t *testing.T) { 39 | h, err := NewFromStrings([]string{ 40 | "H[0.0e+00]=1", 41 | "H[1.0e+01]=1", 42 | "H[2.0e+02]=1", 43 | }, false) 44 | if err != nil { 45 | t.Error("could not read from strings for test") 46 | } 47 | 48 | buf := bytes.NewBuffer([]byte{}) 49 | if err = h.Serialize(buf); err != nil { 50 | t.Error(err) 51 | } 52 | 53 | h2, err := Deserialize(buf) 54 | if err != nil { 55 | t.Error(h2, err) 56 | } 57 | if !h.Equals(h2) { 58 | t.Log(h.DecStrings()) 59 | t.Log(h2.DecStrings()) 60 | t.Error("histograms do not match") 61 | } 62 | } 63 | 64 | func TestCount(t *testing.T) { 65 | h, err := NewFromStrings([]string{ 66 | "H[0.0e+00]=1", 67 | "H[1.0e+01]=1", 68 | "H[2.0e+02]=1", 69 | }, true) 70 | if err != nil { 71 | t.Error("could not read from strings for test") 72 | } 73 | if h.Count() != 3 { 74 | t.Error("the count is incorrect") 75 | } 76 | err = h.RecordValue(10) 77 | if err != nil { 78 | t.Error("could not record new value to histogram") 79 | } 80 | if h.Count() != 4 { 81 | t.Error("the count is incorrect") 82 | } 83 | } 84 | 85 | func TestBinCount(t *testing.T) { 86 | h, err := NewFromStrings([]string{ 87 | "H[0.0e+00]=1", 88 | "H[1.0e+01]=1", 89 | "H[2.0e+02]=1", 90 | }, true) 91 | if err != nil { 92 | t.Error("could not read from strings for test") 93 | } 94 | if h.BinCount() != 3 { 95 | t.Error("bin count is incorrect") 96 | } 97 | } 98 | 99 | func TestJSON(t *testing.T) { 100 | h, err := NewFromStrings([]string{ 101 | "H[0.0e+00]=1", 102 | "H[1.0e+01]=1", 103 | "H[2.0e+02]=1", 104 | }, false) 105 | if err != nil { 106 | t.Errorf("could not read from strings for test error = %v", err) 107 | } 108 | 109 | jh, err := json.Marshal(h) 110 | if err != nil { 111 | t.Errorf("could not marshall json for test error = %v", err) 112 | } 113 | 114 | h2 := &Histogram{} 115 | if err := json.Unmarshal(jh, h2); err != nil { 116 | t.Errorf("could not unmarshall json for test error = %v", err) 117 | } 118 | 119 | if !h.Equals(h2) { 120 | t.Log(h.DecStrings()) 121 | t.Log(h2.DecStrings()) 122 | t.Error("histograms do not match") 123 | } 124 | } 125 | 126 | func helpTestBin(t *testing.T, v float64, val, exp int8) { 127 | b := newBinFromFloat64(v) 128 | if b.val != val || b.exp != exp { 129 | t.Errorf("%v -> [%v,%v] expected, but got [%v,%v]", v, val, exp, b.val, b.exp) 130 | } 131 | } 132 | 133 | func fuzzyEquals(expected, actual float64) bool { 134 | delta := math.Abs(expected / 100000.0) 135 | if actual >= expected-delta && actual <= expected+delta { 136 | return true 137 | } 138 | return false 139 | } 140 | 141 | func TestBins(t *testing.T) { 142 | helpTestBin(t, 0.0, 0, 0) 143 | helpTestBin(t, 100, 10, 2) 144 | helpTestBin(t, 9.9999e-129, 0, 0) 145 | helpTestBin(t, 1e-128, 10, -128) 146 | helpTestBin(t, 1.00001e-128, 10, -128) 147 | helpTestBin(t, 1.09999e-128, 10, -128) 148 | helpTestBin(t, 1.1e-128, 11, -128) 149 | helpTestBin(t, 1e127, 10, 127) 150 | helpTestBin(t, 9.999e127, 99, 127) 151 | helpTestBin(t, 1e128, -1, 0) 152 | helpTestBin(t, -9.9999e-129, 0, 0) 153 | helpTestBin(t, -1e-128, -10, -128) 154 | helpTestBin(t, -1.00001e-128, -10, -128) 155 | helpTestBin(t, -1.09999e-128, -10, -128) 156 | helpTestBin(t, -1.1e-128, -11, -128) 157 | helpTestBin(t, -1e127, -10, 127) 158 | helpTestBin(t, -9.999e127, -99, 127) 159 | helpTestBin(t, -1e128, -1, 0) 160 | helpTestBin(t, 9.999e127, 99, 127) 161 | 162 | h := New() 163 | _ = h.RecordIntScale(100, 0) 164 | if h.bvs[0].val != 10 || h.bvs[0].exp != 2 { 165 | t.Errorf("100 not added correctly") 166 | } 167 | 168 | h = New() 169 | _ = h.RecordValue(100.0) 170 | if h.bvs[0].val != 10 || h.bvs[0].exp != 2 { 171 | t.Errorf("100.0 not added correctly") 172 | } 173 | } 174 | 175 | func TestRecordDuration(t *testing.T) { 176 | tests := []struct { 177 | input []time.Duration 178 | inputUnit time.Duration 179 | approxSum time.Duration 180 | approxMean time.Duration 181 | tolerance time.Duration 182 | }{ 183 | { 184 | input: []time.Duration{time.Nanosecond}, 185 | approxSum: time.Nanosecond, 186 | approxMean: time.Nanosecond, 187 | }, 188 | { 189 | input: []time.Duration{3 * time.Nanosecond}, 190 | approxSum: 3 * time.Nanosecond, 191 | approxMean: 3 * time.Nanosecond, 192 | }, 193 | { 194 | input: []time.Duration{1000 * time.Second}, 195 | approxSum: 1000 * time.Second, 196 | approxMean: 1000 * time.Second, 197 | }, 198 | { 199 | input: []time.Duration{ 200 | 4 * time.Second, 201 | 8 * time.Second, 202 | }, 203 | approxSum: 12.0 * time.Second, 204 | approxMean: 6.0 * time.Second, 205 | }, 206 | } 207 | 208 | fuzzyEquals := func(expected, actual time.Duration) bool { 209 | diff := math.Abs(float64(expected) - float64(actual)) 210 | return (diff / math.Max(float64(expected), float64(actual))) <= 0.05 211 | } 212 | 213 | for n, test := range tests { 214 | test := test 215 | t.Run(fmt.Sprintf("%d", n), func(t *testing.T) { 216 | h := New() 217 | for _, dur := range test.input { 218 | _ = h.RecordDuration(dur) 219 | } 220 | 221 | if v := time.Duration(1000000000.0 * h.ApproxSum()); !fuzzyEquals(v, test.approxSum) { 222 | t.Fatalf("%v approx sum bad: have=%v want=%v", test.input, h.ApproxSum(), test.approxSum) 223 | } 224 | 225 | if v := time.Duration(1000000000.0 * h.ApproxMean()); !fuzzyEquals(v, test.approxMean) { 226 | t.Fatalf("%v approx mean bad: have=%v want=%v", test.input, v, test.approxMean) 227 | } 228 | }) 229 | } 230 | } 231 | 232 | func helpTestVB(t *testing.T, v, b, w float64) { 233 | bin := newBinFromFloat64(v) 234 | out := bin.value() 235 | interval := bin.binWidth() 236 | if out < 0 { 237 | interval *= -1.0 238 | } 239 | if !fuzzyEquals(b, out) { 240 | t.Errorf("%v -> %v != %v\n", v, out, b) 241 | } 242 | if !fuzzyEquals(w, interval) { 243 | t.Errorf("%v -> [%v] != [%v]\n", v, interval, w) 244 | } 245 | } 246 | 247 | func TestBinSizes(t *testing.T) { 248 | helpTestVB(t, 43.3, 43.0, 1.0) 249 | helpTestVB(t, 99.9, 99.0, 1.0) 250 | helpTestVB(t, 10.0, 10.0, 1.0) 251 | helpTestVB(t, 1.0, 1.0, 0.1) 252 | helpTestVB(t, 0.0002, 0.0002, 0.00001) 253 | helpTestVB(t, 0.003, 0.003, 0.0001) 254 | helpTestVB(t, 0.3201, 0.32, 0.01) 255 | helpTestVB(t, 0.0035, 0.0035, 0.0001) 256 | helpTestVB(t, -1.0, -1.0, -0.1) 257 | helpTestVB(t, -0.00123, -0.0012, -0.0001) 258 | helpTestVB(t, -987324, -980000, -10000) 259 | } 260 | 261 | // preloadedTester knows how to preload values, then use them to benchmark a histogram. 262 | type preloadedTester interface { 263 | preload(n int) 264 | run(histogram *Histogram) error 265 | } 266 | 267 | // intScale knows how to benchmark RecordIntScale. 268 | type intScale struct { 269 | // integers hold the integers we will feed RecordIntScale 270 | integers []int64 271 | 272 | // scales hold the scales we will feed RecordIntScale 273 | scales []int 274 | 275 | // scale is the scale of the distribution of values - this allows the benchmark 276 | // to tease apart differences in the usage of a histogram in different applications 277 | // where it may be storing fairly homogenous values or any value whatsoever 278 | scale int 279 | 280 | n int 281 | } 282 | 283 | func (t *intScale) preload(n int) { 284 | t.n = 0 285 | t.integers = make([]int64, n) 286 | t.scales = make([]int, n) 287 | 288 | scaleMin := rand.Intn(math.MaxInt64 - t.scale) 289 | for i := 0; i < n; i++ { 290 | t.integers[i] = rand.Int63() * (rand.Int63n(2) - 1) // allow negatives! 291 | t.scales[i] = rand.Intn(t.scale) + scaleMin 292 | } 293 | } 294 | 295 | func (t *intScale) run(histogram *Histogram) error { 296 | n := t.n 297 | t.n++ 298 | return histogram.RecordIntScale(t.integers[n], t.scales[n]) 299 | } 300 | 301 | // value knows how to benchmark RecordValue. 302 | type value struct { 303 | // values hold the integers we will feed RecordValue 304 | values []float64 305 | 306 | // stddev is the standard deviation of the distribution of values - this allows the 307 | // benchmark to tease apart differences in the usage of a histogram in different 308 | // applications where it may be storing fairly homogenous values or any value whatsoever 309 | stddev float64 310 | 311 | n int 312 | } 313 | 314 | func (t *value) preload(n int) { 315 | t.n = 0 316 | t.values = make([]float64, n) 317 | 318 | mean := float64(rand.Int63() * (rand.Int63n(2) - 1)) // allow negatives! 319 | for i := 0; i < n; i++ { 320 | t.values[i] = rand.NormFloat64()*t.stddev + mean 321 | } 322 | } 323 | 324 | func (t *value) run(histogram *Histogram) error { 325 | n := t.n 326 | t.n++ 327 | return histogram.RecordValue(t.values[n]) 328 | } 329 | 330 | func BenchmarkRecord(b *testing.B) { 331 | benchmarkForHist(b, func() *Histogram { 332 | return New() 333 | }) 334 | } 335 | 336 | func BenchmarkRecordWithoutLookups(b *testing.B) { 337 | benchmarkForHist(b, func() *Histogram { 338 | return New(NoLookup()) 339 | }) 340 | } 341 | 342 | func benchmarkForHist(b *testing.B, constructor func() *Histogram) { 343 | rand.Seed(time.Now().UnixNano()) 344 | for _, scale := range []int{1, 2, 4, 8, 16, 32, 64} { 345 | for _, tester := range []preloadedTester{ 346 | &intScale{scale: scale}, 347 | &value{stddev: math.Pow10(scale)}, 348 | } { 349 | name := fmt.Sprintf("%T", tester) 350 | b.Run(fmt.Sprintf("%s_%d", name[strings.Index(name, ".")+1:], scale), func(b *testing.B) { 351 | histogram := constructor() 352 | tester.preload(b.N) 353 | b.ResetTimer() 354 | for i := 0; i < b.N; i++ { 355 | if err := tester.run(histogram); err != nil { 356 | b.Error(err) 357 | } 358 | } 359 | }) 360 | } 361 | } 362 | } 363 | 364 | // TestCustomRoundTripping tests that clients using the HistogramWithoutLookups 365 | // structure for custom serialization and deserialization get interchangeable 366 | // behavior with the default spec. 367 | func TestCustomRoundTripping(t *testing.T) { 368 | h := New() 369 | rand.Seed(time.Now().UnixNano()) 370 | for i := 0; i < 100; i++ { 371 | if err := h.RecordIntScale(rand.Int63(), rand.Int()); err != nil { 372 | t.Fatalf("could not record numeric value: %v", err) 373 | } 374 | } 375 | 376 | defaultBytes, err := json.Marshal(h) 377 | if err != nil { 378 | t.Fatalf("could not marshal histogram: %v", err) 379 | } 380 | 381 | withoutLookupBytes, err := json.Marshal(&HistogramWithoutLookups{histogram: h}) 382 | if err != nil { 383 | t.Fatalf("could not marshal histogram: %v", err) 384 | } 385 | 386 | if !reflect.DeepEqual(defaultBytes, withoutLookupBytes) { 387 | t.Fatalf("histogram without lookups serialized into something different than default: expected %v, got %v", defaultBytes, withoutLookupBytes) 388 | } 389 | 390 | for source, data := range map[string][]byte{ 391 | "default": defaultBytes, 392 | "withoutLookups": withoutLookupBytes, 393 | } { 394 | var deserializedWithoutLookups HistogramWithoutLookups 395 | if err := json.Unmarshal(data, &deserializedWithoutLookups); err != nil { 396 | t.Fatalf("could not deserialize %s bytes into custom struct: %v", source, err) 397 | } 398 | if deserializedWithoutLookups.histogram.useLookup != false || deserializedWithoutLookups.histogram.lookup != nil { 399 | t.Errorf("after deserializing %s bytes into custom struct, got allocated lookup table", source) 400 | } 401 | extracted := deserializedWithoutLookups.HistogramWithLookups() 402 | if extracted.useLookup != true || len(extracted.lookup) != 256 { 403 | t.Errorf("after deserializing %s bytes into cutom struct and extracting with lookups, did not get allocated lookup table", source) 404 | } 405 | 406 | var deserializedDefault Histogram 407 | if err := json.Unmarshal(data, &deserializedDefault); err != nil { 408 | t.Fatalf("could not deserialize %s bytes into default struct: %v", source, err) 409 | } 410 | if deserializedDefault.useLookup != true || len(deserializedDefault.lookup) != 256 { 411 | t.Errorf("after deserializing %s bytes into default struct, did not get allocated lookup table", source) 412 | } 413 | } 414 | } 415 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/openhistogram/circonusllhist 2 | 3 | go 1.16 4 | --------------------------------------------------------------------------------