├── .gitignore ├── .travis.yml ├── LICENCE.txt ├── README.md ├── whisper.go ├── whisper_test.go └── whisper_test.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.test 2 | cpu.out 3 | tags 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.0 5 | - 1.1 6 | - 1.2 7 | - tip 8 | -------------------------------------------------------------------------------- /LICENCE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012 Rob Young. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of Google Inc. nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Go Whisper 2 | 3 | [![Build Status](https://travis-ci.org/robyoung/go-whisper.png?branch=master)](https://travis-ci.org/robyoung/go-whisper?branch=master) 4 | 5 | Go Whisper is a [Go](http://golang.org/) implementation of the [Whisper](https://github.com/graphite-project/whisper) database, which is part of the [Graphite Project](http://graphite.wikidot.com/). 6 | 7 | To create a new whisper database you must define it's retention levels (see: [storage schemas](https://graphite.readthedocs.io/en/latest/config-carbon.html#storage-schemas-conf)), aggregation method and the xFilesFactor. The xFilesFactor specifies the fraction of data points in a propagation interval that must have known values for a propagation to occur. 8 | 9 | ## Examples 10 | 11 | Create a new whisper database in "/tmp/test.wsp" with two retention levels (1 second for 1 day and 1 hour for 5 weeks), it will sum values when propagating them to the next retention level, and it requires half the values of the first retention level to be set before they are propagated. 12 | ```go 13 | retentions, err := whisper.ParseRetentionDefs("1s:1d,1h:5w") 14 | if err == nil { 15 | wsp, err := whisper.Create("/tmp/test.wsp", retentions, whisper.Sum, 0.5) 16 | } 17 | ``` 18 | 19 | Alternatively you can open an existing whisper database. 20 | ```go 21 | wsp, err := whisper.Open("/tmp/test.wsp") 22 | ``` 23 | 24 | Once you have a whisper database you can set values at given time points. This sets the time point 1 hour ago to 12345.678. 25 | ```go 26 | offset, _ := time.ParseDuration("-1h") 27 | wsp.Update(12345.678, int(time.Now().Add(offset).Unix())) 28 | ``` 29 | 30 | And you can retrieve time series from it. This example fetches a time series for the last 1 hour and then iterates through it's points. 31 | ```go 32 | offset, _ := time.ParseDuration("-1h") 33 | series, err := wsp.Fetch(int(time.Now().Add(offset).Unix()), int(time.Now().Unix())) 34 | if err != nil { 35 | // handle 36 | } 37 | for _, point := range series.Points() { 38 | fmt.Println(point.Time, point.Value) 39 | } 40 | ``` 41 | 42 | ## Thread Safety 43 | 44 | This implementation is *not* thread safe. Writing to a database concurrently will cause bad things to happen. It is up to the user to manage this in their application as they need to. 45 | 46 | ## Licence 47 | 48 | Go Whisper is licenced under a BSD Licence. 49 | -------------------------------------------------------------------------------- /whisper.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package whisper implements Graphite's Whisper database format 3 | */ 4 | package whisper 5 | 6 | import ( 7 | "encoding/binary" 8 | "fmt" 9 | "math" 10 | "os" 11 | "regexp" 12 | "sort" 13 | "strconv" 14 | "strings" 15 | "syscall" 16 | "time" 17 | ) 18 | 19 | const ( 20 | IntSize = 4 21 | FloatSize = 4 22 | Float64Size = 8 23 | PointSize = 12 24 | MetadataSize = 16 25 | ArchiveInfoSize = 12 26 | ) 27 | 28 | const ( 29 | Seconds = 1 30 | Minutes = 60 31 | Hours = 3600 32 | Days = 86400 33 | Weeks = 86400 * 7 34 | Years = 86400 * 365 35 | ) 36 | 37 | type AggregationMethod int 38 | 39 | const ( 40 | Average AggregationMethod = iota + 1 41 | Sum 42 | Last 43 | Max 44 | Min 45 | ) 46 | 47 | func unitMultiplier(s string) (int, error) { 48 | switch { 49 | case strings.HasPrefix(s, "s"): 50 | return Seconds, nil 51 | case strings.HasPrefix(s, "m"): 52 | return Minutes, nil 53 | case strings.HasPrefix(s, "h"): 54 | return Hours, nil 55 | case strings.HasPrefix(s, "d"): 56 | return Days, nil 57 | case strings.HasPrefix(s, "w"): 58 | return Weeks, nil 59 | case strings.HasPrefix(s, "y"): 60 | return Years, nil 61 | } 62 | return 0, fmt.Errorf("Invalid unit multiplier [%v]", s) 63 | } 64 | 65 | var retentionRegexp *regexp.Regexp = regexp.MustCompile("^(\\d+)([smhdwy]+)$") 66 | 67 | func parseRetentionPart(retentionPart string) (int, error) { 68 | part, err := strconv.ParseInt(retentionPart, 10, 32) 69 | if err == nil { 70 | return int(part), nil 71 | } 72 | if !retentionRegexp.MatchString(retentionPart) { 73 | return 0, fmt.Errorf("%v", retentionPart) 74 | } 75 | matches := retentionRegexp.FindStringSubmatch(retentionPart) 76 | value, err := strconv.ParseInt(matches[1], 10, 32) 77 | if err != nil { 78 | panic(fmt.Sprintf("Regex on %v is borked, %v cannot be parsed as int", retentionPart, matches[1])) 79 | } 80 | multiplier, err := unitMultiplier(matches[2]) 81 | return multiplier * int(value), err 82 | } 83 | 84 | /* 85 | Parse a retention definition as you would find in the storage-schemas.conf of a Carbon install. 86 | Note that this only parses a single retention definition, if you have multiple definitions (separated by a comma) 87 | you will have to split them yourself. 88 | 89 | ParseRetentionDef("10s:14d") Retention{10, 120960} 90 | 91 | See: http://graphite.readthedocs.org/en/1.0/config-carbon.html#storage-schemas-conf 92 | */ 93 | func ParseRetentionDef(retentionDef string) (*Retention, error) { 94 | parts := strings.Split(retentionDef, ":") 95 | if len(parts) != 2 { 96 | return nil, fmt.Errorf("Not enough parts in retentionDef [%v]", retentionDef) 97 | } 98 | precision, err := parseRetentionPart(parts[0]) 99 | if err != nil { 100 | return nil, fmt.Errorf("Failed to parse precision: %v", err) 101 | } 102 | 103 | points, err := parseRetentionPart(parts[1]) 104 | if err != nil { 105 | return nil, fmt.Errorf("Failed to parse points: %v", err) 106 | } 107 | points /= precision 108 | 109 | return &Retention{precision, points}, err 110 | } 111 | 112 | func ParseRetentionDefs(retentionDefs string) (Retentions, error) { 113 | retentions := make(Retentions, 0) 114 | for _, retentionDef := range strings.Split(retentionDefs, ",") { 115 | retention, err := ParseRetentionDef(retentionDef) 116 | if err != nil { 117 | return nil, err 118 | } 119 | retentions = append(retentions, retention) 120 | } 121 | return retentions, nil 122 | } 123 | 124 | /* 125 | Represents a Whisper database file. 126 | */ 127 | type Whisper struct { 128 | file *os.File 129 | 130 | // Metadata 131 | aggregationMethod AggregationMethod 132 | maxRetention int 133 | xFilesFactor float32 134 | archives []archiveInfo 135 | } 136 | 137 | /* 138 | Create a new Whisper database file and write it's header. 139 | */ 140 | func Create(path string, retentions Retentions, aggregationMethod AggregationMethod, xFilesFactor float32) (whisper *Whisper, err error) { 141 | sort.Sort(retentionsByPrecision{retentions}) 142 | if err = validateRetentions(retentions); err != nil { 143 | return nil, err 144 | } 145 | _, err = os.Stat(path) 146 | if err == nil { 147 | return nil, os.ErrExist 148 | } 149 | file, err := os.Create(path) 150 | if err != nil { 151 | return nil, err 152 | } 153 | 154 | // Lock file as carbon-cache.py would 155 | if err = syscall.Flock(int(file.Fd()), syscall.LOCK_EX); err != nil { 156 | file.Close() 157 | return nil, err 158 | } 159 | whisper = new(Whisper) 160 | 161 | // Set the metadata 162 | whisper.file = file 163 | whisper.aggregationMethod = aggregationMethod 164 | whisper.xFilesFactor = xFilesFactor 165 | for _, retention := range retentions { 166 | if retention.MaxRetention() > whisper.maxRetention { 167 | whisper.maxRetention = retention.MaxRetention() 168 | } 169 | } 170 | 171 | // Set the archive info 172 | offset := MetadataSize + (ArchiveInfoSize * len(retentions)) 173 | whisper.archives = make([]archiveInfo, 0, len(retentions)) 174 | for _, retention := range retentions { 175 | whisper.archives = append(whisper.archives, archiveInfo{*retention, offset}) 176 | offset += retention.Size() 177 | } 178 | 179 | err = whisper.writeHeader() 180 | if err != nil { 181 | return nil, err 182 | } 183 | 184 | // pre-allocate file size, fallocate proved slower 185 | remaining := whisper.Size() - whisper.MetadataSize() 186 | chunkSize := 16384 187 | zeros := make([]byte, chunkSize) 188 | for remaining > chunkSize { 189 | whisper.file.Write(zeros) 190 | remaining -= chunkSize 191 | } 192 | whisper.file.Write(zeros[:remaining]) 193 | whisper.file.Sync() 194 | 195 | return whisper, nil 196 | } 197 | 198 | func validateRetentions(retentions Retentions) error { 199 | if len(retentions) == 0 { 200 | return fmt.Errorf("No retentions") 201 | } 202 | for i, retention := range retentions { 203 | if i == len(retentions)-1 { 204 | break 205 | } 206 | 207 | nextRetention := retentions[i+1] 208 | if !(retention.secondsPerPoint < nextRetention.secondsPerPoint) { 209 | return fmt.Errorf("A Whisper database may not be configured having two archives with the same precision (archive%v: %v, archive%v: %v)", i, retention, i+1, nextRetention) 210 | } 211 | 212 | if mod(nextRetention.secondsPerPoint, retention.secondsPerPoint) != 0 { 213 | return fmt.Errorf("Higher precision archives' precision must evenly divide all lower precision archives' precision (archive%v: %v, archive%v: %v)", i, retention.secondsPerPoint, i+1, nextRetention.secondsPerPoint) 214 | } 215 | 216 | if retention.MaxRetention() >= nextRetention.MaxRetention() { 217 | return fmt.Errorf("Lower precision archives must cover larger time intervals than higher precision archives (archive%v: %v seconds, archive%v: %v seconds)", i, retention.MaxRetention(), i+1, nextRetention.MaxRetention()) 218 | } 219 | 220 | if retention.numberOfPoints < (nextRetention.secondsPerPoint / retention.secondsPerPoint) { 221 | return fmt.Errorf("Each archive must have at least enough points to consolidate to the next archive (archive%v consolidates %v of archive%v's points but it has only %v total points)", i+1, nextRetention.secondsPerPoint/retention.secondsPerPoint, i, retention.numberOfPoints) 222 | } 223 | } 224 | return nil 225 | } 226 | 227 | /* 228 | Open an existing Whisper database and read it's header 229 | */ 230 | func Open(path string) (whisper *Whisper, err error) { 231 | file, err := os.OpenFile(path, os.O_RDWR, 0666) 232 | if os.IsPermission(err) { 233 | file, err = os.OpenFile(path, os.O_RDONLY, 0666) 234 | } 235 | if err != nil { 236 | return nil, err 237 | } 238 | // Lock file as carbon-cache.py would 239 | if err = syscall.Flock(int(file.Fd()), syscall.LOCK_EX); err != nil { 240 | file.Close() 241 | return nil, err 242 | } 243 | 244 | whisper = new(Whisper) 245 | whisper.file = file 246 | 247 | // read the metadata 248 | b := make([]byte, MetadataSize) 249 | offset := 0 250 | file.Read(b) 251 | whisper.aggregationMethod = AggregationMethod(unpackInt(b[offset : offset+IntSize])) 252 | offset += IntSize 253 | whisper.maxRetention = unpackInt(b[offset : offset+IntSize]) 254 | offset += IntSize 255 | whisper.xFilesFactor = unpackFloat32(b[offset : offset+FloatSize]) 256 | offset += FloatSize 257 | archiveCount := unpackInt(b[offset : offset+IntSize]) 258 | offset += IntSize 259 | 260 | // read the archive info 261 | b = make([]byte, ArchiveInfoSize*archiveCount) 262 | file.Read(b) 263 | whisper.archives = make([]archiveInfo, archiveCount) 264 | for i := 0; i < archiveCount; i++ { 265 | whisper.archives[i] = unpackArchiveInfo(b[i*ArchiveInfoSize : (i+1)*ArchiveInfoSize]) 266 | } 267 | 268 | return whisper, nil 269 | } 270 | 271 | func (whisper *Whisper) writeHeader() (err error) { 272 | b := make([]byte, whisper.MetadataSize()) 273 | i := 0 274 | i += packInt(b, int(whisper.aggregationMethod), i) 275 | i += packInt(b, whisper.maxRetention, i) 276 | i += packFloat32(b, whisper.xFilesFactor, i) 277 | i += packInt(b, len(whisper.archives), i) 278 | for _, archive := range whisper.archives { 279 | i += packInt(b, archive.offset, i) 280 | i += packInt(b, archive.secondsPerPoint, i) 281 | i += packInt(b, archive.numberOfPoints, i) 282 | } 283 | _, err = whisper.file.Write(b) 284 | 285 | return err 286 | } 287 | 288 | /* 289 | Close the whisper file 290 | */ 291 | func (whisper *Whisper) Close() { 292 | // This releases any held Flock style locks 293 | whisper.file.Close() 294 | } 295 | 296 | /* 297 | Calculate the total number of bytes the Whisper file should be according to the metadata. 298 | */ 299 | func (whisper *Whisper) Size() int { 300 | size := whisper.MetadataSize() 301 | for _, archive := range whisper.archives { 302 | size += archive.Size() 303 | } 304 | return size 305 | } 306 | 307 | /* 308 | Calculate the number of bytes the metadata section will be. 309 | */ 310 | func (whisper *Whisper) MetadataSize() int { 311 | return MetadataSize + (ArchiveInfoSize * len(whisper.archives)) 312 | } 313 | 314 | /* 315 | Update a value in the database. 316 | 317 | If the timestamp is in the future or outside of the maximum retention it will 318 | fail immediately. 319 | */ 320 | func (whisper *Whisper) Update(value float64, timestamp int) (err error) { 321 | diff := int(time.Now().Unix()) - timestamp 322 | if !(diff < whisper.maxRetention && diff >= 0) { 323 | return fmt.Errorf("Timestamp not covered by any archives in this database") 324 | } 325 | var archive archiveInfo 326 | var lowerArchives []archiveInfo 327 | var i int 328 | for i, archive = range whisper.archives { 329 | if archive.MaxRetention() < diff { 330 | continue 331 | } 332 | lowerArchives = whisper.archives[i+1:] // TODO: investigate just returning the positions 333 | break 334 | } 335 | 336 | myInterval := timestamp - mod(timestamp, archive.secondsPerPoint) 337 | point := dataPoint{myInterval, value} 338 | 339 | _, err = whisper.file.WriteAt(point.Bytes(), whisper.getPointOffset(myInterval, &archive)) 340 | if err != nil { 341 | return err 342 | } 343 | 344 | higher := archive 345 | for _, lower := range lowerArchives { 346 | propagated, err := whisper.propagate(myInterval, &higher, &lower) 347 | if err != nil { 348 | return err 349 | } else if !propagated { 350 | break 351 | } 352 | higher = lower 353 | } 354 | 355 | return nil 356 | } 357 | 358 | func (whisper *Whisper) UpdateMany(points []*TimeSeriesPoint) { 359 | // sort the points, newest first 360 | sort.Sort(timeSeriesPointsNewestFirst{points}) 361 | 362 | now := int(time.Now().Unix()) // TODO: danger of 2030 something overflow 363 | 364 | var currentPoints []*TimeSeriesPoint 365 | for _, archive := range whisper.archives { 366 | currentPoints, points = extractPoints(points, now, archive.MaxRetention()) 367 | if len(currentPoints) == 0 { 368 | continue 369 | } 370 | // reverse currentPoints 371 | for i, j := 0, len(currentPoints)-1; i < j; i, j = i+1, j-1 { 372 | currentPoints[i], currentPoints[j] = currentPoints[j], currentPoints[i] 373 | } 374 | whisper.archiveUpdateMany(&archive, currentPoints) 375 | 376 | if len(points) == 0 { // nothing left to do 377 | break 378 | } 379 | } 380 | } 381 | 382 | func (whisper *Whisper) archiveUpdateMany(archive *archiveInfo, points []*TimeSeriesPoint) { 383 | alignedPoints := alignPoints(archive, points) 384 | intervals, packedBlocks := packSequences(archive, alignedPoints) 385 | 386 | baseInterval := whisper.getBaseInterval(archive) 387 | if baseInterval == 0 { 388 | baseInterval = intervals[0] 389 | } 390 | 391 | for i := range intervals { 392 | myOffset := archive.PointOffset(baseInterval, intervals[i]) 393 | bytesBeyond := int(myOffset-archive.End()) + len(packedBlocks[i]) 394 | if bytesBeyond > 0 { 395 | pos := len(packedBlocks[i]) - bytesBeyond 396 | whisper.file.WriteAt(packedBlocks[i][:pos], myOffset) 397 | whisper.file.WriteAt(packedBlocks[i][pos:], archive.Offset()) 398 | } else { 399 | whisper.file.WriteAt(packedBlocks[i], myOffset) 400 | } 401 | } 402 | 403 | higher := *archive 404 | lowerArchives := whisper.lowerArchives(archive) 405 | 406 | for _, lower := range lowerArchives { 407 | seen := make(map[int]bool) 408 | propagateFurther := false 409 | for _, point := range alignedPoints { 410 | interval := point.interval - mod(point.interval, lower.secondsPerPoint) 411 | if !seen[interval] { 412 | if propagated, err := whisper.propagate(interval, &higher, &lower); err != nil { 413 | panic("Failed to propagate") 414 | } else if propagated { 415 | propagateFurther = true 416 | } 417 | } 418 | } 419 | if !propagateFurther { 420 | break 421 | } 422 | higher = lower 423 | } 424 | } 425 | 426 | func extractPoints(points []*TimeSeriesPoint, now int, maxRetention int) (currentPoints []*TimeSeriesPoint, remainingPoints []*TimeSeriesPoint) { 427 | maxAge := now - maxRetention 428 | for i, point := range points { 429 | if point.Time < maxAge { 430 | if i > 0 { 431 | return points[:i-1], points[i-1:] 432 | } else { 433 | return []*TimeSeriesPoint{}, points 434 | } 435 | } 436 | } 437 | return points, remainingPoints 438 | } 439 | 440 | func alignPoints(archive *archiveInfo, points []*TimeSeriesPoint) []dataPoint { 441 | alignedPoints := make([]dataPoint, 0, len(points)) 442 | positions := make(map[int]int) 443 | for _, point := range points { 444 | dPoint := dataPoint{point.Time - mod(point.Time, archive.secondsPerPoint), point.Value} 445 | if p, ok := positions[dPoint.interval]; ok { 446 | alignedPoints[p] = dPoint 447 | } else { 448 | alignedPoints = append(alignedPoints, dPoint) 449 | positions[dPoint.interval] = len(alignedPoints) - 1 450 | } 451 | } 452 | return alignedPoints 453 | } 454 | 455 | func packSequences(archive *archiveInfo, points []dataPoint) (intervals []int, packedBlocks [][]byte) { 456 | intervals = make([]int, 0) 457 | packedBlocks = make([][]byte, 0) 458 | for i, point := range points { 459 | if i == 0 || point.interval != intervals[len(intervals)-1]+archive.secondsPerPoint { 460 | intervals = append(intervals, point.interval) 461 | packedBlocks = append(packedBlocks, point.Bytes()) 462 | } else { 463 | packedBlocks[len(packedBlocks)-1] = append(packedBlocks[len(packedBlocks)-1], point.Bytes()...) 464 | } 465 | } 466 | return 467 | } 468 | 469 | /* 470 | Calculate the offset for a given interval in an archive 471 | 472 | This method retrieves the baseInterval and the 473 | */ 474 | func (whisper *Whisper) getPointOffset(start int, archive *archiveInfo) int64 { 475 | baseInterval := whisper.getBaseInterval(archive) 476 | if baseInterval == 0 { 477 | return archive.Offset() 478 | } 479 | return archive.PointOffset(baseInterval, start) 480 | } 481 | 482 | func (whisper *Whisper) getBaseInterval(archive *archiveInfo) int { 483 | baseInterval, err := whisper.readInt(archive.Offset()) 484 | if err != nil { 485 | panic("Failed to read baseInterval") 486 | } 487 | return baseInterval 488 | } 489 | 490 | func (whisper *Whisper) lowerArchives(archive *archiveInfo) (lowerArchives []archiveInfo) { 491 | for i, lower := range whisper.archives { 492 | if lower.secondsPerPoint > archive.secondsPerPoint { 493 | return whisper.archives[i:] 494 | } 495 | } 496 | return 497 | } 498 | 499 | func (whisper *Whisper) propagate(timestamp int, higher, lower *archiveInfo) (bool, error) { 500 | lowerIntervalStart := timestamp - mod(timestamp, lower.secondsPerPoint) 501 | 502 | higherFirstOffset := whisper.getPointOffset(lowerIntervalStart, higher) 503 | 504 | // TODO: extract all this series extraction stuff 505 | higherPoints := lower.secondsPerPoint / higher.secondsPerPoint 506 | higherSize := higherPoints * PointSize 507 | relativeFirstOffset := higherFirstOffset - higher.Offset() 508 | relativeLastOffset := int64(mod(int(relativeFirstOffset+int64(higherSize)), higher.Size())) 509 | higherLastOffset := relativeLastOffset + higher.Offset() 510 | 511 | series := whisper.readSeries(higherFirstOffset, higherLastOffset, higher) 512 | 513 | // and finally we construct a list of values 514 | knownValues := make([]float64, 0, len(series)) 515 | currentInterval := lowerIntervalStart 516 | 517 | for _, dPoint := range series { 518 | if dPoint.interval == currentInterval { 519 | knownValues = append(knownValues, dPoint.value) 520 | } 521 | currentInterval += higher.secondsPerPoint 522 | } 523 | 524 | // propagate aggregateValue to propagate from neighborValues if we have enough known points 525 | if len(knownValues) == 0 { 526 | return false, nil 527 | } 528 | knownPercent := float32(len(knownValues)) / float32(len(series)) 529 | if knownPercent < whisper.xFilesFactor { // check we have enough data points to propagate a value 530 | return false, nil 531 | } else { 532 | aggregateValue := aggregate(whisper.aggregationMethod, knownValues) 533 | point := dataPoint{lowerIntervalStart, aggregateValue} 534 | whisper.file.WriteAt(point.Bytes(), whisper.getPointOffset(lowerIntervalStart, lower)) 535 | } 536 | return true, nil 537 | } 538 | 539 | // TODO: add error handling 540 | func (whisper *Whisper) readSeries(start, end int64, archive *archiveInfo) []dataPoint { 541 | var b []byte 542 | if start < end { 543 | b = make([]byte, end-start) 544 | whisper.file.ReadAt(b, start) 545 | } else { 546 | b = make([]byte, archive.End()-start) 547 | whisper.file.ReadAt(b, start) 548 | b2 := make([]byte, end-archive.Offset()) 549 | whisper.file.ReadAt(b2, archive.Offset()) 550 | b = append(b, b2...) 551 | } 552 | return unpackDataPoints(b) 553 | } 554 | 555 | /* 556 | Calculate the starting time for a whisper db. 557 | */ 558 | func (whisper *Whisper) StartTime() int { 559 | now := int(time.Now().Unix()) // TODO: danger of 2030 something overflow 560 | return now - whisper.maxRetention 561 | } 562 | 563 | /* 564 | Fetch a TimeSeries for a given time span from the file. 565 | */ 566 | func (whisper *Whisper) Fetch(fromTime, untilTime int) (timeSeries *TimeSeries, err error) { 567 | now := int(time.Now().Unix()) // TODO: danger of 2030 something overflow 568 | if fromTime > untilTime { 569 | return nil, fmt.Errorf("Invalid time interval: from time '%d' is after until time '%d'", fromTime, untilTime) 570 | } 571 | oldestTime := whisper.StartTime() 572 | // range is in the future 573 | if fromTime > now { 574 | return nil, nil 575 | } 576 | // range is beyond retention 577 | if untilTime < oldestTime { 578 | return nil, nil 579 | } 580 | if fromTime < oldestTime { 581 | fromTime = oldestTime 582 | } 583 | if untilTime > now { 584 | untilTime = now 585 | } 586 | 587 | // TODO: improve this algorithm it's ugly 588 | diff := now - fromTime 589 | var archive archiveInfo 590 | for _, archive = range whisper.archives { 591 | if archive.MaxRetention() >= diff { 592 | break 593 | } 594 | } 595 | 596 | fromInterval := archive.Interval(fromTime) 597 | untilInterval := archive.Interval(untilTime) 598 | baseInterval := whisper.getBaseInterval(&archive) 599 | 600 | if baseInterval == 0 { 601 | step := archive.secondsPerPoint 602 | points := (untilInterval - fromInterval) / step 603 | values := make([]float64, points) 604 | for i, _ := range values { 605 | values[i] = math.NaN() 606 | } 607 | return &TimeSeries{fromInterval, untilInterval, step, values}, nil 608 | } 609 | 610 | fromOffset := archive.PointOffset(baseInterval, fromInterval) 611 | untilOffset := archive.PointOffset(baseInterval, untilInterval) 612 | 613 | series := whisper.readSeries(fromOffset, untilOffset, &archive) 614 | 615 | values := make([]float64, len(series)) 616 | for i, _ := range values { 617 | values[i] = math.NaN() 618 | } 619 | currentInterval := fromInterval 620 | step := archive.secondsPerPoint 621 | 622 | for i, dPoint := range series { 623 | if dPoint.interval == currentInterval { 624 | values[i] = dPoint.value 625 | } 626 | currentInterval += step 627 | } 628 | 629 | return &TimeSeries{fromInterval, untilInterval, step, values}, nil 630 | } 631 | 632 | func (whisper *Whisper) readInt(offset int64) (int, error) { 633 | // TODO: make errors better 634 | b := make([]byte, IntSize) 635 | _, err := whisper.file.ReadAt(b, offset) 636 | if err != nil { 637 | return 0, err 638 | } 639 | 640 | return unpackInt(b), nil 641 | } 642 | 643 | /* 644 | A retention level. 645 | 646 | Retention levels describe a given archive in the database. How detailed it is and how far back 647 | it records. 648 | */ 649 | type Retention struct { 650 | secondsPerPoint int 651 | numberOfPoints int 652 | } 653 | 654 | func (retention *Retention) MaxRetention() int { 655 | return retention.secondsPerPoint * retention.numberOfPoints 656 | } 657 | 658 | func (retention *Retention) Size() int { 659 | return retention.numberOfPoints * PointSize 660 | } 661 | 662 | type Retentions []*Retention 663 | 664 | func (r Retentions) Len() int { 665 | return len(r) 666 | } 667 | 668 | func (r Retentions) Swap(i, j int) { 669 | r[i], r[j] = r[j], r[i] 670 | } 671 | 672 | type retentionsByPrecision struct{ Retentions } 673 | 674 | func (r retentionsByPrecision) Less(i, j int) bool { 675 | return r.Retentions[i].secondsPerPoint < r.Retentions[j].secondsPerPoint 676 | } 677 | 678 | /* 679 | Describes a time series in a file. 680 | 681 | The only addition this type has over a Retention is the offset at which it exists within the 682 | whisper file. 683 | */ 684 | type archiveInfo struct { 685 | Retention 686 | offset int 687 | } 688 | 689 | func (archive *archiveInfo) Offset() int64 { 690 | return int64(archive.offset) 691 | } 692 | 693 | func (archive *archiveInfo) PointOffset(baseInterval, interval int) int64 { 694 | timeDistance := interval - baseInterval 695 | pointDistance := timeDistance / archive.secondsPerPoint 696 | byteDistance := pointDistance * PointSize 697 | myOffset := archive.Offset() + int64(mod(byteDistance, archive.Size())) 698 | 699 | return myOffset 700 | } 701 | 702 | func (archive *archiveInfo) End() int64 { 703 | return archive.Offset() + int64(archive.Size()) 704 | } 705 | 706 | func (archive *archiveInfo) Interval(time int) int { 707 | return time - mod(time, archive.secondsPerPoint) + archive.secondsPerPoint 708 | } 709 | 710 | type TimeSeries struct { 711 | fromTime int 712 | untilTime int 713 | step int 714 | values []float64 715 | } 716 | 717 | func (ts *TimeSeries) FromTime() int { 718 | return ts.fromTime 719 | } 720 | 721 | func (ts *TimeSeries) UntilTime() int { 722 | return ts.untilTime 723 | } 724 | 725 | func (ts *TimeSeries) Step() int { 726 | return ts.step 727 | } 728 | 729 | func (ts *TimeSeries) Values() []float64 { 730 | return ts.values 731 | } 732 | 733 | func (ts *TimeSeries) Points() []*TimeSeriesPoint { 734 | points := make([]*TimeSeriesPoint, len(ts.values)) 735 | for i, value := range ts.values { 736 | points[i] = &TimeSeriesPoint{Time: ts.fromTime + ts.step*i, Value: value} 737 | } 738 | return points 739 | } 740 | 741 | func (ts *TimeSeries) String() string { 742 | return fmt.Sprintf("TimeSeries{'%v' '%-v' %v %v}", time.Unix(int64(ts.fromTime), 0), time.Unix(int64(ts.untilTime), 0), ts.step, ts.values) 743 | } 744 | 745 | type TimeSeriesPoint struct { 746 | Time int 747 | Value float64 748 | } 749 | 750 | type timeSeriesPoints []*TimeSeriesPoint 751 | 752 | func (p timeSeriesPoints) Len() int { 753 | return len(p) 754 | } 755 | 756 | func (p timeSeriesPoints) Swap(i, j int) { 757 | p[i], p[j] = p[j], p[i] 758 | } 759 | 760 | type timeSeriesPointsNewestFirst struct { 761 | timeSeriesPoints 762 | } 763 | 764 | func (p timeSeriesPointsNewestFirst) Less(i, j int) bool { 765 | return p.timeSeriesPoints[i].Time > p.timeSeriesPoints[j].Time 766 | } 767 | 768 | type dataPoint struct { 769 | interval int 770 | value float64 771 | } 772 | 773 | func (point *dataPoint) Bytes() []byte { 774 | b := make([]byte, PointSize) 775 | packInt(b, point.interval, 0) 776 | packFloat64(b, point.value, IntSize) 777 | return b 778 | } 779 | 780 | func sum(values []float64) float64 { 781 | result := 0.0 782 | for _, value := range values { 783 | result += value 784 | } 785 | return result 786 | } 787 | 788 | func aggregate(method AggregationMethod, knownValues []float64) float64 { 789 | switch method { 790 | case Average: 791 | return sum(knownValues) / float64(len(knownValues)) 792 | case Sum: 793 | return sum(knownValues) 794 | case Last: 795 | return knownValues[len(knownValues)-1] 796 | case Max: 797 | max := knownValues[0] 798 | for _, value := range knownValues { 799 | if value > max { 800 | max = value 801 | } 802 | } 803 | return max 804 | case Min: 805 | min := knownValues[0] 806 | for _, value := range knownValues { 807 | if value < min { 808 | min = value 809 | } 810 | } 811 | return min 812 | } 813 | panic("Invalid aggregation method") 814 | } 815 | 816 | func packInt(b []byte, v, i int) int { 817 | binary.BigEndian.PutUint32(b[i:i+IntSize], uint32(v)) 818 | return IntSize 819 | } 820 | 821 | func packFloat32(b []byte, v float32, i int) int { 822 | binary.BigEndian.PutUint32(b[i:i+FloatSize], math.Float32bits(v)) 823 | return FloatSize 824 | } 825 | 826 | func packFloat64(b []byte, v float64, i int) int { 827 | binary.BigEndian.PutUint64(b[i:i+Float64Size], math.Float64bits(v)) 828 | return Float64Size 829 | } 830 | 831 | func unpackInt(b []byte) int { 832 | return int(binary.BigEndian.Uint32(b)) 833 | } 834 | 835 | func unpackFloat32(b []byte) float32 { 836 | return math.Float32frombits(binary.BigEndian.Uint32(b)) 837 | } 838 | 839 | func unpackFloat64(b []byte) float64 { 840 | return math.Float64frombits(binary.BigEndian.Uint64(b)) 841 | } 842 | 843 | func unpackArchiveInfo(b []byte) archiveInfo { 844 | return archiveInfo{Retention{unpackInt(b[IntSize : IntSize*2]), unpackInt(b[IntSize*2 : IntSize*3])}, unpackInt(b[:IntSize])} 845 | } 846 | 847 | func unpackDataPoint(b []byte) dataPoint { 848 | return dataPoint{unpackInt(b[0:IntSize]), unpackFloat64(b[IntSize:PointSize])} 849 | } 850 | 851 | func unpackDataPoints(b []byte) (series []dataPoint) { 852 | series = make([]dataPoint, 0, len(b)/PointSize) 853 | for i := 0; i < len(b); i += PointSize { 854 | series = append(series, unpackDataPoint(b[i:i+PointSize])) 855 | } 856 | return 857 | } 858 | 859 | /* 860 | Implementation of modulo that works like Python 861 | Thanks @timmow for this 862 | */ 863 | func mod(a, b int) int { 864 | return a - (b * int(math.Floor(float64(a)/float64(b)))) 865 | } 866 | -------------------------------------------------------------------------------- /whisper_test.go: -------------------------------------------------------------------------------- 1 | package whisper 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "os" 7 | "sort" 8 | "testing" 9 | "time" 10 | ) 11 | 12 | func checkBytes(t *testing.T, expected, received []byte) { 13 | if len(expected) != len(received) { 14 | t.Fatalf("Invalid number of bytes. Expected %v, received %v", len(expected), len(received)) 15 | } 16 | for i := range expected { 17 | if expected[i] != received[i] { 18 | t.Fatalf("Incorrect byte at %v. Expected %v, received %v", i+1, expected[i], received[i]) 19 | } 20 | } 21 | } 22 | 23 | func testParseRetentionDef(t *testing.T, retentionDef string, expectedPrecision, expectedPoints int, hasError bool) { 24 | errTpl := fmt.Sprintf("Expected %%v to be %%v but received %%v for retentionDef %v", retentionDef) 25 | 26 | retention, err := ParseRetentionDef(retentionDef) 27 | 28 | if (err == nil && hasError) || (err != nil && !hasError) { 29 | if hasError { 30 | t.Fatalf("Expected error but received none for retentionDef %v", retentionDef) 31 | } else { 32 | t.Fatalf("Expected no error but received %v for retentionDef %v", err, retentionDef) 33 | } 34 | } 35 | if err == nil { 36 | if retention.secondsPerPoint != expectedPrecision { 37 | t.Fatalf(errTpl, "precision", expectedPrecision, retention.secondsPerPoint) 38 | } 39 | if retention.numberOfPoints != expectedPoints { 40 | t.Fatalf(errTpl, "points", expectedPoints, retention.numberOfPoints) 41 | } 42 | } 43 | } 44 | 45 | func TestParseRetentionDef(t *testing.T) { 46 | testParseRetentionDef(t, "1s:5m", 1, 300, false) 47 | testParseRetentionDef(t, "1m:30m", 60, 30, false) 48 | testParseRetentionDef(t, "1m", 0, 0, true) 49 | testParseRetentionDef(t, "1m:30m:20s", 0, 0, true) 50 | testParseRetentionDef(t, "1f:30s", 0, 0, true) 51 | testParseRetentionDef(t, "1m:30f", 0, 0, true) 52 | } 53 | 54 | func TestParseRetentionDefs(t *testing.T) { 55 | retentions, err := ParseRetentionDefs("1s:5m,1m:30m") 56 | if err != nil { 57 | t.Fatalf("Unexpected error: %v", err) 58 | } 59 | if length := len(retentions); length != 2 { 60 | t.Fatalf("Expected 2 retentions, received %v", length) 61 | } 62 | } 63 | 64 | func TestSortRetentions(t *testing.T) { 65 | retentions := Retentions{{300, 12}, {60, 30}, {1, 300}} 66 | sort.Sort(retentionsByPrecision{retentions}) 67 | if retentions[0].secondsPerPoint != 1 { 68 | t.Fatalf("Retentions array is not sorted") 69 | } 70 | } 71 | 72 | func setUpCreate() (path string, fileExists func(string) bool, archiveList Retentions, tearDown func()) { 73 | path = "/tmp/whisper-testing.wsp" 74 | os.Remove(path) 75 | fileExists = func(path string) bool { 76 | fi, _ := os.Lstat(path) 77 | return fi != nil 78 | } 79 | archiveList = Retentions{{1, 300}, {60, 30}, {300, 12}} 80 | tearDown = func() { 81 | os.Remove(path) 82 | } 83 | return path, fileExists, archiveList, tearDown 84 | } 85 | 86 | func TestCreateCreatesFile(t *testing.T) { 87 | path, fileExists, retentions, tearDown := setUpCreate() 88 | expected := []byte{ 89 | // Metadata 90 | 0x00, 0x00, 0x00, 0x01, // Aggregation type 91 | 0x00, 0x00, 0x0e, 0x10, // Max retention 92 | 0x3f, 0x00, 0x00, 0x00, // xFilesFactor 93 | 0x00, 0x00, 0x00, 0x03, // Retention count 94 | // Archive Info 95 | // Retention 1 (1, 300) 96 | 0x00, 0x00, 0x00, 0x34, // offset 97 | 0x00, 0x00, 0x00, 0x01, // secondsPerPoint 98 | 0x00, 0x00, 0x01, 0x2c, // numberOfPoints 99 | // Retention 2 (60, 30) 100 | 0x00, 0x00, 0x0e, 0x44, // offset 101 | 0x00, 0x00, 0x00, 0x3c, // secondsPerPoint 102 | 0x00, 0x00, 0x00, 0x1e, // numberOfPoints 103 | // Retention 3 (300, 12) 104 | 0x00, 0x00, 0x0f, 0xac, // offset 105 | 0x00, 0x00, 0x01, 0x2c, // secondsPerPoint 106 | 0x00, 0x00, 0x00, 0x0c} // numberOfPoints 107 | whisper, err := Create(path, retentions, Average, 0.5) 108 | if err != nil { 109 | t.Fatalf("Failed to create whisper file: %v", err) 110 | } 111 | if whisper.aggregationMethod != Average { 112 | t.Fatalf("Unexpected aggregationMethod %v, expected %v", whisper.aggregationMethod, Average) 113 | } 114 | if whisper.maxRetention != 3600 { 115 | t.Fatalf("Unexpected maxRetention %v, expected 3600", whisper.maxRetention) 116 | } 117 | if whisper.xFilesFactor != 0.5 { 118 | t.Fatalf("Unexpected xFilesFactor %v, expected 0.5", whisper.xFilesFactor) 119 | } 120 | if len(whisper.archives) != 3 { 121 | t.Fatalf("Unexpected archive count %v, expected 3", len(whisper.archives)) 122 | } 123 | whisper.Close() 124 | if !fileExists(path) { 125 | t.Fatalf("File does not exist after create") 126 | } 127 | file, err := os.Open(path) 128 | if err != nil { 129 | t.Fatalf("Failed to open whisper file") 130 | } 131 | contents := make([]byte, len(expected)) 132 | file.Read(contents) 133 | 134 | for i := 0; i < len(contents); i++ { 135 | if expected[i] != contents[i] { 136 | // Show what is being written 137 | // for i := 0; i < 13; i++ { 138 | // for j := 0; j < 4; j ++ { 139 | // fmt.Printf(" %02x", contents[(i*4)+j]) 140 | // } 141 | // fmt.Print("\n") 142 | // } 143 | t.Fatalf("File is incorrect at character %v, expected %x got %x", i, expected[i], contents[i]) 144 | } 145 | } 146 | 147 | // test size 148 | info, err := os.Stat(path) 149 | if info.Size() != 4156 { 150 | t.Fatalf("File size is incorrect, expected %v got %v", 4156, info.Size()) 151 | } 152 | tearDown() 153 | } 154 | 155 | func TestCreateFileAlreadyExists(t *testing.T) { 156 | path, _, retentions, tearDown := setUpCreate() 157 | os.Create(path) 158 | wsp, err := Create(path, retentions, Average, 0.5) 159 | if err == nil { 160 | wsp.Close() 161 | t.Fatalf("Existing file should cause create to fail.") 162 | } 163 | tearDown() 164 | } 165 | 166 | func TestCreateFileInvalidRetentionDefs(t *testing.T) { 167 | path, _, retentions, tearDown := setUpCreate() 168 | // Add a small retention def on the end 169 | retentions = append(retentions, &Retention{1, 200}) 170 | wsp, err := Create(path, retentions, Average, 0.5) 171 | if err == nil { 172 | wsp.Close() 173 | t.Fatalf("Invalid retention definitions should cause create to fail.") 174 | } 175 | tearDown() 176 | } 177 | 178 | // XXX: This test violates locking constrains -- disabling 179 | // 2015/06/11 -- jjneely 180 | func testOpenFile(t *testing.T) { 181 | path, _, retentions, tearDown := setUpCreate() 182 | whisper1, err := Create(path, retentions, Average, 0.5) 183 | if err != nil { 184 | fmt.Errorf("Failed to create: %v", err) 185 | } 186 | 187 | // write some points 188 | now := int(time.Now().Unix()) 189 | for i := 0; i < 2; i++ { 190 | whisper1.Update(100, now-(i*1)) 191 | } 192 | 193 | whisper2, err := Open(path) 194 | if err != nil { 195 | t.Fatalf("Failed to open whisper file: %v", err) 196 | } 197 | if whisper1.aggregationMethod != whisper2.aggregationMethod { 198 | t.Fatalf("aggregationMethod did not match, expected %v, received %v", whisper1.aggregationMethod, whisper2.aggregationMethod) 199 | } 200 | if whisper1.maxRetention != whisper2.maxRetention { 201 | t.Fatalf("maxRetention did not match, expected %v, received %v", whisper1.maxRetention, whisper2.maxRetention) 202 | } 203 | if whisper1.xFilesFactor != whisper2.xFilesFactor { 204 | t.Fatalf("xFilesFactor did not match, expected %v, received %v", whisper1.xFilesFactor, whisper2.xFilesFactor) 205 | } 206 | if len(whisper1.archives) != len(whisper2.archives) { 207 | t.Fatalf("archive count does not match, expected %v, received %v", len(whisper1.archives), len(whisper2.archives)) 208 | } 209 | for i := range whisper1.archives { 210 | if whisper1.archives[i].offset != whisper2.archives[i].offset { 211 | t.Fatalf("archive mismatch offset at %v [%v, %v]", i, whisper1.archives[i].offset, whisper2.archives[i].offset) 212 | } 213 | if whisper1.archives[i].Retention.secondsPerPoint != whisper2.archives[i].Retention.secondsPerPoint { 214 | t.Fatalf("Retention.secondsPerPoint mismatch offset at %v [%v, %v]", i, whisper1.archives[i].Retention.secondsPerPoint, whisper2.archives[i].Retention.secondsPerPoint) 215 | } 216 | if whisper1.archives[i].Retention.numberOfPoints != whisper2.archives[i].Retention.numberOfPoints { 217 | t.Fatalf("Retention.numberOfPoints mismatch offset at %v [%v, %v]", i, whisper1.archives[i].Retention.numberOfPoints, whisper2.archives[i].Retention.numberOfPoints) 218 | } 219 | 220 | } 221 | 222 | result1, err := whisper1.Fetch(now-3, now) 223 | if err != nil { 224 | t.Fatalf("Error retrieving result from created whisper") 225 | } 226 | result2, err := whisper2.Fetch(now-3, now) 227 | if err != nil { 228 | t.Fatalf("Error retrieving result from opened whisper") 229 | } 230 | 231 | if result1.String() != result2.String() { 232 | t.Fatalf("Results do not match") 233 | } 234 | 235 | tearDown() 236 | } 237 | 238 | // TestReadOnlyFile tests that we can open and read from a read only file 239 | // successfully. It also confirms that we cannot write to it. 240 | func TestReadOnlyFile(t *testing.T) { 241 | path, _, retentions, tearDown := setUpCreate() 242 | wsp, err := Create(path, retentions, Average, 0.5) 243 | if err != nil { 244 | fmt.Errorf("Failed to create: %v", err) 245 | } 246 | 247 | // write some points 248 | now := int(time.Now().Unix()) 249 | for i := 0; i < 2; i++ { 250 | wsp.Update(100, now-(i*1)) 251 | } 252 | knownResults, err := wsp.Fetch(now-3, now) 253 | if err != nil { 254 | t.Fatalf("Unable to fetch from whisper file: %s", err.Error()) 255 | } 256 | wsp.Close() 257 | 258 | // Change permissions to 0400 read only for the creating user 259 | os.Chmod(path, 0400) 260 | 261 | wsp, err = Open(path) 262 | if err != nil { 263 | t.Fatalf("Unable to open read only file: %s", path) 264 | } 265 | 266 | results, err := wsp.Fetch(now-3, now) 267 | if err != nil { 268 | t.Fatalf("Unable to fetch from read only file: %s", path) 269 | } 270 | if results.String() != knownResults.String() { 271 | t.Fatalf("Fetch results from read only file do not match") 272 | } 273 | 274 | err = wsp.Update(100, now-4) 275 | if err == nil { 276 | // Yes, we are looking for success here 277 | t.Fatalf("Able to write to read only file: %s", path) 278 | } else { 279 | t.Logf("Write to read only file returned expected error: %s", err.Error()) 280 | } 281 | wsp.Close() 282 | 283 | // Revert permissions 284 | os.Chmod(path, 0600) 285 | tearDown() 286 | } 287 | 288 | /* 289 | Test the full cycle of creating a whisper file, adding some 290 | data points to it and then fetching a time series. 291 | */ 292 | func testCreateUpdateFetch(t *testing.T, aggregationMethod AggregationMethod, xFilesFactor float32, secondsAgo, fromAgo, fetchLength, step int, currentValue, increment float64) *TimeSeries { 293 | var whisper *Whisper 294 | var err error 295 | path, _, archiveList, tearDown := setUpCreate() 296 | whisper, err = Create(path, archiveList, aggregationMethod, xFilesFactor) 297 | if err != nil { 298 | t.Fatalf("Failed create: %v", err) 299 | } 300 | defer whisper.Close() 301 | oldestTime := whisper.StartTime() 302 | now := int(time.Now().Unix()) 303 | 304 | if (now - whisper.maxRetention) != oldestTime { 305 | t.Fatalf("Invalid whisper start time, expected %v, received %v", oldestTime, now-whisper.maxRetention) 306 | } 307 | 308 | for i := 0; i < secondsAgo; i++ { 309 | err = whisper.Update(currentValue, now-secondsAgo+i) 310 | if err != nil { 311 | t.Fatalf("Unexpected error for %v: %v", i, err) 312 | } 313 | currentValue += increment 314 | } 315 | 316 | fromTime := now - fromAgo 317 | untilTime := fromTime + fetchLength 318 | 319 | timeSeries, err := whisper.Fetch(fromTime, untilTime) 320 | if err != nil { 321 | t.Fatalf("Unexpected error: %v", err) 322 | } 323 | if !validTimestamp(timeSeries.fromTime, fromTime, step) { 324 | t.Fatalf("Invalid fromTime [%v/%v], expected %v, received %v", secondsAgo, fromAgo, fromTime, timeSeries.fromTime) 325 | } 326 | if !validTimestamp(timeSeries.untilTime, untilTime, step) { 327 | t.Fatalf("Invalid untilTime [%v/%v], expected %v, received %v", secondsAgo, fromAgo, untilTime, timeSeries.untilTime) 328 | } 329 | if timeSeries.step != step { 330 | t.Fatalf("Invalid step [%v/%v], expected %v, received %v", secondsAgo, fromAgo, step, timeSeries.step) 331 | } 332 | tearDown() 333 | 334 | return timeSeries 335 | } 336 | 337 | func validTimestamp(value, stamp, step int) bool { 338 | return value == nearestStep(stamp, step) || value == nearestStep(stamp, step)+step 339 | } 340 | func nearestStep(stamp, step int) int { 341 | return stamp - (stamp % step) + step 342 | } 343 | 344 | func assertFloatAlmostEqual(t *testing.T, received, expected, slop float64) { 345 | if math.Abs(expected-received) > slop { 346 | t.Fatalf("Expected %v to be within %v of %v", expected, slop, received) 347 | } 348 | } 349 | 350 | func assertFloatEqual(t *testing.T, received, expected float64) { 351 | if math.Abs(expected-received) > 0.00001 { 352 | t.Fatalf("Expected %v, received %v", expected, received) 353 | } 354 | } 355 | 356 | func TestFetchEmptyTimeseries(t *testing.T) { 357 | path, _, archiveList, tearDown := setUpCreate() 358 | whisper, err := Create(path, archiveList, Sum, 0.5) 359 | if err != nil { 360 | t.Fatalf("Failed create: %v", err) 361 | } 362 | defer whisper.Close() 363 | 364 | now := int(time.Now().Unix()) 365 | result, err := whisper.Fetch(now-3, now) 366 | for _, point := range result.Points() { 367 | if !math.IsNaN(point.Value) { 368 | t.Fatalf("Expecting NaN values got '%v'", point.Value) 369 | } 370 | } 371 | 372 | tearDown() 373 | } 374 | 375 | func TestCreateUpdateFetch(t *testing.T) { 376 | var timeSeries *TimeSeries 377 | timeSeries = testCreateUpdateFetch(t, Average, 0.5, 3500, 3500, 1000, 300, 0.5, 0.2) 378 | assertFloatAlmostEqual(t, timeSeries.values[1], 150.1, 58.0) 379 | assertFloatAlmostEqual(t, timeSeries.values[2], 210.75, 28.95) 380 | 381 | timeSeries = testCreateUpdateFetch(t, Sum, 0.5, 600, 600, 500, 60, 0.5, 0.2) 382 | assertFloatAlmostEqual(t, timeSeries.values[0], 18.35, 5.95) 383 | assertFloatAlmostEqual(t, timeSeries.values[1], 30.35, 5.95) 384 | // 4 is a crazy one because it fluctuates between 60 and ~4k 385 | assertFloatAlmostEqual(t, timeSeries.values[5], 4356.05, 500.0) 386 | 387 | timeSeries = testCreateUpdateFetch(t, Last, 0.5, 300, 300, 200, 1, 0.5, 0.2) 388 | assertFloatAlmostEqual(t, timeSeries.values[0], 0.7, 0.001) 389 | assertFloatAlmostEqual(t, timeSeries.values[10], 2.7, 0.001) 390 | assertFloatAlmostEqual(t, timeSeries.values[20], 4.7, 0.001) 391 | 392 | } 393 | 394 | func BenchmarkCreateUpdateFetch(b *testing.B) { 395 | path, _, archiveList, tearDown := setUpCreate() 396 | var err error 397 | var whisper *Whisper 398 | var secondsAgo, now, fromTime, untilTime int 399 | var currentValue, increment float64 400 | for i := 0; i < b.N; i++ { 401 | whisper, err = Create(path, archiveList, Average, 0.5) 402 | if err != nil { 403 | b.Fatalf("Failed create %v", err) 404 | } 405 | 406 | secondsAgo = 3500 407 | currentValue = 0.5 408 | increment = 0.2 409 | now = int(time.Now().Unix()) 410 | 411 | for i := 0; i < secondsAgo; i++ { 412 | err = whisper.Update(currentValue, now-secondsAgo+i) 413 | if err != nil { 414 | b.Fatalf("Unexpected error for %v: %v", i, err) 415 | } 416 | currentValue += increment 417 | } 418 | 419 | fromTime = now - secondsAgo 420 | untilTime = fromTime + 1000 421 | 422 | whisper.Fetch(fromTime, untilTime) 423 | whisper.Close() 424 | tearDown() 425 | } 426 | } 427 | 428 | func BenchmarkFairCreateUpdateFetch(b *testing.B) { 429 | path, _, archiveList, tearDown := setUpCreate() 430 | var err error 431 | var whisper *Whisper 432 | var secondsAgo, now, fromTime, untilTime int 433 | var currentValue, increment float64 434 | for i := 0; i < b.N; i++ { 435 | whisper, err = Create(path, archiveList, Average, 0.5) 436 | if err != nil { 437 | b.Fatalf("Failed create %v", err) 438 | } 439 | whisper.Close() 440 | 441 | secondsAgo = 3500 442 | currentValue = 0.5 443 | increment = 0.2 444 | now = int(time.Now().Unix()) 445 | 446 | for i := 0; i < secondsAgo; i++ { 447 | whisper, err = Open(path) 448 | err = whisper.Update(currentValue, now-secondsAgo+i) 449 | if err != nil { 450 | b.Fatalf("Unexpected error for %v: %v", i, err) 451 | } 452 | currentValue += increment 453 | whisper.Close() 454 | } 455 | 456 | fromTime = now - secondsAgo 457 | untilTime = fromTime + 1000 458 | 459 | whisper, err = Open(path) 460 | whisper.Fetch(fromTime, untilTime) 461 | whisper.Close() 462 | tearDown() 463 | } 464 | } 465 | 466 | func testCreateUpdateManyFetch(t *testing.T, aggregationMethod AggregationMethod, xFilesFactor float32, points []*TimeSeriesPoint, fromAgo, fetchLength int) *TimeSeries { 467 | var whisper *Whisper 468 | var err error 469 | path, _, archiveList, tearDown := setUpCreate() 470 | whisper, err = Create(path, archiveList, aggregationMethod, xFilesFactor) 471 | if err != nil { 472 | t.Fatalf("Failed create: %v", err) 473 | } 474 | defer whisper.Close() 475 | now := int(time.Now().Unix()) 476 | 477 | whisper.UpdateMany(points) 478 | 479 | fromTime := now - fromAgo 480 | untilTime := fromTime + fetchLength 481 | 482 | timeSeries, err := whisper.Fetch(fromTime, untilTime) 483 | if err != nil { 484 | t.Fatalf("Unexpected error: %v", err) 485 | } 486 | tearDown() 487 | 488 | return timeSeries 489 | } 490 | 491 | func makeGoodPoints(count, step int, value func(int) float64) []*TimeSeriesPoint { 492 | points := make([]*TimeSeriesPoint, count) 493 | now := int(time.Now().Unix()) 494 | for i := 0; i < count; i++ { 495 | points[i] = &TimeSeriesPoint{now - (i * step), value(i)} 496 | } 497 | return points 498 | } 499 | 500 | func makeBadPoints(count, minAge int) []*TimeSeriesPoint { 501 | points := make([]*TimeSeriesPoint, count) 502 | now := int(time.Now().Unix()) 503 | for i := 0; i < count; i++ { 504 | points[i] = &TimeSeriesPoint{now - (minAge + i), 123.456} 505 | } 506 | return points 507 | } 508 | 509 | func printPoints(points []*TimeSeriesPoint) { 510 | fmt.Print("[") 511 | for i, point := range points { 512 | if i > 0 { 513 | fmt.Print(", ") 514 | } 515 | fmt.Printf("%v", point) 516 | } 517 | fmt.Println("]") 518 | } 519 | 520 | func TestCreateUpdateManyFetch(t *testing.T) { 521 | var timeSeries *TimeSeries 522 | 523 | points := makeGoodPoints(1000, 2, func(i int) float64 { return float64(i) }) 524 | points = append(points, points[len(points)-1]) 525 | timeSeries = testCreateUpdateManyFetch(t, Sum, 0.5, points, 1000, 800) 526 | 527 | // fmt.Println(timeSeries) 528 | 529 | assertFloatAlmostEqual(t, timeSeries.values[0], 455, 15) 530 | 531 | // all the ones 532 | points = makeGoodPoints(10000, 1, func(_ int) float64 { return 1 }) 533 | timeSeries = testCreateUpdateManyFetch(t, Sum, 0.5, points, 10000, 10000) 534 | for i := 0; i < 6; i++ { 535 | assertFloatEqual(t, timeSeries.values[i], 1) 536 | } 537 | for i := 6; i < 10; i++ { 538 | assertFloatEqual(t, timeSeries.values[i], 5) 539 | } 540 | } 541 | 542 | // should not panic if all points are out of range 543 | func TestCreateUpdateManyOnly_old_points(t *testing.T) { 544 | points := makeBadPoints(1, 10000) 545 | 546 | path, _, archiveList, tearDown := setUpCreate() 547 | whisper, err := Create(path, archiveList, Sum, 0.5) 548 | if err != nil { 549 | t.Fatalf("Failed create: %v", err) 550 | } 551 | defer whisper.Close() 552 | 553 | whisper.UpdateMany(points) 554 | 555 | tearDown() 556 | } 557 | 558 | func Test_extractPoints(t *testing.T) { 559 | points := makeGoodPoints(100, 1, func(i int) float64 { return float64(i) }) 560 | now := int(time.Now().Unix()) 561 | currentPoints, remainingPoints := extractPoints(points, now, 50) 562 | if length := len(currentPoints); length != 50 { 563 | t.Fatalf("First: %v", length) 564 | } 565 | if length := len(remainingPoints); length != 50 { 566 | t.Fatalf("Second: %v", length) 567 | } 568 | } 569 | 570 | // extractPoints should return empty slices if the first point is out of range 571 | func Test_extractPoints_only_old_points(t *testing.T) { 572 | now := int(time.Now().Unix()) 573 | points := makeBadPoints(1, 100) 574 | 575 | currentPoints, remainingPoints := extractPoints(points, now, 50) 576 | if length := len(currentPoints); length != 0 { 577 | t.Fatalf("First: %v", length) 578 | } 579 | if length := len(remainingPoints); length != 1 { 580 | t.Fatalf("Second2: %v", length) 581 | } 582 | } 583 | 584 | func test_aggregate(t *testing.T, method AggregationMethod, expected float64) { 585 | received := aggregate(method, []float64{1.0, 2.0, 3.0, 5.0, 4.0}) 586 | if expected != received { 587 | t.Fatalf("Expected %v, received %v", expected, received) 588 | } 589 | } 590 | func Test_aggregateAverage(t *testing.T) { 591 | test_aggregate(t, Average, 3.0) 592 | } 593 | 594 | func Test_aggregateSum(t *testing.T) { 595 | test_aggregate(t, Sum, 15.0) 596 | } 597 | 598 | func Test_aggregateLast(t *testing.T) { 599 | test_aggregate(t, Last, 4.0) 600 | } 601 | 602 | func Test_aggregateMax(t *testing.T) { 603 | test_aggregate(t, Max, 5.0) 604 | } 605 | 606 | func Test_aggregateMin(t *testing.T) { 607 | test_aggregate(t, Min, 1.0) 608 | } 609 | 610 | func TestDataPointBytes(t *testing.T) { 611 | point := dataPoint{1234, 567.891} 612 | b := []byte{0, 0, 4, 210, 64, 129, 191, 32, 196, 155, 165, 227} 613 | checkBytes(t, b, point.Bytes()) 614 | } 615 | 616 | func TestTimeSeriesPoints(t *testing.T) { 617 | ts := TimeSeries{fromTime: 1348003785, untilTime: 1348003795, step: 1, values: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}} 618 | points := ts.Points() 619 | if length := len(points); length != 10 { 620 | t.Fatalf("Unexpected number of points in time series, %v", length) 621 | } 622 | } 623 | 624 | func TestUpdateManyWithManyRetentions(t *testing.T) { 625 | path, _, archiveList, tearDown := setUpCreate() 626 | lastArchive := archiveList[len(archiveList)-1] 627 | 628 | valueMin := 41 629 | valueMax := 43 630 | 631 | whisper, err := Create(path, archiveList, Average, 0.5) 632 | if err != nil { 633 | t.Fatalf("Failed create: %v", err) 634 | } 635 | 636 | points := make([]*TimeSeriesPoint, 1) 637 | 638 | now := int(time.Now().Unix()) 639 | for i := 0; i < lastArchive.secondsPerPoint*2; i++ { 640 | points[0] = &TimeSeriesPoint{ 641 | Time: now - i, 642 | Value: float64(valueMin*(i%2) + valueMax*((i+1)%2)), // valueMin, valueMax, valueMin... 643 | } 644 | whisper.UpdateMany(points) 645 | } 646 | 647 | whisper.Close() 648 | 649 | // check data in last archive 650 | whisper, err = Open(path) 651 | if err != nil { 652 | t.Fatalf("Failed open: %v", err) 653 | } 654 | 655 | result, err := whisper.Fetch(now-lastArchive.numberOfPoints*lastArchive.secondsPerPoint, now) 656 | if err != nil { 657 | t.Fatalf("Failed fetch: %v", err) 658 | } 659 | 660 | foundValues := 0 661 | for i := 0; i < len(result.values); i++ { 662 | if !math.IsNaN(result.values[i]) { 663 | if result.values[i] >= float64(valueMin) && 664 | result.values[i] <= float64(valueMax) { 665 | foundValues++ 666 | } 667 | } 668 | } 669 | if foundValues < 2 { 670 | t.Fatalf("Not found values in archive %#v", lastArchive) 671 | } 672 | 673 | whisper.Close() 674 | 675 | tearDown() 676 | } 677 | -------------------------------------------------------------------------------- /whisper_test.py: -------------------------------------------------------------------------------- 1 | import whisper 2 | import os 3 | import time 4 | 5 | def set_up_create(): 6 | path = "/tmp/whisper-testing.wsp" 7 | try: 8 | os.remove(path) 9 | except: 10 | pass 11 | archive_list = [[1,300], [60,30], [300,12]] 12 | def tear_down(): 13 | os.remove(path) 14 | 15 | return path, archive_list, tear_down 16 | 17 | def benchmark_create_update_fetch(): 18 | path, archive_list, tear_down = set_up_create() 19 | # start timer 20 | start_time = time.clock() 21 | for i in range(100): 22 | whisper.create(path, archive_list) 23 | 24 | seconds_ago = 3500 25 | current_value = 0.5 26 | increment = 0.2 27 | now = time.time() 28 | # file_update closes the file so we have to reopen every time 29 | for i in range(seconds_ago): 30 | whisper.update(path, current_value, now - seconds_ago + i) 31 | current_value += increment 32 | 33 | from_time = now - seconds_ago 34 | until_time = from_time + 1000 35 | 36 | whisper.fetch(path, from_time, until_time) 37 | tear_down() 38 | 39 | # end timer 40 | end_time = time.clock() 41 | elapsed_time = end_time - start_time 42 | 43 | print "Executed 100 iterations in %ss (%i ns/op)" % (elapsed_time, (elapsed_time * 1000 * 1000 * 1000) / 100) 44 | 45 | if __name__ == "__main__": 46 | benchmark_create_update_fetch() 47 | --------------------------------------------------------------------------------