├── VERSION ├── .gitignore ├── eval ├── eval-results.png ├── helper.go └── main.go ├── .travis.yml ├── LICENSE ├── fuzz.go ├── README.md ├── bstream_test.go ├── testdata └── data.go ├── bstream.go ├── Makefile ├── tsz_test.go └── tsz.go /VERSION: -------------------------------------------------------------------------------- 1 | 1.0.0 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | eval/eval 2 | /target 3 | *.test 4 | -------------------------------------------------------------------------------- /eval/eval-results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dgryski/go-tsz/HEAD/eval/eval-results.png -------------------------------------------------------------------------------- /eval/helper.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "math" 5 | ) 6 | 7 | func Round(f float64) float64 { 8 | if f < 0 { 9 | return math.Ceil(f - 0.5) 10 | } 11 | return math.Floor(f + .5) 12 | } 13 | 14 | func RoundNum(f float64, places int) float64 { 15 | shift := math.Pow(10, float64(places)) 16 | return Round(f*shift) / shift 17 | } 18 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | sudo: false 4 | 5 | branches: 6 | except: 7 | - release 8 | 9 | branches: 10 | only: 11 | - master 12 | - develop 13 | - travis 14 | 15 | go: 16 | - 1.9 17 | - tip 18 | 19 | matrix: 20 | allow_failures: 21 | - go: tip 22 | 23 | before_install: 24 | - if [ -n "$GH_USER" ]; then git config --global github.user ${GH_USER}; fi; 25 | - if [ -n "$GH_TOKEN" ]; then git config --global github.token ${GH_TOKEN}; fi; 26 | - go get github.com/mattn/goveralls 27 | 28 | before_script: 29 | - make deps 30 | 31 | script: 32 | - make qa 33 | 34 | after_failure: 35 | - cat ./target/test/report.xml 36 | 37 | after_success: 38 | - if [ "$TRAVIS_GO_VERSION" = "1.9" ]; then $HOME/gopath/bin/goveralls -covermode=count -coverprofile=target/report/coverage.out -service=travis-ci; fi; 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015,2016 Damian Gryski 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 15 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 16 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 18 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 20 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 21 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 22 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 23 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | -------------------------------------------------------------------------------- /fuzz.go: -------------------------------------------------------------------------------- 1 | // +build gofuzz 2 | 3 | package tsz 4 | 5 | import ( 6 | "encoding/binary" 7 | "fmt" 8 | "math" 9 | 10 | "github.com/dgryski/go-tsz/testdata" 11 | ) 12 | 13 | func Fuzz(data []byte) int { 14 | 15 | fuzzUnpack(data) 16 | 17 | if len(data) < 9 { 18 | return 0 19 | } 20 | 21 | t0 := uint32(1456236677) 22 | 23 | v := float64(10000) 24 | 25 | var vals []testdata.Point 26 | s := New(t0) 27 | t := t0 28 | for len(data) >= 10 { 29 | tdelta := uint32(binary.LittleEndian.Uint16(data)) 30 | if t == t0 { 31 | tdelta &= (1 << 14) - 1 32 | } 33 | t += tdelta 34 | data = data[2:] 35 | v += float64(int16(binary.LittleEndian.Uint16(data))) + float64(binary.LittleEndian.Uint16(data[2:]))/float64(math.MaxUint16) 36 | data = data[8:] 37 | vals = append(vals, testdata.Point{V: v, T: t}) 38 | s.Push(t, v) 39 | } 40 | 41 | it := s.Iter() 42 | 43 | var i int 44 | for it.Next() { 45 | gt, gv := it.Values() 46 | if gt != vals[i].T || (gv != vals[i].V || math.IsNaN(gv) && math.IsNaN(vals[i].V)) { 47 | panic(fmt.Sprintf("failure: gt=%v vals[i].T=%v gv=%v vals[i].V=%v", gt, vals[i].T, gv, vals[i].V)) 48 | } 49 | i++ 50 | } 51 | 52 | if i != len(vals) { 53 | panic("extra data") 54 | } 55 | 56 | return 1 57 | } 58 | 59 | func fuzzUnpack(data []byte) { 60 | 61 | it, err := NewIterator(data) 62 | if err != nil { 63 | return 64 | } 65 | 66 | for it.Next() { 67 | _, _ = it.Values() 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-tsz 2 | 3 | * Package tsz implement time-series compression http://www.vldb.org/pvldb/vol8/p1816-teller.pdf in Go* 4 | 5 | [![Master Branch](https://img.shields.io/badge/branch-master-lightgray.svg)](https://github.com/dgryski/go-tsz/tree/master) 6 | [![Master Build Status](https://secure.travis-ci.org/dgryski/go-tsz.svg?branch=master)](https://travis-ci.org/dgryski/go-tsz?branch=master) 7 | [![Master Coverage Status](https://coveralls.io/repos/dgryski/go-tsz/badge.svg?branch=master&service=github)](https://coveralls.io/github/dgryski/go-tsz?branch=master) 8 | [![Go Report Card](https://goreportcard.com/badge/github.com/dgryski/go-tsz)](https://goreportcard.com/report/github.com/dgryski/go-tsz) 9 | [![GoDoc](https://godoc.org/github.com/dgryski/go-tsz?status.svg)](http://godoc.org/github.com/dgryski/go-tsz) 10 | 11 | ## Description 12 | 13 | Package tsz implement the Gorilla Time Series Databasetime-series compression as described in: 14 | http://www.vldb.org/pvldb/vol8/p1816-teller.pdf 15 | 16 | 17 | ## Getting started 18 | 19 | This application is written in Go language, please refer to the guides in https://golang.org for getting started. 20 | 21 | This project include a Makefile that allows you to test and build the project with simple commands. 22 | To see all available options: 23 | ```bash 24 | make help 25 | ``` 26 | 27 | ## Running all tests 28 | 29 | Before committing the code, please check if it passes all tests using 30 | ```bash 31 | make qa 32 | ``` 33 | -------------------------------------------------------------------------------- /bstream_test.go: -------------------------------------------------------------------------------- 1 | package tsz 2 | 3 | import ( 4 | "io" 5 | "testing" 6 | ) 7 | 8 | func TestNewBWriter(t *testing.T) { 9 | b := newBWriter(1) 10 | if b.count != 0 { 11 | t.Errorf("Unexpected value: %v\n", b.count) 12 | } 13 | } 14 | 15 | func TestReadBitEOF1(t *testing.T) { 16 | b := newBWriter(1) 17 | _, err := b.readBit() 18 | if err != io.EOF { 19 | t.Errorf("Unexpected value: %v\n", err) 20 | } 21 | } 22 | 23 | func TestReadBitEOF2(t *testing.T) { 24 | b := newBReader([]byte{1}) 25 | b.count = 0 26 | _, err := b.readBit() 27 | if err != io.EOF { 28 | t.Errorf("Unexpected value: %v\n", err) 29 | } 30 | } 31 | 32 | func TestReadByteEOF1(t *testing.T) { 33 | b := newBWriter(1) 34 | _, err := b.readByte() 35 | if err != io.EOF { 36 | t.Errorf("Unexpected value: %v\n", err) 37 | } 38 | } 39 | 40 | func TestReadByteEOF2(t *testing.T) { 41 | b := newBReader([]byte{1}) 42 | b.count = 0 43 | _, err := b.readByte() 44 | if err != io.EOF { 45 | t.Errorf("Unexpected value: %v\n", err) 46 | } 47 | } 48 | 49 | func TestReadByteEOF3(t *testing.T) { 50 | b := newBReader([]byte{1}) 51 | b.count = 16 52 | _, err := b.readByte() 53 | if err != io.EOF { 54 | t.Errorf("Unexpected value: %v\n", err) 55 | } 56 | } 57 | 58 | func TestReadBitsEOF(t *testing.T) { 59 | b := newBReader([]byte{1}) 60 | _, err := b.readBits(9) 61 | if err != io.EOF { 62 | t.Errorf("Unexpected value: %v\n", err) 63 | } 64 | } 65 | 66 | func TestUnmarshalBinaryErr(t *testing.T) { 67 | b := &bstream{} 68 | err := b.UnmarshalBinary([]byte{}) 69 | if err == nil { 70 | t.Errorf("An error was expected\n") 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /testdata/data.go: -------------------------------------------------------------------------------- 1 | package testdata 2 | 3 | type Point struct { 4 | V float64 5 | T uint32 6 | } 7 | 8 | // 120 points every 60s 9 | var TwoHoursData = []Point{ 10 | {761, 1440583200}, {727, 1440583260}, {765, 1440583320}, {706, 1440583380}, {700, 1440583440}, 11 | {679, 1440583500}, {757, 1440583560}, {708, 1440583620}, {739, 1440583680}, {707, 1440583740}, 12 | {699, 1440583800}, {740, 1440583860}, {729, 1440583920}, {766, 1440583980}, {730, 1440584040}, 13 | {715, 1440584100}, {705, 1440584160}, {693, 1440584220}, {765, 1440584280}, {724, 1440584340}, 14 | {799, 1440584400}, {761, 1440584460}, {737, 1440584520}, {766, 1440584580}, {756, 1440584640}, 15 | {719, 1440584700}, {722, 1440584760}, {801, 1440584820}, {747, 1440584880}, {731, 1440584940}, 16 | {742, 1440585000}, {744, 1440585060}, {791, 1440585120}, {750, 1440585180}, {759, 1440585240}, 17 | {809, 1440585300}, {751, 1440585360}, {705, 1440585420}, {770, 1440585480}, {792, 1440585540}, 18 | {727, 1440585600}, {762, 1440585660}, {772, 1440585720}, {721, 1440585780}, {748, 1440585840}, 19 | {753, 1440585900}, {744, 1440585960}, {716, 1440586020}, {776, 1440586080}, {659, 1440586140}, 20 | {789, 1440586200}, {766, 1440586260}, {758, 1440586320}, {690, 1440586380}, {795, 1440586440}, 21 | {770, 1440586500}, {758, 1440586560}, {723, 1440586620}, {767, 1440586680}, {765, 1440586740}, 22 | {693, 1440586800}, {706, 1440586860}, {681, 1440586920}, {727, 1440586980}, {724, 1440587040}, 23 | {780, 1440587100}, {678, 1440587160}, {696, 1440587220}, {758, 1440587280}, {740, 1440587340}, 24 | {735, 1440587400}, {700, 1440587460}, {742, 1440587520}, {747, 1440587580}, {752, 1440587640}, 25 | {734, 1440587700}, {743, 1440587760}, {732, 1440587820}, {746, 1440587880}, {770, 1440587940}, 26 | {780, 1440588000}, {710, 1440588060}, {731, 1440588120}, {712, 1440588180}, {712, 1440588240}, 27 | {741, 1440588300}, {770, 1440588360}, {770, 1440588420}, {754, 1440588480}, {718, 1440588540}, 28 | {670, 1440588600}, {775, 1440588660}, {749, 1440588720}, {795, 1440588780}, {756, 1440588840}, 29 | {741, 1440588900}, {787, 1440588960}, {721, 1440589020}, {745, 1440589080}, {782, 1440589140}, 30 | {765, 1440589200}, {780, 1440589260}, {811, 1440589320}, {790, 1440589380}, {836, 1440589440}, 31 | {743, 1440589500}, {858, 1440589560}, {739, 1440589620}, {762, 1440589680}, {770, 1440589740}, 32 | {752, 1440589800}, {763, 1440589860}, {795, 1440589920}, {792, 1440589980}, {746, 1440590040}, 33 | {786, 1440590100}, {785, 1440590160}, {774, 1440590220}, {786, 1440590280}, {718, 1440590340}, 34 | } 35 | -------------------------------------------------------------------------------- /bstream.go: -------------------------------------------------------------------------------- 1 | package tsz 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "io" 7 | ) 8 | 9 | // bstream is a stream of bits 10 | type bstream struct { 11 | // the data stream 12 | stream []byte 13 | 14 | // how many bits are valid in current byte 15 | count uint8 16 | } 17 | 18 | func newBReader(b []byte) *bstream { 19 | return &bstream{stream: b, count: 8} 20 | } 21 | 22 | func newBWriter(size int) *bstream { 23 | return &bstream{stream: make([]byte, 0, size), count: 0} 24 | } 25 | 26 | func (b *bstream) clone() *bstream { 27 | d := make([]byte, len(b.stream)) 28 | copy(d, b.stream) 29 | return &bstream{stream: d, count: b.count} 30 | } 31 | 32 | func (b *bstream) bytes() []byte { 33 | return b.stream 34 | } 35 | 36 | type bit bool 37 | 38 | const ( 39 | zero bit = false 40 | one bit = true 41 | ) 42 | 43 | func (b *bstream) writeBit(bit bit) { 44 | 45 | if b.count == 0 { 46 | b.stream = append(b.stream, 0) 47 | b.count = 8 48 | } 49 | 50 | i := len(b.stream) - 1 51 | 52 | if bit { 53 | b.stream[i] |= 1 << (b.count - 1) 54 | } 55 | 56 | b.count-- 57 | } 58 | 59 | func (b *bstream) writeByte(byt byte) { 60 | 61 | if b.count == 0 { 62 | b.stream = append(b.stream, 0) 63 | b.count = 8 64 | } 65 | 66 | i := len(b.stream) - 1 67 | 68 | // fill up b.b with b.count bits from byt 69 | b.stream[i] |= byt >> (8 - b.count) 70 | 71 | b.stream = append(b.stream, 0) 72 | i++ 73 | b.stream[i] = byt << b.count 74 | } 75 | 76 | func (b *bstream) writeBits(u uint64, nbits int) { 77 | u <<= (64 - uint(nbits)) 78 | for nbits >= 8 { 79 | byt := byte(u >> 56) 80 | b.writeByte(byt) 81 | u <<= 8 82 | nbits -= 8 83 | } 84 | 85 | for nbits > 0 { 86 | b.writeBit((u >> 63) == 1) 87 | u <<= 1 88 | nbits-- 89 | } 90 | } 91 | 92 | func (b *bstream) readBit() (bit, error) { 93 | 94 | if len(b.stream) == 0 { 95 | return false, io.EOF 96 | } 97 | 98 | if b.count == 0 { 99 | b.stream = b.stream[1:] 100 | // did we just run out of stuff to read? 101 | if len(b.stream) == 0 { 102 | return false, io.EOF 103 | } 104 | b.count = 8 105 | } 106 | 107 | b.count-- 108 | d := b.stream[0] & 0x80 109 | b.stream[0] <<= 1 110 | return d != 0, nil 111 | } 112 | 113 | func (b *bstream) readByte() (byte, error) { 114 | 115 | if len(b.stream) == 0 { 116 | return 0, io.EOF 117 | } 118 | 119 | if b.count == 0 { 120 | b.stream = b.stream[1:] 121 | 122 | if len(b.stream) == 0 { 123 | return 0, io.EOF 124 | } 125 | 126 | b.count = 8 127 | } 128 | 129 | if b.count == 8 { 130 | b.count = 0 131 | return b.stream[0], nil 132 | } 133 | 134 | byt := b.stream[0] 135 | b.stream = b.stream[1:] 136 | 137 | if len(b.stream) == 0 { 138 | return 0, io.EOF 139 | } 140 | 141 | byt |= b.stream[0] >> b.count 142 | b.stream[0] <<= (8 - b.count) 143 | 144 | return byt, nil 145 | } 146 | 147 | func (b *bstream) readBits(nbits int) (uint64, error) { 148 | 149 | var u uint64 150 | 151 | for nbits >= 8 { 152 | byt, err := b.readByte() 153 | if err != nil { 154 | return 0, err 155 | } 156 | 157 | u = (u << 8) | uint64(byt) 158 | nbits -= 8 159 | } 160 | 161 | if nbits == 0 { 162 | return u, nil 163 | } 164 | 165 | if nbits > int(b.count) { 166 | u = (u << uint(b.count)) | uint64(b.stream[0]>>(8-b.count)) 167 | nbits -= int(b.count) 168 | b.stream = b.stream[1:] 169 | 170 | if len(b.stream) == 0 { 171 | return 0, io.EOF 172 | } 173 | b.count = 8 174 | } 175 | 176 | u = (u << uint(nbits)) | uint64(b.stream[0]>>(8-uint(nbits))) 177 | b.stream[0] <<= uint(nbits) 178 | b.count -= uint8(nbits) 179 | return u, nil 180 | } 181 | 182 | // MarshalBinary implements the encoding.BinaryMarshaler interface 183 | func (b *bstream) MarshalBinary() ([]byte, error) { 184 | buf := new(bytes.Buffer) 185 | err := binary.Write(buf, binary.BigEndian, b.count) 186 | if err != nil { 187 | return nil, err 188 | } 189 | err = binary.Write(buf, binary.BigEndian, b.stream) 190 | if err != nil { 191 | return nil, err 192 | } 193 | return buf.Bytes(), nil 194 | } 195 | 196 | // UnmarshalBinary implements the encoding.BinaryUnmarshaler interface 197 | func (b *bstream) UnmarshalBinary(bIn []byte) error { 198 | buf := bytes.NewReader(bIn) 199 | err := binary.Read(buf, binary.BigEndian, &b.count) 200 | if err != nil { 201 | return err 202 | } 203 | b.stream = make([]byte, buf.Len()) 204 | return binary.Read(buf, binary.BigEndian, &b.stream) 205 | } 206 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # MAKEFILE 2 | # 3 | # @author Nicola Asuni 4 | # @link https://github.com/dgryski/go-tsz 5 | # 6 | # This file is intended to be executed in a Linux-compatible system. 7 | # It also assumes that the project has been cloned in the right path under GOPATH: 8 | # $GOPATH/src/github.com/dgryski/go-tsz 9 | # 10 | # ------------------------------------------------------------------------------ 11 | 12 | # List special make targets that are not associated with files 13 | .PHONY: help all test format fmtcheck vet lint coverage cyclo ineffassign misspell structcheck varcheck errcheck gosimple astscan qa deps clean nuke 14 | 15 | # Use bash as shell (Note: Ubuntu now uses dash which doesn't support PIPESTATUS). 16 | SHELL=/bin/bash 17 | 18 | # CVS path (path to the parent dir containing the project) 19 | CVSPATH=github.com/dgryski 20 | 21 | # Project owner 22 | OWNER=dgryski 23 | 24 | # Project vendor 25 | VENDOR=dgryski 26 | 27 | # Project name 28 | PROJECT=go-tsz 29 | 30 | # Project version 31 | VERSION=$(shell cat VERSION) 32 | 33 | # Name of RPM or DEB package 34 | PKGNAME=${VENDOR}-${PROJECT} 35 | 36 | # Current directory 37 | CURRENTDIR=$(shell pwd) 38 | 39 | # GO lang path 40 | ifneq ($(GOPATH),) 41 | ifeq ($(findstring $(GOPATH),$(CURRENTDIR)),) 42 | # the defined GOPATH is not valid 43 | GOPATH= 44 | endif 45 | endif 46 | ifeq ($(GOPATH),) 47 | # extract the GOPATH 48 | GOPATH=$(firstword $(subst /src/, ,$(CURRENTDIR))) 49 | endif 50 | 51 | # --- MAKE TARGETS --- 52 | 53 | # Display general help about this command 54 | help: 55 | @echo "" 56 | @echo "$(PROJECT) Makefile." 57 | @echo "GOPATH=$(GOPATH)" 58 | @echo "The following commands are available:" 59 | @echo "" 60 | @echo " make qa : Run all the tests" 61 | @echo " make test : Run the unit tests" 62 | @echo "" 63 | @echo " make format : Format the source code" 64 | @echo " make fmtcheck : Check if the source code has been formatted" 65 | @echo " make vet : Check for suspicious constructs" 66 | @echo " make lint : Check for style errors" 67 | @echo " make coverage : Generate the coverage report" 68 | @echo " make cyclo : Generate the cyclomatic complexity report" 69 | @echo " make ineffassign : Detect ineffectual assignments" 70 | @echo " make misspell : Detect commonly misspelled words in source files" 71 | @echo " make structcheck : Find unused struct fields" 72 | @echo " make varcheck : Find unused global variables and constants" 73 | @echo " make errcheck : Check that error return values are used" 74 | @echo " make gosimple : Suggest code simplifications" 75 | @echo " make astscan : GO AST scanner" 76 | @echo "" 77 | @echo " make docs : Generate source code documentation" 78 | @echo "" 79 | @echo " make deps : Get the dependencies" 80 | @echo " make clean : Remove any build artifact" 81 | @echo " make nuke : Deletes any intermediate file" 82 | @echo "" 83 | 84 | 85 | # Alias for help target 86 | all: help 87 | 88 | # Run the unit tests 89 | test: 90 | @mkdir -p target/test 91 | @mkdir -p target/report 92 | GOPATH=$(GOPATH) \ 93 | go test \ 94 | -covermode=atomic \ 95 | -bench=. \ 96 | -race \ 97 | -cpuprofile=target/report/cpu.out \ 98 | -memprofile=target/report/mem.out \ 99 | -mutexprofile=target/report/mutex.out \ 100 | -coverprofile=target/report/coverage.out \ 101 | -v . | \ 102 | tee >(PATH=$(GOPATH)/bin:$(PATH) go-junit-report > target/test/report.xml); \ 103 | test $${PIPESTATUS[0]} -eq 0 104 | 105 | # Format the source code 106 | format: 107 | @find . -type f -name "*.go" -exec gofmt -s -w {} \; 108 | 109 | # Check if the source code has been formatted 110 | fmtcheck: 111 | @mkdir -p target 112 | @find . -type f -name "*.go" -exec gofmt -s -d {} \; | tee target/format.diff 113 | @test ! -s target/format.diff || { echo "ERROR: the source code has not been formatted - please use 'make format' or 'gofmt'"; exit 1; } 114 | 115 | # Check for syntax errors 116 | vet: 117 | GOPATH=$(GOPATH) go vet . 118 | 119 | # Check for style errors 120 | lint: 121 | GOPATH=$(GOPATH) PATH=$(GOPATH)/bin:$(PATH) golint . 122 | 123 | # Generate the coverage report 124 | coverage: 125 | @mkdir -p target/report 126 | GOPATH=$(GOPATH) \ 127 | go tool cover -html=target/report/coverage.out -o target/report/coverage.html 128 | 129 | # Report cyclomatic complexity 130 | cyclo: 131 | @mkdir -p target/report 132 | GOPATH=$(GOPATH) gocyclo -avg ./ | tee target/report/cyclo.txt ; test $${PIPESTATUS[0]} -eq 0 133 | 134 | # Detect ineffectual assignments 135 | ineffassign: 136 | @mkdir -p target/report 137 | GOPATH=$(GOPATH) ineffassign ./ | tee target/report/ineffassign.txt ; test $${PIPESTATUS[0]} -eq 0 138 | 139 | # Detect commonly misspelled words in source files 140 | misspell: 141 | @mkdir -p target/report 142 | GOPATH=$(GOPATH) misspell -error ./ | tee target/report/misspell.txt ; test $${PIPESTATUS[0]} -eq 0 143 | 144 | # Find unused struct fields 145 | structcheck: 146 | @mkdir -p target/report 147 | GOPATH=$(GOPATH) structcheck -a ./ | tee target/report/structcheck.txt 148 | 149 | # Find unused global variables and constants 150 | varcheck: 151 | @mkdir -p target/report 152 | GOPATH=$(GOPATH) varcheck -e ./ | tee target/report/varcheck.txt 153 | 154 | # Check that error return values are used 155 | errcheck: 156 | @mkdir -p target/report 157 | GOPATH=$(GOPATH) errcheck ./ | tee target/report/errcheck.txt 158 | 159 | # Suggest code simplifications 160 | gosimple: 161 | @mkdir -p target/report 162 | GOPATH=$(GOPATH) gosimple ./ | tee target/report/gosimple.txt 163 | 164 | # AST scanner 165 | astscan: 166 | @mkdir -p target/report 167 | GOPATH=$(GOPATH) gas .//*.go | tee target/report/astscan.txt 168 | 169 | # Generate source docs 170 | docs: 171 | @mkdir -p target/docs 172 | nohup sh -c 'GOPATH=$(GOPATH) godoc -http=127.0.0.1:6060' > target/godoc_server.log 2>&1 & 173 | wget --directory-prefix=target/docs/ --execute robots=off --retry-connrefused --recursive --no-parent --adjust-extension --page-requisites --convert-links http://127.0.0.1:6060/pkg/github.com/${VENDOR}/${PROJECT}/ ; kill -9 `lsof -ti :6060` 174 | @echo ''${PKGNAME}' Documentation ...' > target/docs/index.html 175 | 176 | # Alias to run all quality-assurance checks 177 | qa: fmtcheck test vet lint coverage cyclo ineffassign misspell structcheck varcheck errcheck gosimple astscan 178 | 179 | # --- INSTALL --- 180 | 181 | # Get the dependencies 182 | deps: 183 | GOPATH=$(GOPATH) go get ./... 184 | GOPATH=$(GOPATH) go get github.com/golang/lint/golint 185 | GOPATH=$(GOPATH) go get github.com/jstemmer/go-junit-report 186 | GOPATH=$(GOPATH) go get github.com/axw/gocov/gocov 187 | GOPATH=$(GOPATH) go get github.com/fzipp/gocyclo 188 | GOPATH=$(GOPATH) go get github.com/gordonklaus/ineffassign 189 | GOPATH=$(GOPATH) go get github.com/client9/misspell/cmd/misspell 190 | GOPATH=$(GOPATH) go get github.com/opennota/check/cmd/structcheck 191 | GOPATH=$(GOPATH) go get github.com/opennota/check/cmd/varcheck 192 | GOPATH=$(GOPATH) go get github.com/kisielk/errcheck 193 | GOPATH=$(GOPATH) go get honnef.co/go/tools/cmd/gosimple 194 | GOPATH=$(GOPATH) go get github.com/GoASTScanner/gas 195 | 196 | # Remove any build artifact 197 | clean: 198 | GOPATH=$(GOPATH) go clean ./... 199 | 200 | # Deletes any intermediate file 201 | nuke: 202 | rm -rf ./target 203 | GOPATH=$(GOPATH) go clean -i ./... 204 | -------------------------------------------------------------------------------- /tsz_test.go: -------------------------------------------------------------------------------- 1 | package tsz 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/dgryski/go-tsz/testdata" 8 | ) 9 | 10 | func TestMarshalBinary(t *testing.T) { 11 | s1 := New(testdata.TwoHoursData[0].T) 12 | for _, p := range testdata.TwoHoursData { 13 | s1.Push(p.T, p.V) 14 | } 15 | it1 := s1.Iter() 16 | it1.Next() 17 | b, err := s1.MarshalBinary() 18 | if err != nil { 19 | t.Error(err) 20 | } 21 | s2 := New(s1.T0) 22 | err = s2.UnmarshalBinary(b) 23 | if err != nil { 24 | t.Error(err) 25 | } 26 | it := s2.Iter() 27 | for _, w := range testdata.TwoHoursData { 28 | if !it.Next() { 29 | t.Fatalf("Next()=false, want true") 30 | } 31 | tt, vv := it.Values() 32 | // t.Logf("it.Values()=(%+v, %+v)\n", time.Unix(int64(tt), 0), vv) 33 | if w.T != tt || w.V != vv { 34 | t.Errorf("Values()=(%v,%v), want (%v,%v)\n", tt, vv, w.T, w.V) 35 | } 36 | } 37 | } 38 | 39 | func BenchmarkMarshalBinary(b *testing.B) { 40 | var err error 41 | b.StopTimer() 42 | s1 := New(testdata.TwoHoursData[0].T) 43 | for _, p := range testdata.TwoHoursData { 44 | s1.Push(p.T, p.V) 45 | } 46 | s1.Finish() 47 | b.ReportAllocs() 48 | b.StartTimer() 49 | for i := 0; i < b.N; i++ { 50 | _, err = s1.MarshalBinary() 51 | } 52 | if err != nil { 53 | b.Errorf("Unexpected error: %v\n", err) 54 | } 55 | } 56 | 57 | func BenchmarkUnmarshalBinary(b *testing.B) { 58 | var err error 59 | b.StopTimer() 60 | s1 := New(testdata.TwoHoursData[0].T) 61 | for _, p := range testdata.TwoHoursData { 62 | s1.Push(p.T, p.V) 63 | } 64 | s1.Finish() 65 | buf, err := s1.MarshalBinary() 66 | if err != nil { 67 | b.Error(err) 68 | } 69 | b.ReportAllocs() 70 | s2 := New(s1.T0) 71 | b.StartTimer() 72 | for i := 0; i < b.N; i++ { 73 | err = s2.UnmarshalBinary(buf) 74 | } 75 | if err != nil { 76 | b.Errorf("Unexpected error: %v\n", err) 77 | } 78 | } 79 | 80 | func TestExampleEncoding(t *testing.T) { 81 | 82 | // Example from the paper 83 | t0, _ := time.ParseInLocation("Jan _2 2006 15:04:05", "Mar 24 2015 02:00:00", time.Local) 84 | tunix := uint32(t0.Unix()) 85 | 86 | s := New(tunix) 87 | 88 | tunix += 62 89 | s.Push(tunix, 12) 90 | 91 | tunix += 60 92 | s.Push(tunix, 12) 93 | 94 | tunix += 60 95 | s.Push(tunix, 24) 96 | 97 | // extra tests 98 | 99 | // floating point masking/shifting bug 100 | tunix += 60 101 | s.Push(tunix, 13) 102 | 103 | tunix += 60 104 | s.Push(tunix, 24) 105 | 106 | // delta-of-delta sizes 107 | tunix += 300 // == delta-of-delta of 240 108 | s.Push(tunix, 24) 109 | 110 | tunix += 900 // == delta-of-delta of 600 111 | s.Push(tunix, 24) 112 | 113 | tunix += 900 + 2050 // == delta-of-delta of 600 114 | s.Push(tunix, 24) 115 | 116 | it := s.Iter() 117 | 118 | tunix = uint32(t0.Unix()) 119 | want := []struct { 120 | t uint32 121 | v float64 122 | }{ 123 | {tunix + 62, 12}, 124 | {tunix + 122, 12}, 125 | {tunix + 182, 24}, 126 | 127 | {tunix + 242, 13}, 128 | {tunix + 302, 24}, 129 | 130 | {tunix + 602, 24}, 131 | {tunix + 1502, 24}, 132 | {tunix + 4452, 24}, 133 | } 134 | 135 | for _, w := range want { 136 | if !it.Next() { 137 | t.Fatalf("Next()=false, want true") 138 | } 139 | tt, vv := it.Values() 140 | if w.t != tt || w.v != vv { 141 | t.Errorf("Values()=(%v,%v), want (%v,%v)\n", tt, vv, w.t, w.v) 142 | } 143 | } 144 | 145 | if it.Next() { 146 | t.Fatalf("Next()=true, want false") 147 | } 148 | 149 | if err := it.Err(); err != nil { 150 | t.Errorf("it.Err()=%v, want nil", err) 151 | } 152 | } 153 | 154 | func TestRoundtrip(t *testing.T) { 155 | 156 | s := New(testdata.TwoHoursData[0].T) 157 | for _, p := range testdata.TwoHoursData { 158 | s.Push(p.T, p.V) 159 | } 160 | 161 | it := s.Iter() 162 | for _, w := range testdata.TwoHoursData { 163 | if !it.Next() { 164 | t.Fatalf("Next()=false, want true") 165 | } 166 | tt, vv := it.Values() 167 | // t.Logf("it.Values()=(%+v, %+v)\n", time.Unix(int64(tt), 0), vv) 168 | if w.T != tt || w.V != vv { 169 | t.Errorf("Values()=(%v,%v), want (%v,%v)\n", tt, vv, w.T, w.V) 170 | } 171 | } 172 | 173 | if it.Next() { 174 | t.Fatalf("Next()=true, want false") 175 | } 176 | 177 | if err := it.Err(); err != nil { 178 | t.Errorf("it.Err()=%v, want nil", err) 179 | } 180 | } 181 | 182 | func TestConcurrentRoundtripImmediateWrites(t *testing.T) { 183 | testConcurrentRoundtrip(t, time.Duration(0)) 184 | } 185 | func TestConcurrentRoundtrip1MsBetweenWrites(t *testing.T) { 186 | testConcurrentRoundtrip(t, time.Millisecond) 187 | } 188 | func TestConcurrentRoundtrip10MsBetweenWrites(t *testing.T) { 189 | testConcurrentRoundtrip(t, 10*time.Millisecond) 190 | } 191 | 192 | // Test reading while writing at the same time. 193 | func testConcurrentRoundtrip(t *testing.T, sleep time.Duration) { 194 | s := New(testdata.TwoHoursData[0].T) 195 | 196 | //notify the reader about the number of points that have been written. 197 | writeNotify := make(chan int) 198 | 199 | // notify the reader when we have finished. 200 | done := make(chan struct{}) 201 | 202 | // continuously iterate over the values of the series. 203 | // when a write is made, the total number of points in the series 204 | // will be sent over the channel, so we can make sure we are reading 205 | // the correct amount of values. 206 | go func(numPoints chan int, finished chan struct{}) { 207 | written := 0 208 | for { 209 | select { 210 | case written = <-numPoints: 211 | default: 212 | read := 0 213 | it := s.Iter() 214 | // read all of the points in the series. 215 | for it.Next() { 216 | tt, vv := it.Values() 217 | expectedT := testdata.TwoHoursData[read].T 218 | expectedV := testdata.TwoHoursData[read].V 219 | if expectedT != tt || expectedV != vv { 220 | t.Errorf("metric values dont match what was written. (%d, %f) != (%d, %f)\n", tt, vv, expectedT, expectedV) 221 | } 222 | read++ 223 | } 224 | //check that the number of points read matches the number of points 225 | // written to the series. 226 | if read != written && read != written+1 { 227 | // check if a point was written while we were running 228 | select { 229 | case written = <-numPoints: 230 | // a new point was written. 231 | if read != written && read != written+1 { 232 | t.Errorf("expexcted %d values in series, got %d", written, read) 233 | } 234 | default: 235 | t.Errorf("expexcted %d values in series, got %d", written, read) 236 | } 237 | } 238 | } 239 | //check if we have finished writing points. 240 | select { 241 | case <-finished: 242 | return 243 | default: 244 | } 245 | } 246 | }(writeNotify, done) 247 | 248 | // write points to the series. 249 | for i := 0; i < 100; i++ { 250 | s.Push(testdata.TwoHoursData[i].T, testdata.TwoHoursData[i].V) 251 | writeNotify <- i + 1 252 | time.Sleep(sleep) 253 | } 254 | done <- struct{}{} 255 | } 256 | 257 | func BenchmarkEncode(b *testing.B) { 258 | b.SetBytes(int64(len(testdata.TwoHoursData) * 12)) 259 | for i := 0; i < b.N; i++ { 260 | s := New(testdata.TwoHoursData[0].T) 261 | for _, tt := range testdata.TwoHoursData { 262 | s.Push(tt.T, tt.V) 263 | } 264 | } 265 | } 266 | 267 | func BenchmarkDecodeSeries(b *testing.B) { 268 | b.SetBytes(int64(len(testdata.TwoHoursData) * 12)) 269 | s := New(testdata.TwoHoursData[0].T) 270 | for _, tt := range testdata.TwoHoursData { 271 | s.Push(tt.T, tt.V) 272 | } 273 | 274 | b.ResetTimer() 275 | 276 | for i := 0; i < b.N; i++ { 277 | it := s.Iter() 278 | var j int 279 | for it.Next() { 280 | j++ 281 | } 282 | } 283 | } 284 | 285 | func BenchmarkDecodeByteSlice(b *testing.B) { 286 | b.SetBytes(int64(len(testdata.TwoHoursData) * 12)) 287 | s := New(testdata.TwoHoursData[0].T) 288 | for _, tt := range testdata.TwoHoursData { 289 | s.Push(tt.T, tt.V) 290 | } 291 | 292 | s.Finish() 293 | bytes := s.Bytes() 294 | buf := make([]byte, len(bytes)) 295 | b.ResetTimer() 296 | 297 | for i := 0; i < b.N; i++ { 298 | copy(buf, bytes) 299 | it, _ := NewIterator(buf) 300 | var j int 301 | for it.Next() { 302 | j++ 303 | } 304 | } 305 | } 306 | 307 | func TestEncodeSimilarFloats(t *testing.T) { 308 | tunix := uint32(time.Unix(0, 0).Unix()) 309 | s := New(tunix) 310 | want := []struct { 311 | t uint32 312 | v float64 313 | }{ 314 | {tunix, 6.00065e+06}, 315 | {tunix + 1, 6.000656e+06}, 316 | {tunix + 2, 6.000657e+06}, 317 | {tunix + 3, 6.000659e+06}, 318 | {tunix + 4, 6.000661e+06}, 319 | } 320 | 321 | for _, v := range want { 322 | s.Push(v.t, v.v) 323 | } 324 | 325 | s.Finish() 326 | 327 | it := s.Iter() 328 | 329 | for _, w := range want { 330 | if !it.Next() { 331 | t.Fatalf("Next()=false, want true") 332 | } 333 | tt, vv := it.Values() 334 | if w.t != tt || w.v != vv { 335 | t.Errorf("Values()=(%v,%v), want (%v,%v)\n", tt, vv, w.v, w.v) 336 | } 337 | } 338 | 339 | if it.Next() { 340 | t.Fatalf("Next()=true, want false") 341 | } 342 | 343 | if err := it.Err(); err != nil { 344 | t.Errorf("it.Err()=%v, want nil", err) 345 | } 346 | } 347 | 348 | func TestBstreamIteratorError(t *testing.T) { 349 | b := newBReader([]byte("")) 350 | _, err := bstreamIterator(b) 351 | if err == nil { 352 | t.Errorf("An error was expected") 353 | } 354 | } 355 | -------------------------------------------------------------------------------- /tsz.go: -------------------------------------------------------------------------------- 1 | // Package tsz implement time-series compression 2 | /* 3 | 4 | http://www.vldb.org/pvldb/vol8/p1816-teller.pdf 5 | 6 | */ 7 | package tsz 8 | 9 | import ( 10 | "bytes" 11 | "encoding/binary" 12 | "io" 13 | "math" 14 | "math/bits" 15 | "sync" 16 | ) 17 | 18 | // Series is the basic series primitive 19 | // you can concurrently put values, finish the stream, and create iterators 20 | type Series struct { 21 | sync.Mutex 22 | 23 | // TODO(dgryski): timestamps in the paper are uint64 24 | T0 uint32 25 | t uint32 26 | val float64 27 | 28 | bw bstream 29 | leading uint8 30 | trailing uint8 31 | finished bool 32 | 33 | tDelta uint32 34 | } 35 | 36 | // New series 37 | func New(t0 uint32) *Series { 38 | s := Series{ 39 | T0: t0, 40 | leading: ^uint8(0), 41 | } 42 | 43 | // block header 44 | s.bw.writeBits(uint64(t0), 32) 45 | 46 | return &s 47 | 48 | } 49 | 50 | // Bytes value of the series stream 51 | func (s *Series) Bytes() []byte { 52 | s.Lock() 53 | defer s.Unlock() 54 | return s.bw.bytes() 55 | } 56 | 57 | func finish(w *bstream) { 58 | // write an end-of-stream record 59 | w.writeBits(0x0f, 4) 60 | w.writeBits(0xffffffff, 32) 61 | w.writeBit(zero) 62 | } 63 | 64 | // Finish the series by writing an end-of-stream record 65 | func (s *Series) Finish() { 66 | s.Lock() 67 | if !s.finished { 68 | finish(&s.bw) 69 | s.finished = true 70 | } 71 | s.Unlock() 72 | } 73 | 74 | // Push a timestamp and value to the series 75 | func (s *Series) Push(t uint32, v float64) { 76 | s.Lock() 77 | defer s.Unlock() 78 | 79 | if s.t == 0 { 80 | // first point 81 | s.t = t 82 | s.val = v 83 | s.tDelta = t - s.T0 84 | s.bw.writeBits(uint64(s.tDelta), 14) 85 | s.bw.writeBits(math.Float64bits(v), 64) 86 | return 87 | } 88 | 89 | tDelta := t - s.t 90 | dod := int32(tDelta - s.tDelta) 91 | 92 | switch { 93 | case dod == 0: 94 | s.bw.writeBit(zero) 95 | case -63 <= dod && dod <= 64: 96 | s.bw.writeBits(0x02, 2) // '10' 97 | s.bw.writeBits(uint64(dod), 7) 98 | case -255 <= dod && dod <= 256: 99 | s.bw.writeBits(0x06, 3) // '110' 100 | s.bw.writeBits(uint64(dod), 9) 101 | case -2047 <= dod && dod <= 2048: 102 | s.bw.writeBits(0x0e, 4) // '1110' 103 | s.bw.writeBits(uint64(dod), 12) 104 | default: 105 | s.bw.writeBits(0x0f, 4) // '1111' 106 | s.bw.writeBits(uint64(dod), 32) 107 | } 108 | 109 | vDelta := math.Float64bits(v) ^ math.Float64bits(s.val) 110 | 111 | if vDelta == 0 { 112 | s.bw.writeBit(zero) 113 | } else { 114 | s.bw.writeBit(one) 115 | 116 | leading := uint8(bits.LeadingZeros64(vDelta)) 117 | trailing := uint8(bits.TrailingZeros64(vDelta)) 118 | 119 | // clamp number of leading zeros to avoid overflow when encoding 120 | if leading >= 32 { 121 | leading = 31 122 | } 123 | 124 | // TODO(dgryski): check if it's 'cheaper' to reset the leading/trailing bits instead 125 | if s.leading != ^uint8(0) && leading >= s.leading && trailing >= s.trailing { 126 | s.bw.writeBit(zero) 127 | s.bw.writeBits(vDelta>>s.trailing, 64-int(s.leading)-int(s.trailing)) 128 | } else { 129 | s.leading, s.trailing = leading, trailing 130 | 131 | s.bw.writeBit(one) 132 | s.bw.writeBits(uint64(leading), 5) 133 | 134 | // Note that if leading == trailing == 0, then sigbits == 64. But that value doesn't actually fit into the 6 bits we have. 135 | // Luckily, we never need to encode 0 significant bits, since that would put us in the other case (vdelta == 0). 136 | // So instead we write out a 0 and adjust it back to 64 on unpacking. 137 | sigbits := 64 - leading - trailing 138 | s.bw.writeBits(uint64(sigbits), 6) 139 | s.bw.writeBits(vDelta>>trailing, int(sigbits)) 140 | } 141 | } 142 | 143 | s.tDelta = tDelta 144 | s.t = t 145 | s.val = v 146 | 147 | } 148 | 149 | // Iter lets you iterate over a series. It is not concurrency-safe. 150 | func (s *Series) Iter() *Iter { 151 | s.Lock() 152 | w := s.bw.clone() 153 | s.Unlock() 154 | 155 | finish(w) 156 | iter, _ := bstreamIterator(w) 157 | return iter 158 | } 159 | 160 | // Iter lets you iterate over a series. It is not concurrency-safe. 161 | type Iter struct { 162 | T0 uint32 163 | 164 | t uint32 165 | val float64 166 | 167 | br bstream 168 | leading uint8 169 | trailing uint8 170 | 171 | finished bool 172 | 173 | tDelta uint32 174 | err error 175 | } 176 | 177 | func bstreamIterator(br *bstream) (*Iter, error) { 178 | 179 | br.count = 8 180 | 181 | t0, err := br.readBits(32) 182 | if err != nil { 183 | return nil, err 184 | } 185 | 186 | return &Iter{ 187 | T0: uint32(t0), 188 | br: *br, 189 | }, nil 190 | } 191 | 192 | // NewIterator for the series 193 | func NewIterator(b []byte) (*Iter, error) { 194 | return bstreamIterator(newBReader(b)) 195 | } 196 | 197 | // Next iteration of the series iterator 198 | func (it *Iter) Next() bool { 199 | 200 | if it.err != nil || it.finished { 201 | return false 202 | } 203 | 204 | if it.t == 0 { 205 | // read first t and v 206 | tDelta, err := it.br.readBits(14) 207 | if err != nil { 208 | it.err = err 209 | return false 210 | } 211 | it.tDelta = uint32(tDelta) 212 | it.t = it.T0 + it.tDelta 213 | v, err := it.br.readBits(64) 214 | if err != nil { 215 | it.err = err 216 | return false 217 | } 218 | 219 | it.val = math.Float64frombits(v) 220 | 221 | return true 222 | } 223 | 224 | // read delta-of-delta 225 | var d byte 226 | for i := 0; i < 4; i++ { 227 | d <<= 1 228 | bit, err := it.br.readBit() 229 | if err != nil { 230 | it.err = err 231 | return false 232 | } 233 | if bit == zero { 234 | break 235 | } 236 | d |= 1 237 | } 238 | 239 | var dod int32 240 | var sz uint 241 | switch d { 242 | case 0x00: 243 | // dod == 0 244 | case 0x02: 245 | sz = 7 246 | case 0x06: 247 | sz = 9 248 | case 0x0e: 249 | sz = 12 250 | case 0x0f: 251 | bits, err := it.br.readBits(32) 252 | if err != nil { 253 | it.err = err 254 | return false 255 | } 256 | 257 | // end of stream 258 | if bits == 0xffffffff { 259 | it.finished = true 260 | return false 261 | } 262 | 263 | dod = int32(bits) 264 | } 265 | 266 | if sz != 0 { 267 | bits, err := it.br.readBits(int(sz)) 268 | if err != nil { 269 | it.err = err 270 | return false 271 | } 272 | if bits > (1 << (sz - 1)) { 273 | // or something 274 | bits = bits - (1 << sz) 275 | } 276 | dod = int32(bits) 277 | } 278 | 279 | tDelta := it.tDelta + uint32(dod) 280 | 281 | it.tDelta = tDelta 282 | it.t = it.t + it.tDelta 283 | 284 | // read compressed value 285 | bit, err := it.br.readBit() 286 | if err != nil { 287 | it.err = err 288 | return false 289 | } 290 | 291 | if bit == zero { 292 | // it.val = it.val 293 | } else { 294 | bit, itErr := it.br.readBit() 295 | if itErr != nil { 296 | it.err = err 297 | return false 298 | } 299 | if bit == zero { 300 | // reuse leading/trailing zero bits 301 | // it.leading, it.trailing = it.leading, it.trailing 302 | } else { 303 | bits, err := it.br.readBits(5) 304 | if err != nil { 305 | it.err = err 306 | return false 307 | } 308 | it.leading = uint8(bits) 309 | 310 | bits, err = it.br.readBits(6) 311 | if err != nil { 312 | it.err = err 313 | return false 314 | } 315 | mbits := uint8(bits) 316 | // 0 significant bits here means we overflowed and we actually need 64; see comment in encoder 317 | if mbits == 0 { 318 | mbits = 64 319 | } 320 | it.trailing = 64 - it.leading - mbits 321 | } 322 | 323 | mbits := int(64 - it.leading - it.trailing) 324 | bits, err := it.br.readBits(mbits) 325 | if err != nil { 326 | it.err = err 327 | return false 328 | } 329 | vbits := math.Float64bits(it.val) 330 | vbits ^= (bits << it.trailing) 331 | it.val = math.Float64frombits(vbits) 332 | } 333 | 334 | return true 335 | } 336 | 337 | // Values at the current iterator position 338 | func (it *Iter) Values() (uint32, float64) { 339 | return it.t, it.val 340 | } 341 | 342 | // Err error at the current iterator position 343 | func (it *Iter) Err() error { 344 | return it.err 345 | } 346 | 347 | type errMarshal struct { 348 | w io.Writer 349 | r io.Reader 350 | err error 351 | } 352 | 353 | func (em *errMarshal) write(t interface{}) { 354 | if em.err != nil { 355 | return 356 | } 357 | em.err = binary.Write(em.w, binary.BigEndian, t) 358 | } 359 | 360 | func (em *errMarshal) read(t interface{}) { 361 | if em.err != nil { 362 | return 363 | } 364 | em.err = binary.Read(em.r, binary.BigEndian, t) 365 | } 366 | 367 | // MarshalBinary implements the encoding.BinaryMarshaler interface 368 | func (s *Series) MarshalBinary() ([]byte, error) { 369 | buf := new(bytes.Buffer) 370 | em := &errMarshal{w: buf} 371 | em.write(s.T0) 372 | em.write(s.leading) 373 | em.write(s.t) 374 | em.write(s.tDelta) 375 | em.write(s.trailing) 376 | em.write(s.val) 377 | bStream, err := s.bw.MarshalBinary() 378 | if err != nil { 379 | return nil, err 380 | } 381 | em.write(bStream) 382 | if em.err != nil { 383 | return nil, em.err 384 | } 385 | return buf.Bytes(), nil 386 | } 387 | 388 | // UnmarshalBinary implements the encoding.BinaryUnmarshaler interface 389 | func (s *Series) UnmarshalBinary(b []byte) error { 390 | buf := bytes.NewReader(b) 391 | em := &errMarshal{r: buf} 392 | em.read(&s.T0) 393 | em.read(&s.leading) 394 | em.read(&s.t) 395 | em.read(&s.tDelta) 396 | em.read(&s.trailing) 397 | em.read(&s.val) 398 | outBuf := make([]byte, buf.Len()) 399 | em.read(outBuf) 400 | err := s.bw.UnmarshalBinary(outBuf) 401 | if err != nil { 402 | return err 403 | } 404 | if em.err != nil { 405 | return em.err 406 | } 407 | return nil 408 | } 409 | -------------------------------------------------------------------------------- /eval/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/dgryski/go-tsz" 6 | "github.com/dgryski/go-tsz/testdata" 7 | "math" 8 | "math/rand" 9 | "os" 10 | "text/tabwriter" 11 | ) 12 | 13 | // collection of 24h worth of minutely points, with different characteristics. 14 | var ConstantZero = make([]testdata.Point, 60*24) 15 | var ConstantOne = make([]testdata.Point, 60*24) 16 | var ConstantPos3f = make([]testdata.Point, 60*24) 17 | var ConstantNeg3f = make([]testdata.Point, 60*24) 18 | var ConstantPos0f = make([]testdata.Point, 60*24) 19 | var ConstantNeg0f = make([]testdata.Point, 60*24) 20 | var ConstantNearMaxf = make([]testdata.Point, 60*24) 21 | var ConstantNearMinf = make([]testdata.Point, 60*24) 22 | var ConstantNearMax0f = make([]testdata.Point, 60*24) 23 | var ConstantNearMin0f = make([]testdata.Point, 60*24) 24 | var Batch100ZeroOne = make([]testdata.Point, 60*24) 25 | var FlappingZeroOne = make([]testdata.Point, 60*24) 26 | 27 | var RandomTinyPosf = make([]testdata.Point, 60*24) 28 | var RandomTinyf = make([]testdata.Point, 60*24) 29 | var RandomTinyPos2f = make([]testdata.Point, 60*24) 30 | var RandomTiny2f = make([]testdata.Point, 60*24) 31 | var RandomTinyPos1f = make([]testdata.Point, 60*24) 32 | var RandomTiny1f = make([]testdata.Point, 60*24) 33 | var RandomTinyPos0f = make([]testdata.Point, 60*24) 34 | var RandomTiny0f = make([]testdata.Point, 60*24) 35 | 36 | var RandomSmallPosf = make([]testdata.Point, 60*24) 37 | var RandomSmallf = make([]testdata.Point, 60*24) 38 | var RandomSmallPos2f = make([]testdata.Point, 60*24) 39 | var RandomSmall2f = make([]testdata.Point, 60*24) 40 | var RandomSmallPos1f = make([]testdata.Point, 60*24) 41 | var RandomSmall1f = make([]testdata.Point, 60*24) 42 | var RandomSmallPos0f = make([]testdata.Point, 60*24) 43 | var RandomSmall0f = make([]testdata.Point, 60*24) 44 | 45 | var Random60kPosf = make([]testdata.Point, 60*24) 46 | var Random60kf = make([]testdata.Point, 60*24) 47 | var Random60kPos2f = make([]testdata.Point, 60*24) 48 | var Random60k2f = make([]testdata.Point, 60*24) 49 | var Random60kPos1f = make([]testdata.Point, 60*24) 50 | var Random60k1f = make([]testdata.Point, 60*24) 51 | var Random60kPos0f = make([]testdata.Point, 60*24) 52 | var Random60k0f = make([]testdata.Point, 60*24) 53 | 54 | var SmallTestDataPosf = make([]testdata.Point, 60*24) 55 | var SmallTestDataf = make([]testdata.Point, 60*24) 56 | var SmallTestDataPos0f = make([]testdata.Point, 60*24) 57 | var SmallTestData0f = make([]testdata.Point, 60*24) 58 | 59 | var RandomLargePosf = make([]testdata.Point, 60*24) 60 | var RandomLargef = make([]testdata.Point, 60*24) 61 | var RandomLargePos0f = make([]testdata.Point, 60*24) 62 | var RandomLarge0f = make([]testdata.Point, 60*24) 63 | var LargeTestDataPosf = make([]testdata.Point, 60*24) 64 | var LargeTestDataPos0f = make([]testdata.Point, 60*24) 65 | var LargeTestDataf = make([]testdata.Point, 60*24) 66 | var LargeTestData0f = make([]testdata.Point, 60*24) 67 | 68 | func main() { 69 | for i := 0; i < 60*24; i++ { 70 | ts := uint32(i * 60) 71 | ConstantZero[i] = testdata.Point{float64(0), ts} 72 | ConstantOne[i] = testdata.Point{float64(1), ts} 73 | ConstantPos3f[i] = testdata.Point{float64(1234.567), ts} 74 | ConstantNeg3f[i] = testdata.Point{float64(-1234.567), ts} 75 | ConstantPos0f[i] = testdata.Point{float64(1234), ts} 76 | ConstantNeg0f[i] = testdata.Point{float64(-1235), ts} 77 | ConstantNearMaxf[i] = testdata.Point{math.MaxFloat64 / 100, ts} 78 | ConstantNearMinf[i] = testdata.Point{-math.MaxFloat64 / 100, ts} 79 | ConstantNearMax0f[i] = testdata.Point{math.Floor(ConstantNearMaxf[i].V), ts} 80 | ConstantNearMin0f[i] = testdata.Point{math.Floor(ConstantNearMinf[i].V), ts} 81 | if i%200 < 100 { 82 | Batch100ZeroOne[i] = testdata.Point{float64(0), ts} 83 | } else { 84 | Batch100ZeroOne[i] = testdata.Point{float64(1), ts} 85 | } 86 | if i%2 == 0 { 87 | FlappingZeroOne[i] = testdata.Point{float64(0), ts} 88 | } else { 89 | FlappingZeroOne[i] = testdata.Point{float64(1), ts} 90 | } 91 | 92 | RandomTinyPosf[i] = testdata.Point{rand.ExpFloat64(), ts} // 0-inf, but most vals are very low, mostly between 0 and 2, rarely goes over 10 93 | RandomTinyf[i] = testdata.Point{rand.NormFloat64(), ts} // -inf to + inf, as many pos as neg, but similar as above, rarely goes under -10 or over +10 94 | RandomTinyPos2f[i] = testdata.Point{RoundNum(RandomTinyPosf[i].V, 2), ts} 95 | RandomTiny2f[i] = testdata.Point{RoundNum(RandomTinyf[i].V, 2), ts} 96 | RandomTinyPos1f[i] = testdata.Point{RoundNum(RandomTinyPosf[i].V, 1), ts} 97 | RandomTiny1f[i] = testdata.Point{RoundNum(RandomTinyf[i].V, 1), ts} 98 | RandomTinyPos0f[i] = testdata.Point{math.Floor(RandomTinyPosf[i].V), ts} 99 | RandomTiny0f[i] = testdata.Point{math.Floor(RandomTinyf[i].V), ts} 100 | 101 | RandomSmallPosf[i] = testdata.Point{RandomTinyPosf[i].V * 100, ts} // 0-inf, but most vals are very low, mostly between 0 and 200, rarely goes over 1000 102 | RandomSmallf[i] = testdata.Point{RandomTinyf[i].V * 100, ts} // -inf to + inf, as many pos as neg, but similar as above, rarely goes under -1000 or over +1000 103 | RandomSmallPos2f[i] = testdata.Point{RoundNum(RandomSmallPosf[i].V, 2), ts} 104 | RandomSmall2f[i] = testdata.Point{RoundNum(RandomSmallf[i].V, 2), ts} 105 | RandomSmallPos1f[i] = testdata.Point{RoundNum(RandomSmallPosf[i].V, 1), ts} 106 | RandomSmall1f[i] = testdata.Point{RoundNum(RandomSmallf[i].V, 1), ts} 107 | RandomSmallPos0f[i] = testdata.Point{math.Floor(RandomSmallPosf[i].V), ts} 108 | RandomSmall0f[i] = testdata.Point{math.Floor(RandomSmallf[i].V), ts} 109 | 110 | Random60kPosf[i] = testdata.Point{rand.Float64() * 60000, ts} 111 | Random60kf[i] = testdata.Point{Random60kPosf[i].V, ts} 112 | if rand.Int()%2 == 0 { 113 | Random60kf[i].V *= -1.0 114 | } 115 | Random60kPos2f[i] = testdata.Point{RoundNum(Random60kPosf[i].V, 2), ts} 116 | Random60k2f[i] = testdata.Point{RoundNum(Random60kf[i].V, 2), ts} 117 | Random60kPos1f[i] = testdata.Point{RoundNum(Random60kPosf[i].V, 1), ts} 118 | Random60k1f[i] = testdata.Point{RoundNum(Random60kf[i].V, 1), ts} 119 | Random60kPos0f[i] = testdata.Point{math.Floor(Random60kPosf[i].V), ts} 120 | Random60k0f[i] = testdata.Point{math.Floor(Random60kf[i].V), ts} 121 | 122 | SmallTestDataPosf[i] = testdata.Point{float64(testdata.TwoHoursData[i%120].V) * 1.234567, ts} // THD is 650-680, so this is 0-150 123 | if rand.Int()%2 == 0 { 124 | SmallTestDataf[i] = testdata.Point{SmallTestDataPosf[i].V, ts} // -150 - 150 125 | } else { 126 | SmallTestDataf[i] = testdata.Point{-1 * SmallTestDataPosf[i].V, ts} 127 | } 128 | SmallTestDataPos0f[i] = testdata.Point{math.Floor(SmallTestDataPosf[i].V), ts} // 0-150 129 | SmallTestData0f[i] = testdata.Point{math.Floor(SmallTestDataf[i].V), ts} // -150 - 150 130 | 131 | RandomLargePosf[i] = testdata.Point{rand.ExpFloat64() * 0.0001 * math.MaxFloat64, ts} // 0-inf, rarely goes over maxfloat/1000 132 | RandomLargef[i] = testdata.Point{rand.NormFloat64() * 0.0001 * math.MaxFloat64, ts} // same buth also negative 133 | RandomLargePos0f[i] = testdata.Point{math.Floor(RandomLargePosf[i].V), ts} 134 | RandomLarge0f[i] = testdata.Point{math.Floor(RandomLargef[i].V), ts} 135 | 136 | LargeTestDataPosf[i] = testdata.Point{float64(testdata.TwoHoursData[i%120].V) * 0.00001234567 * math.MaxFloat64, ts} // 0-maxfloat/1000 137 | if rand.Int()%2 == 0 { 138 | LargeTestDataf[i] = testdata.Point{LargeTestDataPosf[i].V, ts} // -maxfloat/1000 ~maxfloat/1000 139 | } else { 140 | LargeTestDataf[i] = testdata.Point{-1 * LargeTestDataPosf[i].V, ts} 141 | } 142 | 143 | LargeTestDataPos0f[i] = testdata.Point{math.Floor(LargeTestDataPosf[i].V), ts} // 0-maxfloat/1000 144 | LargeTestData0f[i] = testdata.Point{math.Floor(LargeTestDataf[i].V), ts} // -mf/1000 ~ mx/1000 145 | } 146 | 147 | intervals := []int{10, 30, 60, 120, 360, 720, 1440} 148 | do := func(data []testdata.Point, comment string) string { 149 | str := "" 150 | for _, points := range intervals { 151 | s := tsz.New(data[0].T) 152 | for _, tt := range data[0:points] { 153 | s.Push(tt.T, tt.V) 154 | } 155 | size := len(s.Bytes()) 156 | BPerPoint := float64(size) / float64(points) 157 | str += fmt.Sprintf("\033[31m%d\033[39m\t%.2f\t", size, BPerPoint) 158 | } 159 | str += comment + "\t" 160 | return str 161 | } 162 | w := new(tabwriter.Writer) 163 | w.Init(os.Stdout, 5, 0, 1, ' ', tabwriter.AlignRight) 164 | fmt.Println("=== help ===") 165 | fmt.Println("CS = chunk size in Bytes") 166 | fmt.Println("BPP = Bytes per point (CS/num-points)") 167 | fmt.Println("d = integers stored as float64") 168 | fmt.Println("f = float64's with a bunch of decimal numbers") 169 | fmt.Println(".Xf = float64's with X decimal numbers") 170 | fmt.Println("[num1] a - b [num2]: a range between a and b with the occasional outliers up to num1 and num2") 171 | fmt.Println("=== data ===") 172 | str := "test" 173 | for _, points := range intervals { 174 | str += fmt.Sprintf("\t \033[39m%dCS\033[39m\t%dBPP", points, points) 175 | } 176 | cmtTinyPos := "0 ~ 10 [inf]" 177 | cmtTinyPosNeg := "[-inf] -10 ~ 10 [inf]" 178 | cmtSmallPos := "0 ~ 1000 [inf]" 179 | cmtSmallPosNeg := "[-inf] -1000 ~ 1000 [inf]" 180 | cmt60kPos := "0 ~60k" 181 | cmt60kPosNeg := "-60k ~ 60k" 182 | cmtSmallTestPos := "0~150" 183 | cmtSmallTestPosNeg := "-150~150" 184 | cmtRandomLargePos := "0 ~ MaxFloat64/1000 [inf]" 185 | cmtRandomLargePosNeg := "[-inf] -MaxFloat64/1000 ~ MaxFloat64/1000 [inf]" 186 | cmtLargeTestPos := "0 ~ MaxFloat64/1000" 187 | cmtLargeTestPosNeg := "-MaxFloat64/1000 ~ MaxFloat64/1000" 188 | fmt.Fprintln(w, str+"\tcomment\t") 189 | fmt.Fprintln(w, "constant zero d\t"+do(ConstantZero, "")) 190 | fmt.Fprintln(w, "constant one d\t"+do(ConstantOne, "")) 191 | fmt.Fprintln(w, "constant pos .3f\t"+do(ConstantPos3f, "")) 192 | fmt.Fprintln(w, "constant neg .3f\t"+do(ConstantNeg3f, "")) 193 | fmt.Fprintln(w, "constant pos .0f\t"+do(ConstantPos0f, "")) 194 | fmt.Fprintln(w, "constant neg .0f\t"+do(ConstantNeg0f, "")) 195 | fmt.Fprintln(w, "constant nearmax f\t"+do(ConstantNearMaxf, "")) 196 | fmt.Fprintln(w, "constant nearmin f\t"+do(ConstantNearMinf, "")) 197 | fmt.Fprintln(w, "constant nearmax .0f\t"+do(ConstantNearMax0f, "")) 198 | fmt.Fprintln(w, "constant nearmin .0f\t"+do(ConstantNearMin0f, "")) 199 | fmt.Fprintln(w, "batch100 zero/one d\t"+do(Batch100ZeroOne, "")) 200 | fmt.Fprintln(w, "flapping zero/one d\t"+do(FlappingZeroOne, "")) 201 | fmt.Fprintln(w, "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t") 202 | fmt.Fprintln(w, "random tiny pos f\t"+do(RandomTinyPosf, cmtTinyPos)) 203 | fmt.Fprintln(w, "random tiny pos/neg f\t"+do(RandomTinyf, cmtTinyPosNeg)) 204 | fmt.Fprintln(w, "random tiny pos .2f\t"+do(RandomTinyPos2f, cmtTinyPos)) 205 | fmt.Fprintln(w, "random tiny pos/neg .2f\t"+do(RandomTiny2f, cmtTinyPosNeg)) 206 | fmt.Fprintln(w, "random tiny pos .1f\t"+do(RandomTinyPos1f, cmtTinyPos)) 207 | fmt.Fprintln(w, "random tiny pos/neg .1f\t"+do(RandomTiny1f, cmtTinyPosNeg)) 208 | fmt.Fprintln(w, "random tiny pos .0f\t"+do(RandomTinyPos0f, cmtTinyPos)) 209 | fmt.Fprintln(w, "random tiny pos/neg .0f\t"+do(RandomTiny0f, cmtTinyPosNeg)) 210 | fmt.Fprintln(w, "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t") 211 | fmt.Fprintln(w, "testdata small pos f\t"+do(SmallTestDataPosf, cmtSmallTestPos)) 212 | fmt.Fprintln(w, "testdata small pos/neg f\t"+do(SmallTestDataf, cmtSmallTestPosNeg)) 213 | fmt.Fprintln(w, "testdata small pos .0f\t"+do(SmallTestDataPos0f, cmtSmallTestPos)) 214 | fmt.Fprintln(w, "testdata small pos/neg .0f\t"+do(SmallTestData0f, cmtSmallTestPosNeg)) 215 | fmt.Fprintln(w, "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t") 216 | fmt.Fprintln(w, "random small pos f\t"+do(RandomSmallPosf, cmtSmallPos)) 217 | fmt.Fprintln(w, "random small pos/neg f\t"+do(RandomSmallf, cmtSmallPosNeg)) 218 | fmt.Fprintln(w, "random small pos .2f\t"+do(RandomSmallPos2f, cmtSmallPos)) 219 | fmt.Fprintln(w, "random small pos/neg .2f\t"+do(RandomSmall2f, cmtSmallPosNeg)) 220 | fmt.Fprintln(w, "random small pos .1f\t"+do(RandomSmallPos1f, cmtSmallPos)) 221 | fmt.Fprintln(w, "random small pos/neg .1f\t"+do(RandomSmall1f, cmtSmallPosNeg)) 222 | fmt.Fprintln(w, "random small pos .0f\t"+do(RandomSmallPos0f, cmtSmallPos)) 223 | fmt.Fprintln(w, "random small pos/neg .0f\t"+do(RandomSmall0f, cmtSmallPosNeg)) 224 | fmt.Fprintln(w, "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t") 225 | fmt.Fprintln(w, "random medium pos f\t"+do(Random60kPosf, cmt60kPos)) 226 | fmt.Fprintln(w, "random medium pos/neg f\t"+do(Random60kf, cmt60kPosNeg)) 227 | fmt.Fprintln(w, "random medium pos .2f\t"+do(Random60kPos2f, cmt60kPos)) 228 | fmt.Fprintln(w, "random medium pos/neg .2f\t"+do(Random60k2f, cmt60kPosNeg)) 229 | fmt.Fprintln(w, "random medium pos .1f\t"+do(Random60kPos1f, cmt60kPos)) 230 | fmt.Fprintln(w, "random medium pos/neg .1f\t"+do(Random60k1f, cmt60kPosNeg)) 231 | fmt.Fprintln(w, "random medium pos .0f\t"+do(Random60kPos0f, cmt60kPos)) 232 | fmt.Fprintln(w, "random medium pos/neg .0f\t"+do(Random60k0f, cmt60kPosNeg)) 233 | fmt.Fprintln(w, "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t") 234 | fmt.Fprintln(w, "testdata large pos f\t"+do(LargeTestDataPosf, cmtLargeTestPos)) 235 | fmt.Fprintln(w, "testdata large pos/neg f\t"+do(LargeTestDataf, cmtLargeTestPosNeg)) 236 | fmt.Fprintln(w, "testdata large pos .0f\t"+do(LargeTestDataPos0f, cmtLargeTestPos)) 237 | fmt.Fprintln(w, "testdata large pos/neg .0f\t"+do(LargeTestData0f, cmtLargeTestPosNeg)) 238 | fmt.Fprintln(w, "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t") 239 | fmt.Fprintln(w, "random large pos f\t"+do(RandomLargePosf, cmtRandomLargePos)) 240 | fmt.Fprintln(w, "random large pos/neg f\t"+do(RandomLargef, cmtRandomLargePosNeg)) 241 | fmt.Fprintln(w, "random large pos .0f\t"+do(RandomLargePos0f, cmtRandomLargePos)) 242 | fmt.Fprintln(w, "random large pos/neg .0f\t"+do(RandomLarge0f, cmtRandomLargePosNeg)) 243 | w.Flush() 244 | } 245 | --------------------------------------------------------------------------------