├── .gitignore ├── .gitmodules ├── Makefile ├── README.md ├── bin └── .gitkeep ├── config.yml.example ├── gobatsd ├── config.go ├── counter.go ├── counter_test.go ├── datapoint.go ├── datapoint_test.go ├── datastore.go ├── datastore_test.go ├── gauge.go ├── heartbeat.go ├── math.go ├── metric.go ├── server_test.go ├── timer.go └── timers_test.go ├── hbase_migration ├── counters_export.go ├── gauges_export.go ├── migrate_to_hbase.go └── timers_export.go ├── proxy.go ├── receiver.go ├── server.go ├── test ├── fake_data └── integration │ ├── config.yml │ ├── receiver │ └── server └── truncator.go /.gitignore: -------------------------------------------------------------------------------- 1 | build/* 2 | *.deb 3 | bin/* 4 | *.test 5 | cpuprof* -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "vendor/github.com/noahhl/clamp"] 2 | path = vendor/github.com/noahhl/clamp 3 | url = git@github.com:noahhl/clamp 4 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: bin/server bin/truncator bin/proxy bin/receiver 2 | 3 | bin/server: server.go 4 | go build -o bin/go-batsd-server server.go 5 | 6 | bin/receiver: receiver.go 7 | go build -o bin/go-batsd-receiver receiver.go 8 | 9 | bin/truncator: truncator.go 10 | go build -o bin/go-batsd-truncator truncator.go 11 | 12 | bin/proxy: proxy.go 13 | go build -o bin/go-batsd-proxy proxy.go 14 | 15 | clean: 16 | rm -f bin/* 17 | make all 18 | 19 | package: 20 | make clean 21 | fpm -s dir -t deb -n go-batsd -v ${VERSION} --prefix /usr/local/go-batsd bin 22 | 23 | check: 24 | go test ./gobatsd 25 | ./test/integration/receiver 26 | 27 | bench: 28 | go test -bench=".*" ./test/*_test.go 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Go-Batsd 2 | ====== 3 | 4 | Batsd is a Golang-based daemon for aggregating and storing statistics. It targets 5 | "wireline" compatibility with [Etsy's StatsD 6 | implementation](https://github.com/etsy/statsd), which they described in 7 | a [blog post](http://codeascraft.etsy.com/2011/02/15/measure-anything-measure-everything/). 8 | 9 | It's very similar to https://github.com/noahhl/batsd, which is written in Ruby. 10 | 11 | # License 12 | 13 | Copyright (c) 2013 Noah Lorang 14 | 15 | Permission is hereby granted, free of charge, to any person obtaining 16 | a copy of this software and associated documentation files (the 17 | "Software"), to deal in the Software without restriction, including 18 | without limitation the rights to use, copy, modify, merge, publish, 19 | distribute, sublicense, and/or sell copies of the Software, and to 20 | permit persons to whom the Software is furnished to do so, subject to 21 | the following conditions: 22 | 23 | The above copyright notice and this permission notice shall be 24 | included in all copies or substantial portions of the Software. 25 | 26 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 27 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 28 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 29 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 30 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 31 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 32 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 33 | -------------------------------------------------------------------------------- /bin/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/noahhl/go-batsd/147d2a3addb9248033ef0d58c5f40745ffa715cf/bin/.gitkeep -------------------------------------------------------------------------------- /config.yml.example: -------------------------------------------------------------------------------- 1 | bind: 0.0.0.0 2 | port: 8125 3 | health_port: 8126 4 | root: /u/code/batsd/tmp/statsd 5 | redis: 6 | host: 127.0.0.1 7 | port: 6379 8 | retentions: 9 | - 10 240 #1 hour 10 | - 60 10080 #1 week 11 | - 600 52594 #1 year 12 | autotruncate: false 13 | threadpool_size: 10 14 | serializer: marshal 15 | proxy: 16 | strategy: passthrough 17 | destinations: 18 | backend1: 19 | host: 127.0.0.1 20 | port: 8225 21 | hbase_hosts: 22 | - 1.2.3.4:7496 23 | -------------------------------------------------------------------------------- /gobatsd/config.go: -------------------------------------------------------------------------------- 1 | package gobatsd 2 | 3 | import ( 4 | "flag" 5 | "github.com/kylelemons/go-gypsy/yaml" 6 | "path/filepath" 7 | "strconv" 8 | "strings" 9 | ) 10 | 11 | type Retention struct { 12 | Interval, Count, Duration int64 13 | Index int 14 | } 15 | 16 | type Configuration struct { 17 | Port, Root string 18 | Retentions []Retention 19 | RedisHost string 20 | RedisPort int 21 | TargetInterval int64 22 | HbaseConnections []string 23 | HbaseTable string 24 | Hbase bool 25 | } 26 | 27 | var Config Configuration 28 | var ProfileCPU bool 29 | var channelBufferSize = 10000 30 | 31 | func LoadConfig() { 32 | configPath := flag.String("config", "./config.yml", "config file path") 33 | port := flag.String("port", "default", "port to bind to") 34 | duration := flag.Int64("duration", 0, "duration to operation on") 35 | cpuprofile := flag.Bool("cpuprofile", false, "write cpu profile to file") 36 | hbase := flag.Bool("hbase", false, "send to hbase") 37 | hbaseTable := flag.String("table", "statsd", "hbase table to write to") 38 | flag.Parse() 39 | 40 | absolutePath, _ := filepath.Abs(*configPath) 41 | c, err := yaml.ReadFile(absolutePath) 42 | if err != nil { 43 | panic(err) 44 | } 45 | root, _ := c.Get("root") 46 | if *port == "default" { 47 | *port, _ = c.Get("port") 48 | } 49 | numRetentions, _ := c.Count("retentions") 50 | retentions := make([]Retention, numRetentions) 51 | for i := 0; i < numRetentions; i++ { 52 | retention, _ := c.Get("retentions[" + strconv.Itoa(i) + "]") 53 | parts := strings.Split(retention, " ") 54 | d, _ := strconv.ParseInt(parts[0], 0, 64) 55 | n, _ := strconv.ParseInt(parts[1], 0, 64) 56 | retentions[i] = Retention{d, n, d * n, i} 57 | } 58 | p, _ := c.Get("redis.port") 59 | redisPort, _ := strconv.Atoi(p) 60 | redisHost, _ := c.Get("redis.host") 61 | 62 | numHbases, err := c.Count("hbase_hosts") 63 | if err != nil && *hbase { 64 | panic(err) 65 | } 66 | var hbaseConnections []string 67 | if *hbase { 68 | hbaseConnections = make([]string, numHbases) 69 | for i := 0; i < numHbases; i++ { 70 | hbaseConnections[i], _ = c.Get("hbase_hosts[" + strconv.Itoa(i) + "]") 71 | } 72 | } 73 | 74 | Config = Configuration{*port, root, retentions, redisHost, redisPort, *duration, hbaseConnections, *hbaseTable, *hbase} 75 | 76 | ProfileCPU = *cpuprofile 77 | } 78 | -------------------------------------------------------------------------------- /gobatsd/counter.go: -------------------------------------------------------------------------------- 1 | package gobatsd 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "strconv" 7 | "time" 8 | ) 9 | 10 | type Counter struct { 11 | Key string 12 | Values []float64 13 | channels []chan float64 14 | Paths []string 15 | } 16 | 17 | const counterInternalBufferSize = 10 18 | 19 | func NewCounter(name string) Metric { 20 | c := &Counter{} 21 | c.Key = name 22 | c.Values = make([]float64, len(Config.Retentions)) 23 | c.channels = make([]chan float64, len(Config.Retentions)) 24 | for i := range c.channels { 25 | c.channels[i] = make(chan float64, counterInternalBufferSize) 26 | } 27 | c.Paths = make([]string, len(Config.Retentions)) 28 | for i := range c.Paths { 29 | c.Paths[i] = CalculateFilename(fmt.Sprintf("counters:%v:%v", c.Key, Config.Retentions[i].Interval), Config.Root) 30 | } 31 | c.Start() 32 | return c 33 | } 34 | 35 | func (c *Counter) Start() { 36 | for i := range Config.Retentions { 37 | go func(retention Retention) { 38 | ticker := NewTickerWithOffset(time.Duration(retention.Interval)*time.Second, 39 | time.Duration(rand.Intn(int(retention.Interval)))*time.Second) 40 | for { 41 | select { 42 | case now := <-ticker: 43 | //fmt.Printf("%v: Time to save %v at retention %v\n", now, c.Key, retention) 44 | c.save(retention, now) 45 | case val := <-c.channels[retention.Index]: 46 | c.Values[retention.Index] += val 47 | } 48 | } 49 | }(Config.Retentions[i]) 50 | 51 | } 52 | } 53 | 54 | func (c *Counter) Update(value float64) { 55 | for i := range c.channels { 56 | c.channels[i] <- value 57 | } 58 | } 59 | 60 | func (c *Counter) save(retention Retention, now time.Time) { 61 | aggregateValue := c.Values[retention.Index] 62 | c.Values[retention.Index] = 0 63 | timestamp := now.Unix() - now.Unix()%retention.Interval 64 | //fmt.Printf("%v: Ready to store %v, value now %v, retention #%v\n", timestamp, aggregateValue, c.Values[retention.Index], retention.Index) 65 | if aggregateValue == 0 { 66 | return 67 | } 68 | 69 | if retention.Index == 0 { 70 | observation := AggregateObservation{Name: "counters:" + c.Key, Content: fmt.Sprintf("%d%v", timestamp, aggregateValue), Timestamp: timestamp, RawName: "counters:" + c.Key, Path: ""} 71 | StoreInRedis(observation) 72 | } else { 73 | observation := AggregateObservation{Name: "counters:" + c.Key + ":" + strconv.FormatInt(retention.Interval, 10), Content: fmt.Sprintf("%d %v\n", timestamp, aggregateValue), 74 | Timestamp: timestamp, RawName: "counters:" + c.Key, Path: c.Paths[retention.Index], SummaryValues: map[string]float64{"value": aggregateValue}, Interval: retention.Interval} 75 | StoreOnDisk(observation) 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /gobatsd/counter_test.go: -------------------------------------------------------------------------------- 1 | package gobatsd 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestCounterIncrement(t *testing.T) { 10 | Config.Retentions = []Retention{{10, 10, 100, 0}} 11 | Config.RedisHost = "127.0.0.1" 12 | Config.RedisPort = 6379 13 | SetupDatastore() 14 | 15 | c := NewCounter("test").(*Counter) 16 | c.Update(1.3) 17 | time.Sleep(1 * time.Millisecond) 18 | if c.Values[0] != 1.3 { 19 | t.Errorf("Expected counter to increment") 20 | } 21 | c.save(Config.Retentions[0], time.Now()) 22 | } 23 | 24 | func BenchmarkCounterIncrement(b *testing.B) { 25 | Config.Retentions = []Retention{{10, 10, 100, 0}} 26 | Config.RedisHost = "127.0.0.1" 27 | Config.RedisPort = 6379 28 | SetupDatastore() 29 | 30 | c := NewCounter("test").(*Counter) 31 | for j := 0; j < b.N; j++ { 32 | c.Update(1) 33 | } 34 | } 35 | 36 | func TestCounterSave(t *testing.T) { 37 | Config.Retentions = []Retention{{10, 10, 100, 0}, {20, 10, 200, 1}} 38 | Config.RedisHost = "127.0.0.1" 39 | Config.RedisPort = 6379 40 | SetupDatastore() 41 | 42 | c := NewCounter("test").(*Counter) 43 | c.Values[0] = 123 44 | now := time.Now() 45 | c.save(Config.Retentions[0], now) 46 | obs := <-datastore.redisChannel 47 | expected := fmt.Sprintf("%v%v", now.Unix()-now.Unix()%10, 123) 48 | if obs.Content != expected { 49 | t.Errorf("Counter send to redis was not properly structured, got '%v' expected '%v'", obs.Content, expected) 50 | } 51 | c.Values[1] = 123 52 | c.save(Config.Retentions[1], now) 53 | obs = <-datastore.diskChannel 54 | expected = fmt.Sprintf("%v %v\n", now.Unix()-now.Unix()%20, 123) 55 | if obs.Content != expected { 56 | t.Errorf("Counter send to disk was not properly structured, got '%v' expected '%v'", obs.Content, expected) 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /gobatsd/datapoint.go: -------------------------------------------------------------------------------- 1 | package gobatsd 2 | 3 | import ( 4 | "strconv" 5 | "strings" 6 | ) 7 | 8 | type Datapoint struct { 9 | Name string 10 | Value float64 11 | Datatype string 12 | } 13 | 14 | type AggregateObservation struct { 15 | Name string 16 | Content string 17 | Timestamp int64 18 | RawName string 19 | Path string 20 | SummaryValues map[string]float64 21 | Interval int64 22 | } 23 | 24 | func ParseDatapointFromString(metric string) Datapoint { 25 | d := Datapoint{} 26 | components := strings.Split(metric, ":") 27 | if len(components) == 2 { 28 | latter_components := strings.Split(components[1], "|") 29 | if len(latter_components) >= 2 { 30 | value, _ := strconv.ParseFloat(latter_components[0], 64) 31 | if len(latter_components) == 3 && latter_components[1] == "c" { 32 | sample_rate, _ := strconv.ParseFloat(strings.Replace(latter_components[2], "@", "", -1), 64) 33 | value = value / sample_rate 34 | } 35 | d = Datapoint{components[0], value, latter_components[1]} 36 | } 37 | } 38 | return d 39 | } 40 | 41 | func ArtisinallyMarshallDatapointJSON(values []map[string]string) []byte { 42 | marshalled := []byte{} 43 | marshalled = append(marshalled, '[') 44 | for i := range values { 45 | if i > 0 { 46 | marshalled = append(marshalled, ',') 47 | } 48 | //newstr := fmt.Sprintf("{\"Timestamp\": %v, \"Value\": %v}", values[i]["Timestamp"], values[i]["Value"]) 49 | for k := range "{\"Timestamp\":\"" { 50 | marshalled = append(marshalled, "{\"Timestamp\":\""[k]) 51 | } 52 | for k := range values[i]["Timestamp"] { 53 | marshalled = append(marshalled, values[i]["Timestamp"][k]) 54 | } 55 | marshalled = append(marshalled, '"', ',') 56 | for k := range "\"Value\":\"" { 57 | marshalled = append(marshalled, "\"Value\":\""[k]) 58 | } 59 | for k := range values[i]["Value"] { 60 | marshalled = append(marshalled, values[i]["Value"][k]) 61 | } 62 | marshalled = append(marshalled, '"', '}') 63 | 64 | } 65 | marshalled = append(marshalled, ']') 66 | return marshalled 67 | } 68 | 69 | type AggregateObservations []AggregateObservation 70 | 71 | func (o AggregateObservations) Len() int { 72 | return len(o) 73 | } 74 | 75 | func (o AggregateObservations) Swap(i, j int) { 76 | o[i], o[j] = o[j], o[i] 77 | } 78 | 79 | func (o AggregateObservations) Less(i, j int) bool { 80 | return o[i].Timestamp < o[j].Timestamp 81 | } 82 | -------------------------------------------------------------------------------- /gobatsd/datapoint_test.go: -------------------------------------------------------------------------------- 1 | package gobatsd 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestDatapointParsing(t *testing.T) { 8 | d := ParseDatapointFromString("foo:1|c") 9 | if d.Datatype != "c" { 10 | t.Errorf("Datatype wrong. Expected %v, got %v\n", "c", d.Datatype) 11 | } 12 | if d.Value != 1.0 { 13 | t.Errorf("Value wrong. Expected %v, got %v\n", 1.0, d.Value) 14 | } 15 | 16 | d = ParseDatapointFromString("foo:1|c|@0.1") 17 | if d.Datatype != "c" { 18 | t.Errorf("Datatype wrong. Expected %v, got %v\n", "c", d.Datatype) 19 | } 20 | if d.Value != 10 { 21 | t.Errorf("Value wrong. Expected %v, got %v\n", 10, d.Value) 22 | } 23 | 24 | d = ParseDatapointFromString("foo:1.7|g") 25 | if d.Datatype != "g" { 26 | t.Errorf("Datatype wrong. Expected %v, got %v\n", "g", d.Datatype) 27 | } 28 | if d.Value != 1.7 { 29 | t.Errorf("Value wrong. Expected %v, got %v\n", 1.7, d.Value) 30 | } 31 | 32 | d = ParseDatapointFromString("foo:30303|ms") 33 | if d.Datatype != "ms" { 34 | t.Errorf("Datatype wrong. Expected %v, got %v\n", "ms", d.Datatype) 35 | } 36 | if d.Value != 30303.0 { 37 | t.Errorf("Value wrong. Expected %v, got %v\n", 30303.0, d.Value) 38 | } 39 | 40 | } 41 | 42 | func BenchmarkDatapointParsing(b *testing.B) { 43 | for j := 0; j < b.N; j++ { 44 | ParseDatapointFromString("foo.bar.baz.a.long.random.metric:3920230|ms") 45 | } 46 | } 47 | 48 | func BenchmarkDatapointParsingWithSampleRate(b *testing.B) { 49 | for j := 0; j < b.N; j++ { 50 | ParseDatapointFromString("foo.bar.baz.a.long.random.metric:0.394|c|@0.1") 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /gobatsd/datastore.go: -------------------------------------------------------------------------------- 1 | package gobatsd 2 | 3 | import ( 4 | "github.com/noahhl/clamp" 5 | 6 | "bufio" 7 | "crypto/md5" 8 | "encoding/binary" 9 | "encoding/hex" 10 | "errors" 11 | "fmt" 12 | "github.com/jinntrance/goh" 13 | "github.com/jinntrance/goh/Hbase" 14 | "github.com/noahhl/Go-Redis" 15 | "io" 16 | "math" 17 | "math/rand" 18 | "os" 19 | "path/filepath" 20 | "sort" 21 | "syscall" 22 | "time" 23 | ) 24 | 25 | type Datastore struct { 26 | diskChannel chan AggregateObservation 27 | redisChannel chan AggregateObservation 28 | hbaseChannel chan AggregateObservation 29 | redisPool *clamp.ConnectionPoolWrapper 30 | hbasePool *clamp.ConnectionPoolWrapper 31 | } 32 | 33 | var numRedisRoutines = 50 34 | var numDiskRoutines = 25 35 | var numHbaseRoutines = 100 36 | 37 | const hbaseBatchSize = 100 38 | 39 | var redisPoolSize = 20 40 | 41 | const diskstoreChannelSize = 100000 42 | 43 | var datastore Datastore 44 | 45 | func SetupDatastore() { 46 | datastore = Datastore{} 47 | datastore.diskChannel = make(chan AggregateObservation, diskstoreChannelSize) 48 | datastore.redisChannel = make(chan AggregateObservation, channelBufferSize) 49 | datastore.redisPool = MakeRedisPool(redisPoolSize) 50 | 51 | if Config.Hbase { 52 | datastore.hbaseChannel = make(chan AggregateObservation, channelBufferSize) 53 | datastore.hbasePool = MakeHbasePool(numHbaseRoutines) 54 | } 55 | 56 | go func() { 57 | c := time.Tick(1 * time.Second) 58 | for { 59 | <-c 60 | clamp.StatsChannel <- clamp.Stat{"datastoreRedisChannelSize", fmt.Sprintf("%v", len(datastore.redisChannel))} 61 | clamp.StatsChannel <- clamp.Stat{"datastoreDiskChannelSize", fmt.Sprintf("%v", len(datastore.diskChannel))} 62 | if Config.Hbase { 63 | clamp.StatsChannel <- clamp.Stat{"datastoreHbaseChannelSize", fmt.Sprintf("%v", len(datastore.hbaseChannel))} 64 | } 65 | } 66 | }() 67 | 68 | for i := 0; i < numHbaseRoutines; i++ { 69 | go func() { 70 | observations := make([]AggregateObservation, 0) 71 | for { 72 | obs := <-datastore.diskChannel 73 | observations = append(observations, obs) 74 | if len(observations) >= hbaseBatchSize { 75 | sort.Sort(AggregateObservations(observations)) 76 | batchStart := 0 77 | for k := 1; k < len(observations); k++ { 78 | if observations[k].Timestamp != observations[k-1].Timestamp { 79 | datastore.writeToHbaseInBulk(observations[batchStart:k]) 80 | batchStart = k 81 | } 82 | } 83 | datastore.writeToHbaseInBulk(observations[batchStart:len(observations)]) 84 | 85 | observations = make([]AggregateObservation, 0) 86 | } 87 | } 88 | }() 89 | } 90 | 91 | for i := 0; i < numRedisRoutines; i++ { 92 | go func() { 93 | for { 94 | obs := <-datastore.redisChannel 95 | datastore.writeToRedis(obs) 96 | } 97 | }() 98 | } 99 | } 100 | 101 | func MakeRedisPool(size int) *clamp.ConnectionPoolWrapper { 102 | pool := &clamp.ConnectionPoolWrapper{} 103 | pool.InitPool(size, openRedisConnection) 104 | return pool 105 | } 106 | 107 | func MakeHbasePool(size int) *clamp.ConnectionPoolWrapper { 108 | pool := &clamp.ConnectionPoolWrapper{} 109 | pool.InitPool(size, OpenHbaseConnection) 110 | return pool 111 | } 112 | 113 | func StoreOnDisk(observation AggregateObservation) { 114 | datastore.diskChannel <- observation 115 | } 116 | 117 | func StoreInRedis(observation AggregateObservation) { 118 | datastore.redisChannel <- observation 119 | } 120 | 121 | func openRedisConnection() (interface{}, error) { 122 | spec := redis.DefaultSpec().Host(Config.RedisHost).Port(Config.RedisPort) 123 | r, err := redis.NewSynchClientWithSpec(spec) 124 | return r, err 125 | } 126 | 127 | func OpenHbaseConnection() (interface{}, error) { 128 | host := Config.HbaseConnections[rand.Intn(len(Config.HbaseConnections))] 129 | fmt.Printf("%v: Opening an hbase connection to %v\n", time.Now(), host) 130 | hbaseClient, err := goh.NewTcpClient(host, goh.TBinaryProtocol, false, 10*time.Second) 131 | if err != nil { 132 | fmt.Println(err) 133 | } 134 | err = hbaseClient.Open() 135 | if err != nil { 136 | fmt.Println(err) 137 | } 138 | return hbaseClient, err 139 | } 140 | 141 | func (d *Datastore) RecordMetric(name string) { 142 | r := d.redisPool.GetConnection().(redis.Client) 143 | defer d.redisPool.ReleaseConnection(r) 144 | r.Sadd("datapoints", []byte(name)) 145 | } 146 | 147 | func (d *Datastore) RecordCurrent(name string, val float64) { 148 | r := d.redisPool.GetConnection().(redis.Client) 149 | defer d.redisPool.ReleaseConnection(r) 150 | r.Set(name, EncodeFloat64(val)) 151 | } 152 | 153 | func (d *Datastore) writeToRedis(observation AggregateObservation) { 154 | r := d.redisPool.GetConnection().(redis.Client) 155 | defer d.redisPool.ReleaseConnection(r) 156 | r.Zadd(observation.Name, float64(observation.Timestamp), []byte(observation.Content)) 157 | } 158 | 159 | func (d *Datastore) writeToDisk(observation AggregateObservation) { 160 | 161 | file, err := os.OpenFile(observation.Path, os.O_APPEND|os.O_WRONLY, 0600) 162 | newFile := false 163 | if err != nil { 164 | if e, ok := err.(*os.PathError); ok && e.Err == syscall.ENOENT { 165 | //fmt.Printf("Creating %v\n", filename) 166 | //Make containing directories if they don't exist 167 | err = os.MkdirAll(filepath.Dir(observation.Path), 0755) 168 | if err != nil { 169 | fmt.Printf("%v\n", err) 170 | } 171 | 172 | file, err = os.Create(observation.Path) 173 | if err != nil { 174 | fmt.Printf("%v\n", err) 175 | } 176 | newFile = true 177 | } else { 178 | panic(err) 179 | } 180 | } 181 | if file != nil { 182 | writer := bufio.NewWriter(file) 183 | if newFile { 184 | writer.WriteString("v2 " + observation.Name + "\n") 185 | } 186 | writer.WriteString(observation.Content) 187 | writer.Flush() 188 | file.Close() 189 | 190 | } 191 | } 192 | 193 | func (d *Datastore) writeToHbase(observation AggregateObservation) { 194 | c := d.hbasePool.GetConnection().(*goh.HClient) 195 | mutations := make([]*Hbase.Mutation, 0) 196 | for k, v := range observation.SummaryValues { 197 | mutations = append(mutations, goh.NewMutation(fmt.Sprintf("interval%v:%v", observation.Interval, k), EncodeFloat64(v))) 198 | } 199 | 200 | err := c.MutateRowTs(Config.HbaseTable, []byte(observation.RawName), mutations, observation.Timestamp, nil) 201 | if err != nil { 202 | fmt.Printf("%v: mutate error - %v\n", time.Now(), err) 203 | } 204 | d.hbasePool.ReleaseConnection(c) 205 | } 206 | 207 | func (d *Datastore) writeToHbaseInBulk(observations []AggregateObservation) { 208 | c := d.hbasePool.GetConnection().(*goh.HClient) 209 | bulk := make([]*Hbase.BatchMutation, len(observations)) 210 | for i := range observations { 211 | mutations := make([]*Hbase.Mutation, 0) 212 | for k, v := range observations[i].SummaryValues { 213 | mutations = append(mutations, goh.NewMutation(fmt.Sprintf("interval%v:%v", observations[i].Interval, k), EncodeFloat64(v))) 214 | } 215 | bulk[i] = goh.NewBatchMutation([]byte(observations[i].RawName), mutations) 216 | 217 | } 218 | err := c.MutateRowsTs(Config.HbaseTable, bulk, observations[0].Timestamp, nil) 219 | if err != nil { 220 | fmt.Printf("%v: mutate error using %v - %v, reconnecting\n", time.Now(), c, err) 221 | x, _ := OpenHbaseConnection() 222 | c = x.(*goh.HClient) 223 | } 224 | d.hbasePool.ReleaseConnection(c) 225 | } 226 | 227 | func CalculateFilename(metric string, root string) string { 228 | h := md5.New() 229 | io.WriteString(h, metric) 230 | metricHash := hex.EncodeToString(h.Sum([]byte{})) 231 | return root + "/" + metricHash[0:2] + "/" + metricHash[2:4] + "/" + metricHash 232 | } 233 | 234 | func EncodeFloat64(float float64) []byte { 235 | bits := math.Float64bits(float) 236 | bytes := make([]byte, 8) 237 | binary.BigEndian.PutUint64(bytes, bits) 238 | return bytes 239 | } 240 | 241 | func DecodeFloat64(b []byte) (float64, error) { 242 | if len(b) == 8 { 243 | bits := binary.BigEndian.Uint64(b) 244 | return math.Float64frombits(bits), nil 245 | } 246 | err := errors.New("Incorrect number of bits for a float64") 247 | return 0, *&err 248 | } 249 | -------------------------------------------------------------------------------- /gobatsd/datastore_test.go: -------------------------------------------------------------------------------- 1 | package gobatsd 2 | 3 | import ( 4 | "fmt" 5 | "github.com/noahhl/Go-Redis" 6 | "github.com/noahhl/clamp" 7 | "math/rand" 8 | "os" 9 | "strings" 10 | "testing" 11 | "time" 12 | ) 13 | 14 | var metric_samples = []string{"timers:sysstat.statsd-101.bread/s.8822.00", "timers:sysstat.statsd-101.rtps.3703.00", "timers:sysstat.statsd-101.rtps.5161.00", "timers:sysstat.statsd-101.wtps.3033.00", "gauges:Syslog-NG.syslog-102.destination.d_app_writeboard_staging.empty.a.processed", "timers:sysstat.statsd-101.bread/s.3965.00", "timers:sysstat.statsd-101.bwrtn/s.3037.00", "timers:sysstat.statsd-101.rtps.8183.00", "timers:sysstat.statsd-101.rtps.6725.00", "timers:sysstat.statsd-101.wtps.6055.00", "timers:sysstat.statsd-101.bread/s.6987.00", "timers:sysstat.statsd-101.bwrtn/s.7540.00", "timers:sysstat.statsd-101.rtps.1868.00", "timers:sysstat.statsd-101.wtps.1198.00", "timers:sysstat.statsd-101.bwrtn/s.2683.00", "timers:sysstat.statsd-101.rtps.317.00", "timers:sysstat.bcx-101.bread/s.443.00", "timers:sysstat.statsd-101.wtps.7619.00", "timers:sysstat.statsd-101.wtps.9077.00", "gauges:Syslog-NG.bcx-109.src_internal.s_local_2.empty.a.stamp", "timers:sysstat.statsd-101.pgpgout/s.176.00", "timers:sysstat.statsd-101.bread/s.1063.00", "timers:sysstat.statsd-101.cswch/s.121.00", "timers:sysstat.statsd-102.bwrtn/s.428.00", "timers:sysstat.bcx-101.wtps.6269.00", "timers:sysstat.statsd-101.bread/s.2627.00", "timers:sysstat.statsd-101.bread/s.4085.00", "timers:sysstat.statsd-101.wtps.1341.00", "timers:sysstat.statsd-101.bread/s.5649.00", "timers:sysstat.bcx-101.rtps.14.00", "timers:sysstat.statsd-101.bwrtn/s.6202.00", "timers:sysstat.statsd-101.bread/s.317.00", "timers:sysstat.statsd-101.bwrtn/s.1345.00", "timers:sysstat.statsd-101.wtps.4363.00", "timers:sysstat.statsd-101.rtps.8409.00", "timers:sysstat.statsd-101.wtps.2905.00", "timers:sysstat.statsd-101.bwrtn/s.9224.00", "timers:sysstat.statsd-101.bwrtn/s.4367.00", "timers:sysstat.statsd-101.bwrtn/s.2909.00", "timers:sysstat.statsd-101.wtps.5927.00", "timers:sysstat.statsd-101.wtps.7385.00", "counters:memcached.shr-memory-102.11212.slab20.cas_hits", "timers:sysstat.statsd-101.bwrtn/s.8870.00", "timers:sysstat.statsd-101.bwrtn/s.7389.00", "timers:sysstat.statsd-101.wtps.8949.00", "timers:sysstat.statsd-102.bwrtn/s.194.00", "timers:sysstat.statsd-101.bwrtn/s.925.00", "timers:sysstat.statsd-101.rtps.2131.00", "timers:sysstat.statsd-101.bread/s.2393.00", "timers:sysstat.bcx-101.meff.3292.00"} 15 | 16 | func TestFilenameCalculation(t *testing.T) { 17 | filename := CalculateFilename("test_metric", "/u/batsd") 18 | expected := "/u/batsd/34/1e/341e012c2e30d7853542921c1d76c8da" 19 | if filename != expected { 20 | t.Errorf("Expected filename to be %v, was %v\n", expected, filename) 21 | } 22 | } 23 | 24 | func BenchmarkFilenameCalculation(b *testing.B) { 25 | for j := 0; j < b.N; j++ { 26 | CalculateFilename(metric_samples[rand.Intn(len(metric_samples))], "/u/batsd") 27 | } 28 | } 29 | 30 | func TestRecordingMetric(t *testing.T) { 31 | Config.RedisHost = "127.0.0.1" 32 | Config.RedisPort = 6379 33 | 34 | d := Datastore{} 35 | d.redisPool = &clamp.ConnectionPoolWrapper{} 36 | d.redisPool.InitPool(redisPoolSize, openRedisConnection) 37 | r := d.redisPool.GetConnection().(redis.Client) 38 | defer d.redisPool.ReleaseConnection(r) 39 | 40 | r.Del("datapoints") 41 | d.RecordMetric("testing") 42 | if n, _ := r.Scard("datapoints"); n != 1 { 43 | t.Errorf("Expected 1 datapoint, got %v\n", n) 44 | } 45 | if ok, _ := r.Sismember("datapoints", []byte("testing")); !ok { 46 | t.Errorf("Expected 'testing' to be a member of datapoints; it's not.\n") 47 | } 48 | 49 | } 50 | 51 | func TestSavingToRedis(t *testing.T) { 52 | Config.RedisHost = "127.0.0.1" 53 | Config.RedisPort = 6379 54 | 55 | d := Datastore{} 56 | d.redisPool = &clamp.ConnectionPoolWrapper{} 57 | d.redisPool.InitPool(redisPoolSize, openRedisConnection) 58 | 59 | r := d.redisPool.GetConnection().(redis.Client) 60 | defer d.redisPool.ReleaseConnection(r) 61 | 62 | r.Del("test_metric") 63 | obs := AggregateObservation{Name: "test_metric", Content: "123451", Timestamp: 1234, RawName: "1", Path: ""} 64 | d.writeToRedis(obs) 65 | 66 | if ok, _ := r.Exists("test_metric"); !ok { 67 | t.Errorf("Metric was not saved.\n") 68 | } 69 | 70 | if n, _ := r.Zcard("test_metric"); n != 1 { 71 | t.Errorf("Expected 1 value, got %v\n", n) 72 | } 73 | 74 | if vals, _ := r.Zrange("test_metric", 0, 1); string(vals[0]) != "123451" { 75 | t.Errorf("Expected value to be %v, was %v\n", obs.Content, string(vals[0])) 76 | } 77 | 78 | } 79 | 80 | func BenchmarkSavingToRedis(b *testing.B) { 81 | Config.RedisHost = "127.0.0.1" 82 | Config.RedisPort = 6379 83 | 84 | d := Datastore{} 85 | d.redisPool = &clamp.ConnectionPoolWrapper{} 86 | d.redisPool.InitPool(redisPoolSize, openRedisConnection) 87 | b.ResetTimer() 88 | for j := 0; j < b.N; j++ { 89 | now := time.Now().UnixNano() 90 | val := rand.Intn(1000) 91 | obs := AggregateObservation{Name: metric_samples[rand.Intn(len(metric_samples))], Content: fmt.Sprintf("%v%v", now, val), Timestamp: now, RawName: "1", Path: ""} 92 | d.writeToRedis(obs) 93 | } 94 | } 95 | 96 | func TestSavingToDisk(t *testing.T) { 97 | Config.Root = "/tmp/batsd" 98 | Config.RedisHost = "127.0.0.1" 99 | Config.RedisPort = 6379 100 | 101 | obs := AggregateObservation{Name: "test_metric", Content: "12345 1\n", Timestamp: 1234, RawName: "1", Path: CalculateFilename("test_metric", "/tmp/batsd")} 102 | obs2 := AggregateObservation{Name: "test_metric", Content: "123456 2\n", Timestamp: 1234, RawName: "1", Path: CalculateFilename("test_metric", "/tmp/batsd")} 103 | d := Datastore{} 104 | d.redisPool = &clamp.ConnectionPoolWrapper{} 105 | d.redisPool.InitPool(redisPoolSize, openRedisConnection) 106 | 107 | os.RemoveAll("/tmp/batsd") 108 | 109 | d.writeToDisk(obs) 110 | d.writeToDisk(obs2) 111 | 112 | file, err := os.Open(CalculateFilename("test_metric", "/tmp/batsd")) 113 | if err != nil { 114 | t.Fatalf("%v\n", err) 115 | } 116 | 117 | buffer := make([]byte, 1024) 118 | n, _ := file.Read(buffer) 119 | vals := strings.Split(string(buffer[0:n]), "\n") 120 | if vals[0] != "v2 test_metric" { 121 | t.Errorf("Expected first line of file to list name of metric, was %v\n", vals[0]) 122 | } 123 | 124 | if vals[1] != "12345 1" { 125 | t.Errorf("Expected second line of file to list timestamp and value, was '%v'\n", vals[1]) 126 | } 127 | if vals[1] != "12345 1" { 128 | t.Errorf("Expected second line of file to list timestamp and value, was '%v'\n", vals[1]) 129 | } 130 | if vals[2] != "123456 2" { 131 | t.Errorf("Expected third line of file to list timestamp and value, was '%v'\n", vals[2]) 132 | } 133 | 134 | file.Close() 135 | 136 | } 137 | 138 | func BenchmarkSavingToDisk(b *testing.B) { 139 | Config.Root = "/tmp/batsd" 140 | Config.RedisHost = "127.0.0.1" 141 | Config.RedisPort = 6379 142 | 143 | d := Datastore{} 144 | d.redisPool = &clamp.ConnectionPoolWrapper{} 145 | d.redisPool.InitPool(redisPoolSize, openRedisConnection) 146 | os.RemoveAll("/tmp/batsd") 147 | b.ResetTimer() 148 | for j := 0; j < b.N; j++ { 149 | val := rand.Intn(1000) 150 | o := AggregateObservation{Name: metric_samples[rand.Intn(len(metric_samples))], Content: fmt.Sprintf("%v %v\n", time.Now().Unix(), val), Timestamp: time.Now().Unix(), RawName: "", Path: CalculateFilename("test_metric", "/tmp/batsd")} 151 | d.writeToDisk(o) 152 | } 153 | } 154 | 155 | func BenchmarkRedisPool(b *testing.B) { 156 | Config.RedisHost = "127.0.0.1" 157 | Config.RedisPort = 6379 158 | 159 | d := Datastore{} 160 | d.redisPool = &clamp.ConnectionPoolWrapper{} 161 | d.redisPool.InitPool(redisPoolSize, openRedisConnection) 162 | b.ResetTimer() 163 | for j := 0; j < b.N; j++ { 164 | r := d.redisPool.GetConnection().(redis.Client) 165 | d.redisPool.ReleaseConnection(r) 166 | } 167 | 168 | } 169 | -------------------------------------------------------------------------------- /gobatsd/gauge.go: -------------------------------------------------------------------------------- 1 | package gobatsd 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | type Gauge struct { 9 | Key string 10 | Path string 11 | } 12 | 13 | const gaugeInternalBufferSize = 10 14 | 15 | func NewGauge(name string) Metric { 16 | g := &Gauge{} 17 | g.Key = name 18 | g.Path = CalculateFilename("gauges:"+g.Key, Config.Root) 19 | return g 20 | } 21 | 22 | func (g *Gauge) Start() { 23 | } 24 | 25 | func (g *Gauge) Update(value float64) { 26 | observation := AggregateObservation{Name: "gauges:" + g.Key, Content: fmt.Sprintf("%d %v\n", time.Now().Unix(), value), Timestamp: time.Now().Unix(), RawName: "gauges:" + g.Key, 27 | Path: g.Path, SummaryValues: map[string]float64{"value": value}, Interval: 0} 28 | StoreOnDisk(observation) 29 | datastore.RecordCurrent("gauges:"+g.Key, value) 30 | } 31 | -------------------------------------------------------------------------------- /gobatsd/heartbeat.go: -------------------------------------------------------------------------------- 1 | package gobatsd 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | var InternalMetrics map[string]Metric 8 | 9 | func InitializeInternalMetrics() { 10 | InternalMetrics = map[string]Metric{ 11 | "countersProcessed": NewCounter("statsd.countersProcessed"), 12 | "gaugesProcessed": NewCounter("statsd.gaugesProcessed"), 13 | "timersProcessed": NewCounter("statsd.timersProcessed"), 14 | "countersKnown": NewTimer("statsd.countersKnown"), 15 | "gaugesKnown": NewTimer("statsd.gaugesKnown"), 16 | "timersKnown": NewTimer("statsd.timersKnown"), 17 | } 18 | 19 | } 20 | 21 | func NewTickerWithOffset(frequency time.Duration, offset time.Duration) chan time.Time { 22 | ch := make(chan time.Time) 23 | go func() { 24 | time.Sleep(offset) 25 | ch <- time.Now() 26 | c := time.Tick(frequency) 27 | for now := range c { 28 | ch <- now 29 | } 30 | }() 31 | return ch 32 | } 33 | -------------------------------------------------------------------------------- /gobatsd/math.go: -------------------------------------------------------------------------------- 1 | package gobatsd 2 | 3 | import ( 4 | "math" 5 | ) 6 | 7 | func SortedMin(a []float64) float64 { 8 | return a[0] 9 | } 10 | 11 | func SortedMax(a []float64) float64 { 12 | return a[len(a)-1] 13 | } 14 | 15 | func Sum(a []float64) float64 { 16 | sum := 0.0 17 | for i := range a { 18 | sum += a[i] 19 | } 20 | return sum 21 | } 22 | 23 | func Mean(a []float64) float64 { 24 | return Sum(a) / float64(len(a)) 25 | } 26 | 27 | func MeanSquared(a []float64) float64 { 28 | ms := 0.0 29 | mean := Mean(a) 30 | for i := range a { 31 | ms += math.Pow(a[i]-mean, 2.0) 32 | } 33 | return ms 34 | 35 | } 36 | 37 | func SortedMedian(a []float64) float64 { 38 | return a[len(a)/2] 39 | } 40 | 41 | func Stddev(a []float64) float64 { 42 | stddev := 0.0 43 | if len(a) > 1 { 44 | stddev = math.Pow(MeanSquared(a)/float64(len(a)-1), 0.5) 45 | } 46 | return stddev 47 | } 48 | 49 | func SortedPercentile(a []float64, p float64) float64 { 50 | return a[int(float64(len(a))*p)] 51 | } 52 | -------------------------------------------------------------------------------- /gobatsd/metric.go: -------------------------------------------------------------------------------- 1 | package gobatsd 2 | 3 | type Metric interface { 4 | Start() 5 | Update(float64) 6 | } 7 | -------------------------------------------------------------------------------- /gobatsd/server_test.go: -------------------------------------------------------------------------------- 1 | package gobatsd 2 | 3 | import ( 4 | "encoding/json" 5 | "math/rand" 6 | "strconv" 7 | "testing" 8 | "time" 9 | ) 10 | 11 | func TestJSONEncoding(t *testing.T) { 12 | values := make([]map[string]string, 0) 13 | for i := 0; i < 100; i++ { 14 | values = append(values, map[string]string{"Timestamp": strconv.FormatInt(time.Now().Unix(), 10), "Value": strconv.FormatFloat(rand.Float64()*1000, 'f', 0, 64)}) 15 | } 16 | 17 | packageJSON, _ := json.Marshal(values) 18 | artisinalJSON := ArtisinallyMarshallDatapointJSON(values) 19 | if string(artisinalJSON) != string(packageJSON) { 20 | t.Errorf("Expected artisinal JSON to match package JSON; artisinal was\n\n%v\n\npackage was \n\n%v ", string(artisinalJSON), string(packageJSON)) 21 | } 22 | } 23 | 24 | func BenchmarkJSONPackageEncoding(b *testing.B) { 25 | values := make([]map[string]string, 0) 26 | for i := 0; i < 1000; i++ { 27 | values = append(values, map[string]string{"Timestamp": strconv.FormatInt(time.Now().Unix(), 10), "Value": strconv.FormatFloat(rand.Float64()*1000, 'f', 0, 64)}) 28 | } 29 | for j := 0; j < b.N; j++ { 30 | json.Marshal(values) 31 | } 32 | } 33 | 34 | func BenchmarkJSONArtisinalEncoding(b *testing.B) { 35 | values := make([]map[string]string, 0) 36 | for i := 0; i < 1000; i++ { 37 | values = append(values, map[string]string{"Timestamp": strconv.FormatInt(time.Now().Unix(), 10), "Value": strconv.FormatFloat(rand.Float64()*1000, 'f', 0, 64)}) 38 | } 39 | for j := 0; j < b.N; j++ { 40 | ArtisinallyMarshallDatapointJSON(values) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /gobatsd/timer.go: -------------------------------------------------------------------------------- 1 | package gobatsd 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "sort" 7 | "strconv" 8 | "time" 9 | ) 10 | 11 | type Timer struct { 12 | Key string 13 | Values [][]float64 14 | channels []chan float64 15 | Paths []string 16 | } 17 | 18 | const timerInternalBufferSize = 10 19 | const timerVersion = "2" 20 | 21 | func NewTimer(name string) Metric { 22 | t := &Timer{} 23 | t.Key = name 24 | t.Values = make([][]float64, len(Config.Retentions)) 25 | for i := range t.Values { 26 | t.Values[i] = make([]float64, 0) 27 | } 28 | t.channels = make([]chan float64, len(Config.Retentions)) 29 | for i := range t.channels { 30 | t.channels[i] = make(chan float64, timerInternalBufferSize) 31 | } 32 | t.Paths = make([]string, len(Config.Retentions)) 33 | for i := range t.Paths { 34 | t.Paths[i] = CalculateFilename(fmt.Sprintf("timers:%v:%v:%v", t.Key, Config.Retentions[i].Interval, timerVersion), Config.Root) 35 | } 36 | t.Start() 37 | return t 38 | } 39 | 40 | func (t *Timer) Start() { 41 | for i := range Config.Retentions { 42 | go func(retention Retention) { 43 | ticker := NewTickerWithOffset(time.Duration(retention.Interval)*time.Second, 44 | time.Duration(rand.Intn(int(retention.Interval)))*time.Second) 45 | for { 46 | select { 47 | case now := <-ticker: 48 | //fmt.Printf("%v: Time to save %v at retention %v\n", now, c.Key, retention) 49 | t.save(retention, now) 50 | case val := <-t.channels[retention.Index]: 51 | t.Values[retention.Index] = append(t.Values[retention.Index], val) 52 | } 53 | } 54 | }(Config.Retentions[i]) 55 | 56 | } 57 | } 58 | 59 | func (t *Timer) Update(value float64) { 60 | for i := range t.channels { 61 | t.channels[i] <- value 62 | } 63 | } 64 | 65 | func (t *Timer) save(retention Retention, now time.Time) { 66 | values := t.Values[retention.Index] 67 | t.Values[retention.Index] = make([]float64, 0) 68 | if len(values) == 0 { 69 | return 70 | } 71 | 72 | go func() { 73 | timestamp := now.Unix() - now.Unix()%retention.Interval 74 | sort.Float64s(values) 75 | count := len(values) 76 | min := SortedMin(values) 77 | max := SortedMax(values) 78 | median := SortedMedian(values) 79 | mean := Mean(values) 80 | stddev := Stddev(values) 81 | percentile_90 := SortedPercentile(values, 0.9) 82 | percentile_95 := SortedPercentile(values, 0.95) 83 | percentile_99 := SortedPercentile(values, 0.99) 84 | aggregates := fmt.Sprintf("%v/%v/%v/%v/%v/%v/%v/%v/%v", count, min, max, median, mean, stddev, percentile_90, percentile_95, percentile_99) 85 | 86 | if retention.Index == 0 { 87 | observation := AggregateObservation{Name: "timers:" + t.Key, Content: fmt.Sprintf("%d%v", timestamp, aggregates), Timestamp: timestamp, RawName: "timers:" + t.Key, Path: ""} 88 | StoreInRedis(observation) 89 | } else { 90 | observation := AggregateObservation{Name: "timers:" + t.Key + ":" + strconv.FormatInt(retention.Interval, 10) + ":" + timerVersion, 91 | Content: fmt.Sprintf("%d %v\n", timestamp, aggregates), Timestamp: timestamp, RawName: "timers:" + t.Key, Path: t.Paths[retention.Index], 92 | SummaryValues: map[string]float64{"count": float64(count), "min": min, "max": max, "median": median, "mean": mean, "stddev": stddev, "percentile_90": percentile_90, "percentile_95": percentile_95, "percentile_99": percentile_99}, Interval: retention.Interval} 93 | StoreOnDisk(observation) 94 | } 95 | }() 96 | } 97 | -------------------------------------------------------------------------------- /gobatsd/timers_test.go: -------------------------------------------------------------------------------- 1 | package gobatsd 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestTimerUpdate(t *testing.T) { 10 | Config.Retentions = []Retention{{10, 10, 100, 0}} 11 | Config.RedisHost = "127.0.0.1" 12 | Config.RedisPort = 6379 13 | SetupDatastore() 14 | 15 | timer := NewTimer("test").(*Timer) 16 | timer.Start() 17 | timer.Update(8.6754) 18 | time.Sleep(1 * time.Millisecond) 19 | if timer.Values[0][0] != 8.6754 || len(timer.Values[0]) != 1 { 20 | t.Errorf("Expected timer to update") 21 | } 22 | timer.save(Config.Retentions[0], time.Now()) 23 | } 24 | 25 | func BenchmarkTimerUpdate(b *testing.B) { 26 | Config.Retentions = []Retention{{10, 10, 100, 0}} 27 | Config.RedisHost = "127.0.0.1" 28 | Config.RedisPort = 6379 29 | SetupDatastore() 30 | 31 | timer := NewTimer("test").(*Timer) 32 | timer.Start() 33 | 34 | for j := 0; j < b.N; j++ { 35 | timer.Update(8.6754) 36 | } 37 | } 38 | 39 | func TestTimerSave(t *testing.T) { 40 | Config.Retentions = []Retention{{10, 10, 100, 0}, {20, 10, 200, 1}} 41 | Config.RedisHost = "127.0.0.1" 42 | Config.RedisPort = 6379 43 | SetupDatastore() 44 | 45 | timer := NewTimer("testttttt").(*Timer) 46 | timer.Values[0] = []float64{1, 2, 3, 4, 5} 47 | now := time.Now() 48 | go func() { 49 | obs := <-datastore.redisChannel 50 | expected := fmt.Sprintf("%v%v", now.Unix()-now.Unix()%10, "5/1/5/3/3/1.5811388300841898/5/5/5") 51 | if obs.Content != expected { 52 | t.Errorf("Timer send to redis was not properly structured, got '%v' expected '%v'", obs.Content, expected) 53 | } 54 | }() 55 | timer.save(Config.Retentions[0], now) 56 | timer.Values[1] = []float64{1, 2, 3, 4, 5} 57 | go func() { 58 | obs := <-datastore.diskChannel 59 | expected := fmt.Sprintf("%v %v\n", now.Unix()-now.Unix()%20, "5/1/5/3/3/1.5811388300841898/5/5/5") 60 | if obs.Content != expected { 61 | t.Errorf("Timer send to disk was not properly structured, got '%v' expected '%v'", obs.Content, expected) 62 | } 63 | }() 64 | timer.save(Config.Retentions[1], now) 65 | 66 | } 67 | -------------------------------------------------------------------------------- /hbase_migration/counters_export.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "github.com/noahhl/Go-Redis" 7 | "github.com/noahhl/go-batsd/gobatsd" 8 | "io" 9 | "os" 10 | "strconv" 11 | "strings" 12 | ) 13 | 14 | var timerHeader = map[string]int{"count": 0, "min": 1, "max": 2, "median": 3, "mean": 4, 15 | "stddev": 5, "percentile_90": 6, "percentile_95": 7, "percentile_99": 8} 16 | 17 | var retentions = []int64{60} 18 | 19 | func main() { 20 | gobatsd.LoadConfig() 21 | 22 | spec := redis.DefaultSpec().Host(gobatsd.Config.RedisHost).Port(gobatsd.Config.RedisPort) 23 | redis, redisErr := redis.NewSynchClientWithSpec(spec) 24 | if redisErr != nil { 25 | panic(redisErr) 26 | } 27 | datapoints, redisErr := redis.Smembers("datapoints") 28 | if redisErr != nil { 29 | panic(redisErr) 30 | } 31 | 32 | for j := range retentions { 33 | outfile, err := os.Create(fmt.Sprintf("counters_export_%v.tsv", retentions[j])) 34 | if err != nil { 35 | panic(err) 36 | } 37 | 38 | writer := bufio.NewWriter(outfile) 39 | for i := range datapoints { 40 | if datapoints[i][0] == 'c' { 41 | path := gobatsd.CalculateFilename(fmt.Sprintf("%v:%v", string(datapoints[i]), retentions[j]), gobatsd.Config.Root) 42 | fmt.Printf("(%v / %v) Migrating %v:%v from %v\n", i, len(datapoints), string(datapoints[i]), retentions[j], path) 43 | file, err := os.Open(path) 44 | if err == nil { 45 | reader := bufio.NewReader(file) 46 | linesRead := 0 47 | for { 48 | line, err := reader.ReadString('\n') 49 | linesRead += 1 50 | if err != nil && err != io.EOF { 51 | panic(err) 52 | } 53 | if err != nil && err == io.EOF { 54 | break 55 | } 56 | if linesRead == 1 { 57 | continue 58 | } 59 | 60 | parts := strings.Split(strings.TrimSpace(line), " ") 61 | value, _ := strconv.ParseFloat(parts[1], 64) 62 | writer.WriteString(fmt.Sprintf("%v\t%v\t%s\n", string(datapoints[i]), parts[0], gobatsd.EncodeFloat64(value))) 63 | } 64 | } else { 65 | fmt.Printf("%v\n", err) 66 | } 67 | file.Close() 68 | 69 | writer.Flush() 70 | } 71 | 72 | } 73 | outfile.Close() 74 | fmt.Printf("Import with: -Dimporttsv.columns=HBASE_ROW_KEY,HBASE_TS_KEY,interval%v:value\n", retentions[j]) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /hbase_migration/gauges_export.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "github.com/noahhl/Go-Redis" 7 | "github.com/noahhl/go-batsd/gobatsd" 8 | "io" 9 | "os" 10 | "strconv" 11 | "strings" 12 | ) 13 | 14 | const yearago = 1398249561 15 | 16 | var maxPerFile = 100000 17 | 18 | func main() { 19 | gobatsd.LoadConfig() 20 | 21 | spec := redis.DefaultSpec().Host(gobatsd.Config.RedisHost).Port(gobatsd.Config.RedisPort) 22 | redis, redisErr := redis.NewSynchClientWithSpec(spec) 23 | if redisErr != nil { 24 | panic(redisErr) 25 | } 26 | datapoints, redisErr := redis.Smembers("datapoints") 27 | if redisErr != nil { 28 | panic(redisErr) 29 | } 30 | 31 | fileNo := 1 32 | outfile, err := os.Create(fmt.Sprintf("gauges_export-%v.tsv", fileNo)) 33 | if err != nil { 34 | panic(err) 35 | } 36 | 37 | writer := bufio.NewWriter(outfile) 38 | for i := range datapoints { 39 | if i == fileNo*maxPerFile { 40 | outfile.Close() 41 | fileNo += 1 42 | outfile, err = os.Create(fmt.Sprintf("gauges_export-%v.tsv", fileNo)) 43 | if err != nil { 44 | panic(err) 45 | } 46 | writer = bufio.NewWriter(outfile) 47 | } 48 | if datapoints[i][0] == 'g' && (len(datapoints[i]) < 18 || string(datapoints[i][0:17]) != "gauges:Syslog-NG.") { 49 | path := gobatsd.CalculateFilename(string(datapoints[i]), gobatsd.Config.Root) 50 | fmt.Printf("(%v / %v) Migrating %v from %v\n", i, len(datapoints), string(datapoints[i]), path) 51 | file, err := os.Open(path) 52 | if err == nil { 53 | reader := bufio.NewReader(file) 54 | linesRead := 0 55 | for { 56 | line, err := reader.ReadString('\n') 57 | linesRead += 1 58 | if err != nil && err != io.EOF { 59 | fmt.Println(err) 60 | } 61 | if err != nil && err == io.EOF { 62 | break 63 | } 64 | if linesRead == 1 { 65 | continue 66 | } 67 | 68 | parts := strings.Split(strings.TrimSpace(line), " ") 69 | ts, _ := strconv.ParseInt(parts[0], 10, 64) 70 | if len(parts) == 2 { 71 | value, _ := strconv.ParseFloat(parts[1], 64) 72 | if value != float64(0) && ts > yearago { 73 | _, err := writer.WriteString(fmt.Sprintf("%v\t%v\t%s\n", string(datapoints[i]), parts[0], gobatsd.EncodeFloat64(value))) 74 | if err != nil { 75 | fmt.Println(err) 76 | outfile.Close() 77 | fileNo += 1 78 | outfile, err = os.Create(fmt.Sprintf("sc_gauges_export-%v.tsv", fileNo)) 79 | if err != nil { 80 | panic(err) 81 | } 82 | writer = bufio.NewWriter(outfile) 83 | 84 | } 85 | } 86 | } 87 | } 88 | } else { 89 | fmt.Printf("%v\n", err) 90 | } 91 | file.Close() 92 | 93 | writer.Flush() 94 | } 95 | 96 | } 97 | outfile.Close() 98 | fmt.Printf("Import with: -Dimporttsv.columns=HBASE_ROW_KEY,HBASE_TS_KEY,interval0:value\n") 99 | 100 | } 101 | -------------------------------------------------------------------------------- /hbase_migration/migrate_to_hbase.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "github.com/jinntrance/goh" 7 | "github.com/jinntrance/goh/Hbase" 8 | "github.com/noahhl/Go-Redis" 9 | "github.com/noahhl/go-batsd/gobatsd" 10 | "io" 11 | "os" 12 | "runtime" 13 | "strconv" 14 | "strings" 15 | "time" 16 | ) 17 | 18 | var timerHeader = map[string]int{"count": 0, "min": 1, "max": 2, "median": 3, "mean": 4, 19 | "stddev": 5, "percentile_90": 6, "percentile_95": 7, "percentile_99": 8} 20 | 21 | var retentions = []int64{600} 22 | 23 | const yearago = 1398249561 24 | 25 | func main() { 26 | gobatsd.LoadConfig() 27 | 28 | runtime.GOMAXPROCS(runtime.NumCPU()) 29 | spec := redis.DefaultSpec().Host(gobatsd.Config.RedisHost).Port(gobatsd.Config.RedisPort) 30 | redis, redisErr := redis.NewSynchClientWithSpec(spec) 31 | if redisErr != nil { 32 | panic(redisErr) 33 | } 34 | datapoints, redisErr := redis.Smembers("datapoints") 35 | if redisErr != nil { 36 | panic(redisErr) 37 | } 38 | nworkers := 10 39 | c := make(chan int, nworkers) 40 | 41 | for k := 0; k < nworkers; k++ { 42 | go func(datapoints [][]byte, c chan int) { 43 | cli, err := gobatsd.OpenHbaseConnection() 44 | if err != nil { 45 | panic(err) 46 | } 47 | hbaseClient := cli.(*goh.HClient) 48 | defer hbaseClient.Close() 49 | 50 | for i := range datapoints { 51 | if datapoints[i][0] == 'g' && (len(datapoints[i]) < 18 || string(datapoints[i][0:17]) != "gauges:Syslog-NG.") { 52 | path := gobatsd.CalculateFilename(string(datapoints[i]), gobatsd.Config.Root) 53 | fmt.Printf("(%v / %v) Migrating %v from %v\n", i, len(datapoints), string(datapoints[i]), path) 54 | file, err := os.Open(path) 55 | if err == nil { 56 | reader := bufio.NewReader(file) 57 | linesRead := 0 58 | for { 59 | line, err := reader.ReadString('\n') 60 | linesRead += 1 61 | if err != nil && err != io.EOF { 62 | panic(err) 63 | } 64 | if err != nil && err == io.EOF { 65 | break 66 | } 67 | if linesRead == 1 { 68 | continue 69 | } 70 | 71 | parts := strings.Split(strings.TrimSpace(line), " ") 72 | ts, _ := strconv.ParseInt(parts[0], 10, 64) 73 | value, _ := strconv.ParseFloat(parts[1], 64) 74 | if value != float64(0) && ts > yearago { 75 | mutations := []*Hbase.Mutation{goh.NewMutation("interval0:value", gobatsd.EncodeFloat64(value))} 76 | 77 | err = hbaseClient.MutateRowTs(gobatsd.Config.HbaseTable, datapoints[i], mutations, ts, nil) 78 | if err != nil { 79 | fmt.Printf("%v: mutate error - %v\n", time.Now(), err) 80 | } 81 | } 82 | } 83 | } else { 84 | fmt.Printf("%v\n", err) 85 | } 86 | /*} else if datapoints[i][0] == 'c' { 87 | for j := range retentions { 88 | path := gobatsd.CalculateFilename(fmt.Sprintf("%v:%v", string(datapoints[i]), retentions[j]), gobatsd.Config.Root) 89 | fmt.Printf("Migrating %v:%v from %v\n", string(datapoints[i]), retentions[j], path) 90 | file, err := os.Open(path) 91 | if err == nil { 92 | reader := bufio.NewReader(file) 93 | linesRead := 0 94 | for { 95 | line, err := reader.ReadString('\n') 96 | linesRead += 1 97 | if err != nil && err != io.EOF { 98 | panic(err) 99 | } 100 | if err != nil && err == io.EOF { 101 | break 102 | } 103 | if linesRead == 1 { 104 | continue 105 | } 106 | 107 | parts := strings.Split(strings.TrimSpace(line), " ") 108 | ts, _ := strconv.ParseInt(parts[0], 10, 64) 109 | value, _ := strconv.ParseFloat(parts[1], 64) 110 | 111 | mutations := []*Hbase.Mutation{goh.NewMutation(fmt.Sprintf("interval%v:value", retentions[j]), gobatsd.EncodeFloat64(value))} 112 | 113 | err = hbaseClient.MutateRowTs(gobatsd.Config.HbaseTable, datapoints[i], mutations, ts, nil) 114 | if err != nil { 115 | err = hbaseClient.MutateRowTs(gobatsd.Config.HbaseTable, datapoints[i], mutations, ts, nil) 116 | if err != nil { 117 | fmt.Printf("%v: mutate error - %v\n", time.Now(), err) 118 | } 119 | } 120 | } 121 | } else { 122 | fmt.Printf("%v\n", err) 123 | } 124 | 125 | }*/ 126 | } else if datapoints[i][0] == 't' && (len(datapoints[i]) < 16 || string(datapoints[i][0:15]) != "timers:sysstat.") { 127 | for j := range retentions { 128 | path := gobatsd.CalculateFilename(fmt.Sprintf("%v:%v:2", string(datapoints[i]), retentions[j]), gobatsd.Config.Root) 129 | fmt.Printf("(%v / %v) Migrating %v:%v from %v\n", i, len(datapoints), string(datapoints[i]), retentions[j], path) 130 | file, err := os.Open(path) 131 | if err == nil { 132 | reader := bufio.NewReader(file) 133 | linesRead := 0 134 | for { 135 | line, err := reader.ReadString('\n') 136 | linesRead += 1 137 | if err != nil && err != io.EOF { 138 | panic(err) 139 | } 140 | if err != nil && err == io.EOF { 141 | break 142 | } 143 | if linesRead == 1 { 144 | continue 145 | } 146 | 147 | parts := strings.Split(strings.TrimSpace(line), " ") 148 | ts, _ := strconv.ParseInt(parts[0], 10, 64) 149 | timerComponents := strings.Split(parts[1], "/") 150 | 151 | if ts > yearago { 152 | mutations := make([]*Hbase.Mutation, 0) 153 | for k, v := range timerHeader { 154 | val, _ := strconv.ParseFloat(timerComponents[v], 64) 155 | mutations = append(mutations, goh.NewMutation(fmt.Sprintf("interval%v:%v", retentions[j], k), gobatsd.EncodeFloat64(val))) 156 | } 157 | err = hbaseClient.MutateRowTs(gobatsd.Config.HbaseTable, datapoints[i], mutations, ts, nil) 158 | if err != nil { 159 | fmt.Printf("%v: mutate error - %v\n", time.Now(), err) 160 | } 161 | } 162 | } 163 | } else { 164 | fmt.Printf("%v\n", err) 165 | } 166 | } 167 | 168 | } 169 | 170 | } 171 | c <- 1 172 | }(datapoints[k*len(datapoints)/nworkers:(k+1)*len(datapoints)/nworkers-1], c) 173 | } 174 | 175 | for i := 0; i < nworkers; i++ { 176 | <-c 177 | } 178 | } 179 | -------------------------------------------------------------------------------- /hbase_migration/timers_export.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "github.com/noahhl/Go-Redis" 7 | "github.com/noahhl/go-batsd/gobatsd" 8 | "io" 9 | "os" 10 | "strconv" 11 | "strings" 12 | ) 13 | 14 | var retentions = []int64{60} 15 | var maxPerFile = 100000 16 | 17 | const yearago = 1398249561 18 | 19 | func main() { 20 | gobatsd.LoadConfig() 21 | 22 | spec := redis.DefaultSpec().Host(gobatsd.Config.RedisHost).Port(gobatsd.Config.RedisPort) 23 | redis, redisErr := redis.NewSynchClientWithSpec(spec) 24 | if redisErr != nil { 25 | panic(redisErr) 26 | } 27 | datapoints, redisErr := redis.Smembers("datapoints") 28 | if redisErr != nil { 29 | panic(redisErr) 30 | } 31 | 32 | for j := range retentions { 33 | fileNo := 1 34 | outfile, err := os.Create(fmt.Sprintf("timers_export_%v-%v.tsv", retentions[j], fileNo)) 35 | if err != nil { 36 | panic(err) 37 | } 38 | 39 | writer := bufio.NewWriter(outfile) 40 | for i := range datapoints { 41 | if i == fileNo*maxPerFile { 42 | outfile.Close() 43 | fileNo += 1 44 | outfile, err = os.Create(fmt.Sprintf("timers_export_%v-%v.tsv", retentions[j], fileNo)) 45 | if err != nil { 46 | panic(err) 47 | } 48 | writer = bufio.NewWriter(outfile) 49 | } 50 | if datapoints[i][0] == 't' && (len(datapoints[i]) < 16 || string(datapoints[i][0:15]) != "timers:sysstat.") { 51 | path := gobatsd.CalculateFilename(fmt.Sprintf("%v:%v:2", string(datapoints[i]), retentions[j]), gobatsd.Config.Root) 52 | fmt.Printf("(%v / %v) Migrating %v:%v from %v\n", i, len(datapoints), string(datapoints[i]), retentions[j], path) 53 | file, err := os.Open(path) 54 | if err == nil { 55 | reader := bufio.NewReader(file) 56 | linesRead := 0 57 | for { 58 | line, err := reader.ReadString('\n') 59 | linesRead += 1 60 | if err != nil && err != io.EOF { 61 | panic(err) 62 | } 63 | if err != nil && err == io.EOF { 64 | break 65 | } 66 | if linesRead == 1 { 67 | continue 68 | } 69 | 70 | parts := strings.Split(strings.TrimSpace(line), " ") 71 | ts, _ := strconv.ParseInt(parts[0], 10, 64) 72 | if ts > 1 { 73 | timerComponents := strings.Split(parts[1], "/") 74 | count, _ := strconv.ParseFloat(timerComponents[0], 64) 75 | min, _ := strconv.ParseFloat(timerComponents[1], 64) 76 | max, _ := strconv.ParseFloat(timerComponents[2], 64) 77 | median, _ := strconv.ParseFloat(timerComponents[3], 64) 78 | mean, _ := strconv.ParseFloat(timerComponents[4], 64) 79 | stddev, _ := strconv.ParseFloat(timerComponents[5], 64) 80 | percentile_90, _ := strconv.ParseFloat(timerComponents[6], 64) 81 | percentile_95, _ := strconv.ParseFloat(timerComponents[7], 64) 82 | percentile_99, _ := strconv.ParseFloat(timerComponents[8], 64) 83 | 84 | writer.WriteString(fmt.Sprintf("%v\t%v\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", string(datapoints[i]), parts[0], 85 | gobatsd.EncodeFloat64(count), gobatsd.EncodeFloat64(min), gobatsd.EncodeFloat64(max), gobatsd.EncodeFloat64(median), 86 | gobatsd.EncodeFloat64(mean), gobatsd.EncodeFloat64(stddev), gobatsd.EncodeFloat64(percentile_90), gobatsd.EncodeFloat64(percentile_95), 87 | gobatsd.EncodeFloat64(percentile_99))) 88 | } 89 | } 90 | } else { 91 | fmt.Printf("%v\n", err) 92 | } 93 | file.Close() 94 | 95 | writer.Flush() 96 | } 97 | 98 | } 99 | outfile.Close() 100 | fmt.Printf("Import with: -Dimporttsv.columns=HBASE_ROW_KEY,HBASE_TS_KEY,interval%v:count\n", retentions[j]) 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /proxy.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/noahhl/go-batsd/gobatsd" 6 | "net" 7 | "os" 8 | "runtime/pprof" 9 | ) 10 | 11 | const bufferLen = 100000 12 | const readLen = 256 13 | 14 | func main() { 15 | prof, err := os.Create("/tmp/proxy.prof") 16 | if err != nil { 17 | panic(err) 18 | } 19 | defer prof.Close() 20 | pprof.StartCPUProfile(prof) 21 | defer pprof.StopCPUProfile() 22 | 23 | gobatsd.LoadConfig() 24 | fmt.Printf("Starting on port %v\n", gobatsd.Config.Port) 25 | 26 | server, err := net.ListenPacket("udp", ":"+gobatsd.Config.Port) 27 | defer server.Close() 28 | if err != nil { 29 | panic(err) 30 | } 31 | 32 | destinationAddr, err := net.ResolveUDPAddr("udp", ":8225") 33 | if err != nil { 34 | panic(err) 35 | } 36 | client, err := net.DialUDP("udp", nil, destinationAddr) 37 | if err != nil { 38 | panic(err) 39 | } 40 | 41 | //c := TCPRelay(client) 42 | buffer := make([]byte, readLen) 43 | 44 | for { 45 | n, _, err := server.ReadFrom(buffer) 46 | if err != nil { 47 | continue 48 | } 49 | client.Write(buffer[0:n]) 50 | // c <- buffer[0:n] 51 | 52 | } 53 | } 54 | 55 | func TCPRelay(client net.Conn) chan []byte { 56 | c := make(chan []byte, bufferLen) 57 | 58 | go func(client net.Conn, c chan []byte) { 59 | for { 60 | data := <-c 61 | client.Write(data) 62 | } 63 | }(client, c) 64 | 65 | return c 66 | } 67 | -------------------------------------------------------------------------------- /receiver.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/noahhl/clamp" 5 | "github.com/noahhl/go-batsd/gobatsd" 6 | 7 | "fmt" 8 | "runtime" 9 | "time" 10 | 11 | "os" 12 | "os/signal" 13 | "runtime/pprof" 14 | ) 15 | 16 | var counterChannel chan gobatsd.Datapoint 17 | var gaugeChannel chan gobatsd.Datapoint 18 | var timerChannel chan gobatsd.Datapoint 19 | 20 | const channelBufferSize = 4000000 21 | const numIncomingMessageProcessors = 20 22 | 23 | func main() { 24 | runtime.GOMAXPROCS(runtime.NumCPU()) 25 | gobatsd.LoadConfig() 26 | if gobatsd.ProfileCPU { 27 | cpuprof, err := os.Create(fmt.Sprintf("cpuprof-%v", time.Now().Unix())) 28 | if err != nil { 29 | panic(err) 30 | } 31 | defer cpuprof.Close() 32 | pprof.StartCPUProfile(cpuprof) 33 | defer pprof.StopCPUProfile() 34 | } 35 | 36 | processingChannel := clamp.StartExplodingDualServer(":" + gobatsd.Config.Port) 37 | clamp.StartStatsServer(":8124") 38 | gobatsd.SetupDatastore() 39 | 40 | fmt.Printf("Starting on port %v\n", gobatsd.Config.Port) 41 | gobatsd.InitializeInternalMetrics() 42 | 43 | gaugeChannel = make(chan gobatsd.Datapoint, channelBufferSize) 44 | counterChannel = make(chan gobatsd.Datapoint, channelBufferSize) 45 | timerChannel = make(chan gobatsd.Datapoint, channelBufferSize) 46 | 47 | channels := map[string]chan gobatsd.Datapoint{"g": gaugeChannel, "c": counterChannel, "ms": timerChannel} 48 | 49 | for i := 0; i < numIncomingMessageProcessors; i++ { 50 | go func(processingChannel chan string) { 51 | for { 52 | message := <-processingChannel 53 | d := gobatsd.ParseDatapointFromString(message) 54 | if ch, ok := channels[d.Datatype]; ok { 55 | ch <- d 56 | } 57 | } 58 | }(processingChannel) 59 | } 60 | 61 | go func() { 62 | c := time.Tick(1 * time.Second) 63 | for { 64 | <-c 65 | clamp.StatsChannel <- clamp.Stat{"gaugeChannelSize", fmt.Sprintf("%v", len(gaugeChannel))} 66 | clamp.StatsChannel <- clamp.Stat{"counterChannelSize", fmt.Sprintf("%v", len(counterChannel))} 67 | clamp.StatsChannel <- clamp.Stat{"timerChannelSize", fmt.Sprintf("%v", len(timerChannel))} 68 | } 69 | }() 70 | 71 | processDatatype("gauges", gaugeChannel, gobatsd.NewGauge) 72 | processDatatype("timers", timerChannel, gobatsd.NewTimer) 73 | processDatatype("counters", counterChannel, gobatsd.NewCounter) 74 | 75 | terminate := make(chan os.Signal) 76 | signal.Notify(terminate, os.Interrupt) 77 | <-terminate 78 | 79 | fmt.Printf("Server stopped") 80 | 81 | } 82 | 83 | func processDatatype(datatypeName string, ch chan gobatsd.Datapoint, metricCreator func(string) gobatsd.Metric) { 84 | metrics := make(map[string]gobatsd.Metric) 85 | go func() { 86 | c := time.Tick(1 * time.Second) 87 | for { 88 | <-c 89 | clamp.StatsChannel <- clamp.Stat{datatypeName, fmt.Sprintf("%v", len(metrics))} 90 | gobatsd.InternalMetrics[datatypeName+"Known"].Update(float64(len(metrics))) 91 | } 92 | }() 93 | go func() { 94 | for { 95 | select { 96 | case d := <-ch: 97 | if m, ok := metrics[d.Name]; ok { 98 | m.Update(d.Value) 99 | } else { 100 | m := metricCreator(d.Name) 101 | metrics[d.Name] = m 102 | m.Update(d.Value) 103 | } 104 | gobatsd.InternalMetrics[datatypeName+"Processed"].Update(1) 105 | } 106 | } 107 | }() 108 | 109 | } 110 | -------------------------------------------------------------------------------- /server.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/noahhl/clamp" 5 | "github.com/noahhl/go-batsd/gobatsd" 6 | 7 | "bufio" 8 | "encoding/json" 9 | "fmt" 10 | "github.com/garyburd/redigo/redis" 11 | "github.com/jinntrance/goh" 12 | "github.com/jinntrance/goh/Hbase" 13 | "io" 14 | "net" 15 | "os" 16 | "sort" 17 | "strconv" 18 | "strings" 19 | "time" 20 | ) 21 | 22 | type Client struct { 23 | net.Conn 24 | redis redis.Conn 25 | } 26 | 27 | var timerHeader = map[string]int{"count": 0, "min": 1, "max": 2, "median": 3, "mean": 4, 28 | "stddev": 5, "percentile_90": 6, "percentile_95": 7, "percentile_99": 8, 29 | "upper_90": 6, "upper_95": 7, "upper_99": 8} 30 | 31 | var hbasePool *clamp.ConnectionPoolWrapper 32 | 33 | func main() { 34 | gobatsd.LoadConfig() 35 | fmt.Printf("Starting on port %v, root dir %v\n", gobatsd.Config.Port, gobatsd.Config.Root) 36 | 37 | server, err := net.Listen("tcp", ":"+gobatsd.Config.Port) 38 | if err != nil { 39 | panic(err) 40 | } 41 | numClients := 0 42 | 43 | hbasePool = gobatsd.MakeHbasePool(10) 44 | 45 | for { 46 | client, err := server.Accept() 47 | if client == nil { 48 | fmt.Printf("couldn't accept: %v", err) 49 | continue 50 | } 51 | numClients++ 52 | fmt.Printf("Opened connection #%d: %v <-> %v\n", numClients, client.LocalAddr(), client.RemoteAddr()) 53 | 54 | go func() { 55 | r, err := redis.Dial("tcp", fmt.Sprintf("%v:%v", gobatsd.Config.RedisHost, gobatsd.Config.RedisPort)) 56 | if err != nil { 57 | panic(err) 58 | } 59 | defer r.Close() 60 | 61 | (&Client{client, r}).Serve() 62 | }() 63 | } 64 | } 65 | 66 | func (c *Client) Serve() { 67 | for { 68 | b := bufio.NewReader(c) 69 | line, err := b.ReadBytes('\n') 70 | if err != nil { 71 | break 72 | } 73 | components := strings.Split(strings.TrimSpace(string(line)), " ") 74 | command := strings.ToLower(components[0]) 75 | switch command { 76 | case "ping": 77 | c.Write([]byte("PONG\n")) 78 | case "quit": 79 | c.Write([]byte("BYE\n")) 80 | c.Close() 81 | case "available": 82 | c.SendAvailableMetrics() 83 | case "values": 84 | c.SendValues(components[1:len(components)]) 85 | default: 86 | c.Write([]byte("Unrecognized command: " + command + "\n")) 87 | } 88 | } 89 | } 90 | 91 | func (c *Client) SendAvailableMetrics() { 92 | available := make([]string, 0) 93 | smembers, err := c.redis.Do("SMEMBERS", "datapoints") 94 | 95 | if err == nil { 96 | stringSmembers, _ := redis.Strings(smembers, err) 97 | for i := range stringSmembers { 98 | available = append(available, stringSmembers[i]) 99 | } 100 | } 101 | json, _ := json.Marshal(available) 102 | c.Write(append(json, '\n')) 103 | } 104 | 105 | func (c *Client) SendValues(properties []string) { 106 | if len(properties) < 3 { 107 | c.Write([]byte("Invalid arguments\n")) 108 | return 109 | } 110 | metric := properties[0] 111 | start_ts, _ := strconv.ParseFloat(properties[1], 64) 112 | end_ts, _ := strconv.ParseFloat(properties[2], 64) 113 | version := ":2" 114 | if len(properties) == 4 && properties[3] == "1" { 115 | version = "" 116 | } 117 | 118 | datatype := strings.Split(metric, ":")[0] 119 | now := time.Now().Unix() 120 | retentionIndex := sort.Search(len(gobatsd.Config.Retentions), func(i int) bool { return gobatsd.Config.Retentions[i].Duration > (now - int64(start_ts)) }) 121 | if retentionIndex == len(gobatsd.Config.Retentions) { 122 | retentionIndex = len(gobatsd.Config.Retentions) - 1 123 | } 124 | retention := gobatsd.Config.Retentions[retentionIndex] 125 | switch datatype { 126 | case "gauges": 127 | c.SendValuesFromHbase(metric, 0, start_ts, end_ts, "value") 128 | case "timers": 129 | pieces := strings.Split(metric, ":") 130 | if len(pieces) >= 3 { 131 | if retention.Index == 0 { 132 | c.SendValuesFromRedis("timer", "timers:"+pieces[1], start_ts, end_ts, version, pieces[2]) 133 | } else { 134 | c.SendValuesFromHbase("timers:"+pieces[1], retention.Interval, start_ts, end_ts, pieces[2]) 135 | } 136 | } else { 137 | c.Write([]byte("[]\n")) 138 | } 139 | 140 | case "counters": 141 | if retention.Index == 0 { 142 | c.SendValuesFromRedis("counter", metric, start_ts, end_ts, version, "") 143 | } else { 144 | c.SendValuesFromHbase(metric, retention.Interval, start_ts, end_ts, "value") 145 | } 146 | 147 | default: 148 | c.Write([]byte("Unrecognized datatype\n")) 149 | } 150 | } 151 | 152 | func (c *Client) SendValuesFromRedis(datatype string, keyname string, start_ts float64, end_ts float64, 153 | version string, operation string) { 154 | 155 | values := make([]map[string]string, 0) 156 | v, redisErr := c.redis.Do("ZRANGEBYSCORE", keyname, start_ts, end_ts) 157 | if redisErr == nil { 158 | stringValues, _ := redis.Strings(v, redisErr) 159 | for i := range stringValues { 160 | parts := strings.Split(stringValues[i], "") 161 | if datatype == "counter" { 162 | values = append(values, map[string]string{"Timestamp": parts[0], "Value": parts[1]}) 163 | } else if datatype == "timer" { 164 | if headerIndex, ok := timerHeader[operation]; ok { 165 | values = append(values, map[string]string{"Timestamp": parts[0], "Value": strings.Split(parts[1], "/")[headerIndex]}) 166 | } 167 | } 168 | 169 | } 170 | } else { 171 | fmt.Printf("%v\n", redisErr) 172 | } 173 | json := gobatsd.ArtisinallyMarshallDatapointJSON(values) 174 | c.Write(append(json, '\n')) 175 | } 176 | 177 | func (c *Client) SendValuesFromDisk(datatype string, path string, start_ts float64, end_ts float64, 178 | version string, operation string) { 179 | file, err := os.Open(path) 180 | values := make([]map[string]string, 0) 181 | if err == nil { 182 | reader := bufio.NewReader(file) 183 | linesRead := 0 184 | for { 185 | line, err := reader.ReadString('\n') 186 | linesRead += 1 187 | if err != nil && err != io.EOF { 188 | panic(err) 189 | } 190 | if err != nil && err == io.EOF { 191 | break 192 | } 193 | //skip the header in v2 files 194 | if linesRead == 1 && version == ":2" { 195 | continue 196 | } 197 | 198 | parts := strings.Split(strings.TrimSpace(line), " ") 199 | ts, _ := strconv.ParseFloat(parts[0], 64) 200 | if ts >= start_ts && ts <= end_ts { 201 | if datatype == "gauge" || datatype == "counter" { 202 | values = append(values, map[string]string{"Timestamp": parts[0], "Value": parts[1]}) 203 | } else if datatype == "timer" { 204 | if headerIndex, ok := timerHeader[operation]; ok { 205 | values = append(values, map[string]string{"Timestamp": parts[0], "Value": strings.Split(parts[1], "/")[headerIndex]}) 206 | } 207 | } 208 | } 209 | if ts > end_ts { 210 | break 211 | } 212 | } 213 | } else { 214 | fmt.Printf("%v\n", err) 215 | } 216 | json := gobatsd.ArtisinallyMarshallDatapointJSON(values) 217 | c.Write(append(json, '\n')) 218 | 219 | } 220 | 221 | func (c *Client) SendValuesFromHbase(keyname string, retentionInterval int64, start_ts float64, end_ts float64, operation string) { 222 | start := time.Now().UnixNano() 223 | hbaseClient := hbasePool.GetConnection().(*goh.HClient) 224 | 225 | var result []*Hbase.TCell 226 | var err error 227 | if keyname[0] == 'g' { //this is a gauge, so no way to know how many of them there are. get a bunch 228 | result, err = hbaseClient.GetVerTs(gobatsd.Config.HbaseTable, []byte(keyname), fmt.Sprintf("interval%v:%v", retentionInterval, operation), int64(end_ts), 1000000, nil) 229 | } else { 230 | result, err = hbaseClient.GetVerTs(gobatsd.Config.HbaseTable, []byte(keyname), fmt.Sprintf("interval%v:%v", retentionInterval, operation), int64(end_ts), int32(int64(end_ts-start_ts)/retentionInterval), nil) 231 | } 232 | if err != nil { 233 | fmt.Printf("%v\n", err) 234 | } 235 | values := make([]map[string]string, 0) 236 | for i := range result { 237 | 238 | decodedVal, err := gobatsd.DecodeFloat64(result[i].Value) 239 | if err == nil { 240 | values = append(values, map[string]string{"Timestamp": strconv.FormatInt(result[i].Timestamp, 10), "Value": strconv.FormatFloat(decodedVal, 'f', -1, 64)}) 241 | } 242 | } 243 | fmt.Printf("Hbase completed request in %v ms, retrieved %v records\n", (time.Now().UnixNano()-start)/int64(time.Millisecond), len(values)) 244 | 245 | json := gobatsd.ArtisinallyMarshallDatapointJSON(values) 246 | c.Write(append(json, '\n')) 247 | hbasePool.ReleaseConnection(hbaseClient) 248 | fmt.Printf("Total time to respond to client: %v ms\n", (time.Now().UnixNano()-start)/int64(time.Millisecond)) 249 | } 250 | -------------------------------------------------------------------------------- /test/fake_data: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require 'rubygems' 3 | require 'optparse' 4 | require 'socket' 5 | require 'benchmark' 6 | 7 | options = {:host => "127.0.0.1", :port => 8125, :types => []} 8 | parser = OptionParser.new do |opts| 9 | opts.banner = "Usage: fake_data [options] " 10 | 11 | opts.separator "" 12 | opts.separator "options:" 13 | 14 | opts.on("-sSERVER", "--server SERVER", "Statsd server host:port") do |x| 15 | options[:host] = x.split(":")[0] 16 | options[:port] = x.split(":")[1].to_i 17 | end 18 | 19 | opts.on("-c", "--counters", "Include counters") do 20 | options[:types] << :counters 21 | end 22 | 23 | opts.on("-t", "--timers", "Include timers") do 24 | options[:types] << :timers 25 | end 26 | 27 | opts.on("-g", "--gauges", "Include gauges") do 28 | options[:types] << :gauges 29 | end 30 | 31 | opts.on("-h", "--help", "Show this message") do 32 | puts opts 33 | exit 34 | end 35 | 36 | end 37 | 38 | parser.parse! 39 | 40 | datapoints = ARGV[0].to_i - 1 41 | per_second = ARGV[1].to_i 42 | @socket = UDPSocket.new 43 | #@socket.connect options[:host], options[:port] 44 | @socket.bind("0.0.0.0", 0) 45 | 46 | puts "Starting to send fake data to #{options[:host]}:#{options[:port]}. Sending #{datapoints} unique datapoints at a rate of #{per_second} per second split across #{options[:types].join(",")}" 47 | loop do 48 | t = Benchmark.measure do 49 | per_second.times do |i| 50 | key = "random_#{(rand*datapoints).round}" 51 | value = rand * 1000 52 | type = options[:types][(rand*options[:types].size).floor] 53 | case type 54 | when :counters 55 | @socket.send "#{key}:1|c", 0, options[:host], options[:port] 56 | when :timers 57 | @socket.send "#{key}:#{value}|ms", 0, options[:host], options[:port] 58 | when :gauges 59 | @socket.send "#{key}:#{value}|g", 0, options[:host], options[:port] 60 | end 61 | end 62 | end 63 | puts "Sent datapoints in #{t.real} seconds" if ENV["VERBOSE"] 64 | if t.real < 1 65 | sleep 1.0 - t.real 66 | end 67 | end 68 | -------------------------------------------------------------------------------- /test/integration/config.yml: -------------------------------------------------------------------------------- 1 | bind: 0.0.0.0 2 | port: 8125 3 | health_port: 8126 4 | root: /tmp/batsd 5 | redis: 6 | host: 127.0.0.1 7 | port: 6379 8 | retentions: 9 | - 2 240 #1 hour 10 | - 6 10080 #1 week 11 | - 10 52594 #1 year 12 | autotruncate: false 13 | threadpool_size: 10 14 | serializer: marshal 15 | proxy: 16 | strategy: passthrough 17 | destinations: 18 | backend1: 19 | host: 127.0.0.1 20 | port: 8225 21 | -------------------------------------------------------------------------------- /test/integration/receiver: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'fileutils' 4 | require 'socket' 5 | require 'digest' 6 | require 'redis' 7 | 8 | FileUtils.rm_rf("/tmp/batsd/") 9 | FileUtils.mkdir_p("/tmp/batsd") 10 | 11 | 12 | #Set up the server 13 | pid = fork do 14 | exec "#{File.expand_path("#{__FILE__}/../../../bin/go-batsd-receiver")} -config #{File.expand_path("#{__FILE__}/../config.yml")} " 15 | end 16 | Process.detach(pid) 17 | 18 | begin 19 | sleep 1 20 | udp_client = UDPSocket.new 21 | udp_client.connect "127.0.0.1", 8125 22 | 23 | redis = Redis.new 24 | redis.flushall 25 | 26 | #Gauges 27 | puts "Executing gauges test." 28 | rand_val = rand() 29 | udp_client.send "test:#{rand_val}|g", 0 30 | sleep 1 31 | hash = Digest::MD5.hexdigest("gauges:test") 32 | vals = File.read("/tmp/batsd/#{hash[0..1]}/#{hash[2..3]}/#{hash}") 33 | if (last_val = vals.split("\n").last.split(" ").last.to_f) != rand_val 34 | puts "Gauge test: FAILED. Expected value to be #{rand_val}, was #{last_val}" 35 | exit 2 36 | else 37 | puts "Gauge test: PASSED" 38 | end 39 | 40 | # Counters 41 | puts "Executing counters test" 42 | udp_client.send "test:1|c", 0 43 | sleep 5 44 | udp_client.send "test:1|c", 0 45 | sleep 15 46 | 47 | vals = redis.zrange "counters:test", 0, 10 48 | if vals.count != 2 || vals.last.split("").last.to_i != 1 49 | puts "Counter test (redis key): FAILED. Expected two values with counter 1, got #{vals}" 50 | exit 2 51 | else 52 | puts "Counter test (redis key): PASSED" 53 | end 54 | 55 | hash = Digest::MD5.hexdigest("counters:test:6") 56 | vals = File.read("/tmp/batsd/#{hash[0..1]}/#{hash[2..3]}/#{hash}") 57 | total = vals.split("\n").collect{|v| v.split(" ").last.to_i}.inject( nil ) { |sum,x| sum ? sum+x : x }; 58 | if total != 2 59 | puts "Counter test (first disk aggregation): FAILED. Expected total of 2, got #{total}" 60 | exit 2 61 | else 62 | puts "Counter test (first disk aggregation): PASSED" 63 | end 64 | 65 | 66 | hash = Digest::MD5.hexdigest("counters:test:10") 67 | vals = File.read("/tmp/batsd/#{hash[0..1]}/#{hash[2..3]}/#{hash}") 68 | if vals.split("\n").count != 2 || vals.split("\n").last.split(" ").last.to_i != 2 69 | puts "Counter test (second disk aggregation: FAILED. Got #{vals}" 70 | exit 2 71 | else 72 | puts "Counter test (second disk aggregation: PASSED" 73 | end 74 | 75 | #Timers 76 | puts "Executing timers test" 77 | udp_client.send "test:1|ms", 0 78 | udp_client.send "test:2|ms", 0 79 | udp_client.send "test:3|ms", 0 80 | udp_client.send "test:4|ms", 0 81 | udp_client.send "test:5|ms", 0 82 | 83 | sleep 10 84 | hash = Digest::MD5.hexdigest("timers:test:6:2") 85 | vals = File.read("/tmp/batsd/#{hash[0..1]}/#{hash[2..3]}/#{hash}") 86 | measurements = vals.split("\n").last.split(" ").last.split("/").map(&:to_f) 87 | if measurements[0] != 5 88 | puts "Timer test count: FAILED. Got #{measurements[0]}" 89 | exit 2 90 | else 91 | puts "Timer test count: PASSED." 92 | end 93 | 94 | if measurements[1] != 1 || measurements[2] != 5 95 | puts "Timer test min/max: FAILED. Got #{measurements[1]}/#{measurements[2]}" 96 | exit 2 97 | else 98 | puts "Timer test min/max: PASSED." 99 | end 100 | 101 | if measurements[3] != 3 102 | puts "Timer test median: FAILED. Got #{measurements[3]}" 103 | exit 2 104 | else 105 | puts "Timer test median: PASSED." 106 | end 107 | 108 | if measurements[4] != 3 109 | puts "Timer test mean: FAILED. Got #{measurements[4]}" 110 | exit 2 111 | else 112 | puts "Timer test mean: PASSED." 113 | end 114 | 115 | if measurements[5] != 1.5811388300841898 116 | puts "Timer test stddev: FAILED. Got #{measurements[5]}" 117 | exit 2 118 | else 119 | puts "Timer test stddev: PASSED." 120 | end 121 | 122 | if measurements[6] != 5 || measurements[7] != 5 || measurements[8] != 5 123 | puts "Timer test percentiles: FAILED. Got #{measurements[6..8]}" 124 | exit 2 125 | else 126 | puts "Timer test percentiles: PASSED." 127 | end 128 | 129 | ensure 130 | Process.kill("HUP", pid) 131 | end -------------------------------------------------------------------------------- /test/integration/server: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'fileutils' 4 | require 'socket' 5 | require 'digest' 6 | require 'redis' 7 | require 'json' 8 | 9 | FileUtils.rm_rf("/tmp/batsd/") 10 | FileUtils.mkdir_p("/tmp/batsd") 11 | 12 | #Set up the server 13 | pid = fork do 14 | exec "#{File.expand_path("#{__FILE__}/../../../bin/go-batsd-server")} -config #{File.expand_path("#{__FILE__}/../config.yml")} " 15 | end 16 | Process.detach(pid) 17 | 18 | begin 19 | sleep 1 20 | client = TCPSocket.new("127.0.0.1", 8125) 21 | 22 | redis = Redis.new 23 | redis.flushall 24 | 25 | redis_ts = Time.now.to_i - 100 26 | #Stick some data in Redis and on disk 27 | redis.sadd "datapoints", ["test1", "test2", "test2", "test3"] 28 | redis.zadd "counters:test1", redis_ts, "#{redis_ts}5923" 29 | hash = Digest::MD5.hexdigest("counters:test2:6") 30 | path = "/tmp/batsd/#{hash[0..1]}/#{hash[2..3]}/#{hash}" 31 | FileUtils.mkdir_p("/tmp/batsd/#{hash[0..1]}/#{hash[2..3]}") 32 | 33 | fileContent = <<-EOF 34 | v2 counters:test2:6 35 | #{Time.now.to_i - 3600} 174 36 | #{Time.now.to_i - 3500} 176 37 | EOF 38 | 39 | File.open(path, "w" ) do |f| 40 | f.write fileContent 41 | f.close 42 | end 43 | 44 | print "Testing ping pong..." 45 | client.puts "ping" 46 | if client.gets.chomp != "PONG" 47 | print "FAILED\n" 48 | exit 2 49 | else 50 | print "Ok\n" 51 | end 52 | 53 | print "Testing available..." 54 | client.puts "available" 55 | avail = JSON(client.gets.chomp) 56 | if avail.sort != %w(test1 test2 test3) 57 | print "FAILED\n" 58 | exit 2 59 | else 60 | print "Ok\n" 61 | end 62 | 63 | print "Testing counter still in redis...." 64 | client.puts "values counters:test1 #{Time.now.to_i - 150} #{Time.now.to_i}" 65 | data =JSON(client.gets.chomp) 66 | if data != [{"Timestamp" => redis_ts, "Value" => 5923}] 67 | print "FAILED\n" 68 | exit 2 69 | else 70 | print "Ok\n" 71 | end 72 | 73 | 74 | print "Testing counter on disk...." 75 | client.puts "values counters:test2 #{Time.now.to_i - 4000} #{Time.now.to_i}" 76 | data =JSON(client.gets.chomp) 77 | if data.count != 2 || data[1]["Value"] != 176 78 | print "FAILED\n" 79 | exit 2 80 | else 81 | print "Ok\n" 82 | end 83 | 84 | print "Testing quit..." 85 | client.puts "quit" 86 | if client.gets.chomp != "BYE" 87 | print "FAILED\n" 88 | exit 2 89 | else 90 | print "Ok\n" 91 | end 92 | 93 | 94 | ensure 95 | Process.kill("HUP", pid) 96 | end -------------------------------------------------------------------------------- /truncator.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "github.com/noahhl/Go-Redis" 7 | "github.com/noahhl/go-batsd/gobatsd" 8 | "io" 9 | "os" 10 | "regexp" 11 | "sort" 12 | "strconv" 13 | "strings" 14 | "time" 15 | ) 16 | 17 | const NWORKERS = 4 18 | 19 | func main() { 20 | gobatsd.LoadConfig() 21 | index := sort.Search(len(gobatsd.Config.Retentions), func(i int) bool { return gobatsd.Config.Retentions[i].Interval == gobatsd.Config.TargetInterval }) 22 | if index == len(gobatsd.Config.Retentions) { 23 | if gobatsd.Config.Retentions[0].Interval == gobatsd.Config.TargetInterval { 24 | index = 0 25 | } else { 26 | panic("You must specify the duration you'd like to truncate") 27 | } 28 | } 29 | retention := gobatsd.Config.Retentions[index] 30 | fmt.Printf("Starting truncation for the %v duration.\n", retention.Interval) 31 | spec := redis.DefaultSpec().Host(gobatsd.Config.RedisHost).Port(gobatsd.Config.RedisPort) 32 | redis, err := redis.NewSynchClientWithSpec(spec) 33 | if err != nil { 34 | panic(err) 35 | } 36 | 37 | datapoints := make([]string, 0) 38 | if index == 0 { 39 | rawDatapoints, err := redis.Keys("*") 40 | if err != nil { 41 | panic(err) 42 | } 43 | 44 | for _, key := range rawDatapoints { 45 | if m, _ := regexp.MatchString("^timers|^counters", string(key)); m { 46 | datapoints = append(datapoints, string(key)) 47 | } 48 | } 49 | } else { 50 | rawDatapoints, err := redis.Smembers("datapoints") 51 | if err != nil { 52 | panic(err) 53 | } 54 | 55 | for _, key := range rawDatapoints { 56 | if m, _ := regexp.MatchString("^timers|^counters", string(key)); m { 57 | datapoints = append(datapoints, string(key)) 58 | } 59 | } 60 | 61 | } 62 | since := float64(time.Now().Unix() - retention.Duration) 63 | fmt.Printf("Truncating %v datapoints since %f.\n", len(datapoints), since) 64 | 65 | if index == 0 { //Redis truncation 66 | for _, key := range datapoints { 67 | redis.Zremrangebyscore(key, 0.0, since) 68 | } 69 | } else { 70 | nworkers := NWORKERS 71 | if nworkers > len(datapoints) { 72 | nworkers = len(datapoints) 73 | } 74 | c := make(chan int, nworkers) 75 | for i := 0; i < nworkers; i++ { 76 | go func(datapoints []string, i int, c chan int) { 77 | for _, key := range datapoints { 78 | metricName := key + ":" + strconv.FormatInt(retention.Interval, 10) 79 | if m, _ := regexp.MatchString("^timers", metricName); m { 80 | metricName += ":2" 81 | } 82 | TruncateOnDisk(metricName, since) 83 | } 84 | c <- 1 85 | }(datapoints[i*len(datapoints)/nworkers:(i+1)*len(datapoints)/nworkers-1], i, c) 86 | 87 | } 88 | 89 | for i := 0; i < nworkers; i++ { 90 | <-c 91 | } 92 | 93 | } 94 | } 95 | 96 | func TruncateOnDisk(metric string, since float64) { 97 | filePath := gobatsd.CalculateFilename(metric, gobatsd.Config.Root) 98 | file, err := os.Open(filePath) 99 | tmpfile, writeErr := os.Create(filePath + "tmp") 100 | if writeErr != nil { 101 | panic(writeErr) 102 | } 103 | if err == nil && writeErr == nil { 104 | linesWritten := 0 105 | reader := bufio.NewReader(file) 106 | writer := bufio.NewWriter(tmpfile) 107 | for { 108 | line, err := reader.ReadString('\n') 109 | if err != nil && err != io.EOF { 110 | panic(err) 111 | } 112 | if err != nil && err == io.EOF { 113 | break 114 | } 115 | //skip the header in v2 files 116 | if line[0] == 'v' { 117 | writer.WriteString(line) 118 | } else { 119 | 120 | parts := strings.Split(strings.TrimSpace(line), " ") 121 | ts, _ := strconv.ParseFloat(parts[0], 64) 122 | if ts >= since { 123 | writer.WriteString(line) 124 | linesWritten++ 125 | } 126 | } 127 | } 128 | writer.Flush() 129 | file.Close() 130 | tmpfile.Close() 131 | if linesWritten > 0 { 132 | os.Rename(filePath+"tmp", filePath) 133 | } else { 134 | os.Remove(filePath + "tmp") 135 | os.Remove(filePath) 136 | } 137 | } 138 | 139 | } 140 | --------------------------------------------------------------------------------