├── .gitignore ├── .travis.yml ├── LICENSE ├── Makefile ├── README.md ├── bytetree ├── bytetree.go └── bytetree_test.go ├── cluster_follow.go ├── cluster_query.go ├── cmd ├── flags.go ├── zeno-cli │ └── zeno-cli.go ├── zeno │ └── zeno.go └── zenotool │ └── zenotool.go ├── common ├── common.go └── common_test.go ├── compression └── compression_test.go ├── core ├── compare.go ├── core.go ├── core_test.go ├── filter.go ├── flatten.go ├── format.go ├── group.go ├── limit.go ├── offset.go ├── sort.go ├── sort_test.go └── unflatten.go ├── encoding ├── encoding.go ├── params.go ├── seq.go ├── seq_test.go ├── time.go └── time_test.go ├── expr ├── aggregate.go ├── aggregates.go ├── aggregates_test.go ├── avg.go ├── binary.go ├── bounded.go ├── calcs.go ├── calcs_test.go ├── combined_test.go ├── conds.go ├── conds_test.go ├── constant.go ├── constant_test.go ├── expr.go ├── field.go ├── field_test.go ├── floatequals.go ├── if.go ├── math.go ├── math_test.go ├── percentile.go ├── percentile_optimized.go ├── percentile_test.go ├── shift.go └── shift_test.go ├── go.mod ├── go.sum ├── insert.go ├── math_bench_test.go ├── merge.go ├── metrics ├── metrics.go └── metrics_test.go ├── planner ├── cluster.go ├── having.go ├── local.go ├── planner.go ├── planner_test.go └── subquery.go ├── query.go ├── quickstart_aliases.props ├── quickstart_schema.yaml ├── row_store.go ├── row_store_test.go ├── rpc ├── msgpack_codec.go ├── rpc.go ├── rpc_client.go ├── server │ ├── rpc_server.go │ └── rpc_test.go └── snappyconn.go ├── schema.go ├── server ├── server.go ├── server_test.go └── signal.go ├── sql ├── duration.go ├── sql.go └── sql_test.go ├── table.go ├── test.sh ├── testsupport ├── expectedresult.go └── testwriter.go ├── web ├── auth.go ├── cache.go ├── cache_test.go ├── cookie_test.go ├── handler.go ├── index.go ├── insert.go ├── metrics.go ├── query.go └── toolsupport.go ├── zenodb.go └── zenodb_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | zenodbdemo/zenodbdemo 2 | zeno/zeno 3 | zeno-cli/zeno-cli 4 | vendor 5 | *.pem 6 | profile.cov 7 | profile_tmp.cov 8 | cmd/zeno/zeno 9 | cmd/zenotool/zenotool 10 | zeno 11 | zeno-cli 12 | zenotool 13 | .idea 14 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - 1.12.5 4 | install: 5 | - go get golang.org/x/tools/cmd/cover 6 | - go get -v github.com/axw/gocov/gocov 7 | - go get -v github.com/mattn/goveralls 8 | script: 9 | - ./test.sh 10 | after_success: 11 | - GOPATH=`pwd`:$GOPATH $HOME/gopath/bin/goveralls -coverprofile=profile.cov -service=travis-ci 12 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SOURCES := $(shell find . -type f -name "*.go" | grep -v /vendor) 2 | 3 | DEPS := go.mod go.sum 4 | 5 | zeno: $(SOURCES) $(DEPS) 6 | GO111MODULE=on GOOS=linux GOARCH=amd64 go build github.com/getlantern/zenodb/cmd/zeno && upx zeno 7 | 8 | zeno-cli: $(SOURCES) $(DEPS) 9 | GO111MODULE=on GOOS=linux GOARCH=amd64 go build github.com/getlantern/zenodb/cmd/zeno-cli && upx zeno-cli 10 | 11 | zenotool: $(SOURCES) $(DEPS) 12 | GO111MODULE=on GOOS=linux GOARCH=amd64 go build github.com/getlantern/zenodb/cmd/zenotool && upx zenotool 13 | -------------------------------------------------------------------------------- /bytetree/bytetree_test.go: -------------------------------------------------------------------------------- 1 | package bytetree 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/getlantern/bytemap" 8 | "github.com/getlantern/zenodb/encoding" 9 | . "github.com/getlantern/zenodb/expr" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | const ctx = 56 14 | 15 | var ( 16 | epoch = time.Date(2015, 1, 1, 0, 0, 0, 0, time.UTC) 17 | ) 18 | 19 | func TestByteTreeSubMerge(t *testing.T) { 20 | doTest(t, func(bt *Tree, resolutionOut time.Duration, eA Expr, eB Expr) { 21 | // Updates that create new keys 22 | bt.Update([]byte("test"), []encoding.Sequence{encoding.NewFloatValue(eA, epoch, 1), encoding.NewFloatValue(eB, epoch, 1)}, nil, nil) 23 | assert.Equal(t, 1, bt.Length()) 24 | bt.Update([]byte("slow"), []encoding.Sequence{encoding.NewFloatValue(eA, epoch, 2), encoding.NewFloatValue(eB, epoch, 2)}, nil, nil) 25 | assert.Equal(t, 2, bt.Length()) 26 | bt.Update(nil, []encoding.Sequence{encoding.NewFloatValue(eA, epoch, 3), encoding.NewFloatValue(eB, epoch, 3)}, nil, nil) 27 | assert.Equal(t, 3, bt.Length()) 28 | bt.Update([]byte("slower"), []encoding.Sequence{encoding.NewFloatValue(eA, epoch, 4), encoding.NewFloatValue(eB, epoch, 4)}, nil, nil) 29 | assert.Equal(t, 4, bt.Length()) 30 | bt.Update([]byte("team"), []encoding.Sequence{encoding.NewFloatValue(eA, epoch, 5), encoding.NewFloatValue(eB, epoch, 5)}, nil, nil) 31 | assert.Equal(t, 5, bt.Length()) 32 | bt.Update([]byte("toast"), []encoding.Sequence{encoding.NewFloatValue(eA, epoch, 6), encoding.NewFloatValue(eB, epoch, 6)}, nil, nil) 33 | assert.Equal(t, 6, bt.Length()) 34 | 35 | // Updates to existing keys 36 | bt.Update([]byte("test"), []encoding.Sequence{encoding.NewFloatValue(eA, epoch, 10), encoding.NewFloatValue(eB, epoch, 10)}, nil, nil) 37 | assert.Equal(t, 6, bt.Length()) 38 | bt.Update([]byte("slow"), []encoding.Sequence{encoding.NewFloatValue(eA, epoch, 10), encoding.NewFloatValue(eB, epoch, 10)}, nil, nil) 39 | assert.Equal(t, 6, bt.Length()) 40 | bt.Update(nil, []encoding.Sequence{encoding.NewFloatValue(eA, epoch, 10), encoding.NewFloatValue(eB, epoch, 10)}, nil, nil) 41 | assert.Equal(t, 6, bt.Length()) 42 | bt.Update([]byte("slower"), []encoding.Sequence{encoding.NewFloatValue(eA, epoch, 10), encoding.NewFloatValue(eB, epoch, 10)}, nil, nil) 43 | assert.Equal(t, 6, bt.Length()) 44 | bt.Update([]byte("team"), []encoding.Sequence{encoding.NewFloatValue(eA, epoch, 10), encoding.NewFloatValue(eB, epoch, 10)}, nil, nil) 45 | assert.Equal(t, 6, bt.Length()) 46 | bt.Update([]byte("toast"), []encoding.Sequence{encoding.NewFloatValue(eA, epoch, 10), encoding.NewFloatValue(eB, epoch, 10)}, nil, nil) 47 | assert.Equal(t, 6, bt.Length()) 48 | 49 | // This should be ignored because it's outside of the time range 50 | bt.Update([]byte("test"), []encoding.Sequence{encoding.NewFloatValue(eA, epoch.Add(-1*resolutionOut), 50), encoding.NewFloatValue(eB, epoch.Add(1*resolutionOut), 10)}, nil, nil) 51 | }) 52 | } 53 | 54 | func TestByteTreeUpdate(t *testing.T) { 55 | doTest(t, func(bt *Tree, resolutionOut time.Duration, eA Expr, eB Expr) { 56 | // Updates that create new keys 57 | bt.Update([]byte("test"), nil, params(1, 1), nil) 58 | assert.Equal(t, 1, bt.Length()) 59 | bt.Update([]byte("slow"), nil, params(2, 2), nil) 60 | assert.Equal(t, 2, bt.Length()) 61 | bt.Update(nil, nil, params(3, 3), nil) 62 | assert.Equal(t, 3, bt.Length()) 63 | bt.Update([]byte("slower"), nil, params(4, 4), nil) 64 | assert.Equal(t, 4, bt.Length()) 65 | bt.Update([]byte("team"), nil, params(5, 5), nil) 66 | assert.Equal(t, 5, bt.Length()) 67 | bt.Update([]byte("toast"), nil, params(6, 6), nil) 68 | assert.Equal(t, 6, bt.Length()) 69 | 70 | // Updates to existing keys 71 | bt.Update([]byte("test"), nil, params(10, 10), nil) 72 | assert.Equal(t, 6, bt.Length()) 73 | bt.Update([]byte("slow"), nil, params(10, 10), nil) 74 | assert.Equal(t, 6, bt.Length()) 75 | bt.Update(nil, nil, params(10, 10), nil) 76 | assert.Equal(t, 6, bt.Length()) 77 | bt.Update([]byte("slower"), nil, params(10, 10), nil) 78 | assert.Equal(t, 6, bt.Length()) 79 | bt.Update([]byte("team"), nil, params(10, 10), nil) 80 | assert.Equal(t, 6, bt.Length()) 81 | bt.Update([]byte("toast"), nil, params(10, 10), nil) 82 | assert.Equal(t, 6, bt.Length()) 83 | 84 | bt.Update([]byte("test"), nil, tsParams(epoch.Add(-1*resolutionOut), 50, 10), nil) 85 | }) 86 | } 87 | 88 | func doTest(t *testing.T, populate func(bt *Tree, resolutionOut time.Duration, eA Expr, eB Expr)) { 89 | resolutionOut := 10 * time.Second 90 | resolutionIn := 1 * time.Second 91 | 92 | asOf := epoch.Add(-1 * resolutionOut) 93 | until := epoch 94 | 95 | eOut := ADD(SUM(FIELD("a")), SUM(FIELD("b"))) 96 | eA := SUM(FIELD("a")) 97 | eB := SUM(FIELD("b")) 98 | 99 | // First test submerging 100 | 101 | bt := New([]Expr{eOut}, []Expr{eA, eB}, resolutionOut, resolutionIn, asOf, until, 0) 102 | populate(bt, resolutionOut, eA, eB) 103 | 104 | // Check tree twice with different contexts to make sure removals don't affect 105 | // other contexts. 106 | checkTree(ctx, t, bt, eOut) 107 | checkTree(98, t, bt, eOut) 108 | 109 | // Copy tree and check again 110 | checkTree(99, t, bt.Copy(), eOut) 111 | } 112 | 113 | func tsParams(ts time.Time, a float64, b float64) encoding.TSParams { 114 | return encoding.NewTSParams(ts, bytemap.NewFloat(map[string]float64{"a": a, "b": b})) 115 | } 116 | 117 | func params(a float64, b float64) encoding.TSParams { 118 | return tsParams(epoch, a, b) 119 | } 120 | 121 | func checkTree(ctx int64, t *testing.T, bt *Tree, e Expr) { 122 | walkedValues := 0 123 | bt.Walk(ctx, func(key []byte, data []encoding.Sequence) (bool, bool, error) { 124 | if assert.Len(t, data, 1) { 125 | walkedValues++ 126 | val, _ := data[0].ValueAt(0, e) 127 | switch string(key) { 128 | case "test": 129 | assert.EqualValues(t, 22, val, "test") 130 | case "slow": 131 | assert.EqualValues(t, 24, val, "slow") 132 | case "": 133 | assert.EqualValues(t, 26, val, "") 134 | case "slower": 135 | assert.EqualValues(t, 28, val, "slower") 136 | case "team": 137 | assert.EqualValues(t, 30, val, "team") 138 | case "toast": 139 | assert.EqualValues(t, 32, val, "toast") 140 | default: 141 | assert.Fail(t, "Unknown key", string(key)) 142 | } 143 | } 144 | return true, true, nil 145 | }) 146 | assert.Equal(t, 6, walkedValues) 147 | 148 | val, _ := bt.Remove(ctx, []byte("test"))[0].ValueAt(0, e) 149 | assert.EqualValues(t, 22, val) 150 | val, _ = bt.Remove(ctx, []byte("slow"))[0].ValueAt(0, e) 151 | assert.EqualValues(t, 24, val) 152 | val, _ = bt.Remove(ctx, nil)[0].ValueAt(0, e) 153 | assert.EqualValues(t, 26, val) 154 | val, _ = bt.Remove(ctx, []byte("slower"))[0].ValueAt(0, e) 155 | assert.EqualValues(t, 28, val) 156 | val, _ = bt.Remove(ctx, []byte("team"))[0].ValueAt(0, e) 157 | assert.EqualValues(t, 30, val) 158 | val, _ = bt.Remove(ctx, []byte("toast"))[0].ValueAt(0, e) 159 | assert.EqualValues(t, 32, val) 160 | assert.Nil(t, bt.Remove(ctx, []byte("unknown"))) 161 | } 162 | -------------------------------------------------------------------------------- /cmd/flags.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "flag" 5 | "net/http" 6 | _ "net/http/pprof" 7 | 8 | "strings" 9 | 10 | "github.com/getlantern/goexpr/isp" 11 | "github.com/getlantern/goexpr/isp/ip2location" 12 | "github.com/getlantern/goexpr/isp/maxmind" 13 | "github.com/getlantern/golog" 14 | redisutils "github.com/getlantern/redis-utils" 15 | "github.com/go-redis/redis/v8" 16 | ) 17 | 18 | const ( 19 | DefaultSentinelPort = 36379 20 | ) 21 | 22 | var ( 23 | log = golog.LoggerFor("cmd") 24 | ) 25 | 26 | var ( 27 | Schema = flag.String("schema", "schema.yaml", "Location of schema file, defaults to ./schema.yaml") 28 | AliasesFile = flag.String("aliases", "", "Optionally specify the path to a file containing expression aliases in the form alias=template(%v,%v) with one alias per line") 29 | EnableGeo = flag.Bool("enablegeo", false, "enable geolocation functions") 30 | ISPFormat = flag.String("ispformat", "ip2location", "ip2location or maxmind") 31 | ISPDB = flag.String("ispdb", "", "In order to enable ISP functions, point this to a ISP database file, either in IP2Location Lite format or MaxMind GeoIP2 ISP format") 32 | RedisAddr = flag.String("redis", "", "Redis address in \"redis[s][+sentinel]://host:port[@sentinel-ip1,sentinel-ip2]\" format") 33 | RedisCA = flag.String("redisca", "", "Certificate for redislabs's CA") 34 | RedisClientPK = flag.String("redisclientpk", "", "Private key for authenticating client to redis's stunnel") 35 | RedisClientCert = flag.String("redisclientcert", "", "Certificate for authenticating client to redis's stunnel") 36 | RedisCacheSize = flag.Int("rediscachesize", 25000, "Configures the maximum size of redis caches for HGET operations, defaults to 25,000 per hash") 37 | PprofAddr = flag.String("pprofaddr", "localhost:4000", "if specified, will listen for pprof connections at the specified tcp address") 38 | ) 39 | 40 | func StartPprof() { 41 | if *PprofAddr != "" { 42 | go func() { 43 | log.Debugf("Starting pprof page at http://%s/debug/pprof", *PprofAddr) 44 | if err := http.ListenAndServe(*PprofAddr, nil); err != nil { 45 | _ = log.Errorf("Unable to start PPROF HTTP interface: %v", err) 46 | } 47 | }() 48 | } 49 | } 50 | 51 | func ISPProvider() isp.Provider { 52 | if *ISPFormat == "" || *ISPDB == "" { 53 | log.Debug("ISP provider not configured") 54 | return nil 55 | } 56 | 57 | log.Debugf("Enabling ISP functions using format %v with db file at %v", *ISPFormat, *ISPDB) 58 | var ispProvider isp.Provider 59 | var providerErr error 60 | switch strings.ToLower(strings.TrimSpace(*ISPFormat)) { 61 | case "ip2location": 62 | ispProvider, providerErr = ip2location.NewProvider(*ISPDB) 63 | case "maxmind": 64 | ispProvider, providerErr = maxmind.NewProvider(*ISPDB) 65 | default: 66 | _ = log.Errorf("Unknown ispdb format %v", *ISPFormat) 67 | } 68 | if providerErr != nil { 69 | _ = log.Errorf("Unable to initialize ISP provider %v from %v: %v", *ISPFormat, *ISPDB, providerErr) 70 | ispProvider = nil 71 | } 72 | 73 | return ispProvider 74 | } 75 | 76 | func RedisClient() *redis.Client { 77 | if *RedisAddr == "" { 78 | log.Debug("Redis not configured") 79 | return nil 80 | } 81 | client, err := redisutils.SetupRedisClient(&redisutils.Config{ 82 | CAFile: *RedisCA, 83 | ClientKeyFile: *RedisClientPK, 84 | ClientCertFile: *RedisClientCert, 85 | URL: *RedisAddr, 86 | }) 87 | if err != nil { 88 | _ = log.Errorf("Unable to connect to redis: %v", err) 89 | return nil 90 | } 91 | return client 92 | } 93 | -------------------------------------------------------------------------------- /cmd/zeno/zeno.go: -------------------------------------------------------------------------------- 1 | // zeno is the executable for the ZenoDB database, and can run as a standalone 2 | // server, as a cluster leader or as a cluster follower. 3 | package main 4 | 5 | import ( 6 | "github.com/getlantern/golog" 7 | "github.com/getlantern/zenodb/cmd" 8 | "github.com/getlantern/zenodb/server" 9 | "github.com/vharitonsky/iniflags" 10 | ) 11 | 12 | var ( 13 | log = golog.LoggerFor("zeno") 14 | ) 15 | 16 | func main() { 17 | srv := &server.Server{} 18 | srv.ConfigureFlags() 19 | iniflags.SetAllowUnknownFlags(true) 20 | iniflags.Parse() 21 | 22 | srv.Schema = *cmd.Schema 23 | srv.AliasesFile = *cmd.AliasesFile 24 | srv.EnableGeo = *cmd.EnableGeo 25 | srv.RedisCacheSize = *cmd.RedisCacheSize 26 | 27 | cmd.StartPprof() 28 | 29 | srv.HandleShutdownSignal() 30 | _, run, err := srv.Prepare() 31 | if err != nil { 32 | log.Fatal(err) 33 | } 34 | if err := run(); err != nil { 35 | log.Fatal(err) 36 | } 37 | log.Debug("Done") 38 | } 39 | -------------------------------------------------------------------------------- /cmd/zenotool/zenotool.go: -------------------------------------------------------------------------------- 1 | // zenotool provides the ability to filter and merge zeno datafiles offline. 2 | package main 3 | 4 | import ( 5 | "encoding/csv" 6 | "flag" 7 | "fmt" 8 | "os" 9 | "strconv" 10 | "strings" 11 | 12 | "github.com/getlantern/golog" 13 | "github.com/getlantern/zenodb" 14 | "github.com/getlantern/zenodb/cmd" 15 | "github.com/getlantern/zenodb/web" 16 | "github.com/vharitonsky/iniflags" 17 | ) 18 | 19 | var ( 20 | log = golog.LoggerFor("zenotool") 21 | 22 | table = flag.String("table", "", "Name of table corresponding to these files") 23 | outFile = flag.String("out", "", "Name of file to which to write output") 24 | where = flag.String("where", "", "SQL WHERE clause for filtering rows") 25 | shouldSort = flag.Bool("sort", false, "Sort the output") 26 | info = flag.Bool("info", false, "If set, this simply shows information about the input files, no schema required") 27 | check = flag.Bool("check", false, "If set, this scans the files and makes sure they're fully readable") 28 | checktable = flag.Bool("checktable", false, "If set, this checks a single datafile for a given table") 29 | permalinks = flag.Bool("permalinks", false, "If set, this returns a list of the permalinks in the database's webcache") 30 | ) 31 | 32 | func main() { 33 | iniflags.SetAllowUnknownFlags(true) 34 | iniflags.Parse() 35 | 36 | inFiles := flag.Args() 37 | if len(inFiles) == 0 { 38 | log.Fatal("Please specify at last one input file") 39 | } 40 | 41 | if *info { 42 | for _, inFile := range inFiles { 43 | offsetsBySource, fieldsString, _, err := zenodb.FileInfo(inFile) 44 | if err != nil { 45 | log.Error(err) 46 | } else { 47 | log.Debugf("%v highWaterMarks: %v fields: %v", inFile, offsetsBySource.TSString(), fieldsString) 48 | } 49 | } 50 | return 51 | } 52 | 53 | if *permalinks { 54 | fmt.Fprintln(os.Stderr, "Listing permalinks") 55 | out := csv.NewWriter(os.Stdout) 56 | out.Write([]string{"Date", "Rows", "Permalink", "SQL"}) 57 | for _, cacheFile := range inFiles { 58 | permalinks, err := web.ListPermalinks(cacheFile) 59 | if err != nil { 60 | log.Error(err) 61 | } else { 62 | for _, permalink := range permalinks { 63 | out.Write([]string{permalink.Timestamp.Format("Jan 02 2006"), strconv.Itoa(permalink.NumRows), permalink.Permalink, permalink.SQL}) 64 | } 65 | } 66 | } 67 | return 68 | } 69 | 70 | if *check { 71 | errors := zenodb.Check(inFiles...) 72 | if len(errors) > 0 { 73 | log.Debug("------------- Files with Error -------------") 74 | for filename, err := range errors { 75 | log.Debugf("%v %v", filename, err) 76 | } 77 | os.Exit(100) 78 | } 79 | log.Debug("All Files Passed Check") 80 | return 81 | } 82 | 83 | if *table == "" { 84 | log.Fatal("Please specify a table using -table") 85 | } 86 | 87 | cmd.StartPprof() 88 | 89 | db, err := zenodb.NewDB(&zenodb.DBOpts{ 90 | SchemaFile: *cmd.Schema, 91 | EnableGeo: *cmd.EnableGeo, 92 | ISPProvider: cmd.ISPProvider(), 93 | AliasesFile: *cmd.AliasesFile, 94 | RedisClient: cmd.RedisClient(), 95 | RedisCacheSize: *cmd.RedisCacheSize, 96 | }) 97 | if err != nil { 98 | log.Fatalf("Unable to initialize DB: %v", err) 99 | } 100 | 101 | if *checktable { 102 | if err := db.CheckTable(*table, inFiles[0]); err != nil { 103 | log.Fatal(err) 104 | } 105 | return 106 | } 107 | 108 | if *outFile == "" { 109 | log.Fatal("Please specify an output file using -out") 110 | } 111 | 112 | err = db.FilterAndMerge(*table, *where, *shouldSort, *outFile, inFiles...) 113 | if err != nil { 114 | log.Fatalf("Unable to perform merge: %v", err) 115 | } 116 | 117 | log.Debugf("Merged %v -> %v", strings.Join(inFiles, " + "), *outFile) 118 | } 119 | -------------------------------------------------------------------------------- /common/common.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strings" 7 | "time" 8 | 9 | "github.com/getlantern/bytemap" 10 | "github.com/getlantern/wal" 11 | 12 | "github.com/getlantern/zenodb/encoding" 13 | ) 14 | 15 | const ( 16 | keyIncludeMemStore = "zenodb.includeMemStore" 17 | 18 | nanosPerMilli = 1000000 19 | ) 20 | 21 | type Partition struct { 22 | Keys []string 23 | Tables []*PartitionTable 24 | } 25 | 26 | type PartitionTable struct { 27 | Name string 28 | Offsets OffsetsBySource 29 | } 30 | 31 | type FollowerID struct { 32 | Partition int 33 | ID int 34 | } 35 | 36 | func (id FollowerID) String() string { 37 | return fmt.Sprintf("%d.%d", id.Partition, id.ID) 38 | } 39 | 40 | type Follow struct { 41 | FollowerID FollowerID 42 | Stream string 43 | EarliestOffset wal.Offset 44 | Partitions map[string]*Partition 45 | } 46 | 47 | type QueryRemote func(sqlString string, includeMemStore bool, isSubQuery bool, subQueryResults [][]interface{}, onValue func(bytemap.ByteMap, []encoding.Sequence)) (hasReadResult bool, err error) 48 | 49 | type QueryMetaData struct { 50 | FieldNames []string 51 | AsOf time.Time 52 | Until time.Time 53 | Resolution time.Duration 54 | Plan string 55 | } 56 | 57 | // QueryStats captures stats about query 58 | type QueryStats struct { 59 | NumPartitions int 60 | NumSuccessfulPartitions int 61 | LowestHighWaterMark int64 62 | HighestHighWaterMark int64 63 | MissingPartitions []int 64 | } 65 | 66 | // Retriable is a marker for retriable errors 67 | type Retriable interface { 68 | error 69 | 70 | Retriable() bool 71 | } 72 | 73 | type retriable struct { 74 | wrapped error 75 | } 76 | 77 | func (err *retriable) Error() string { 78 | return fmt.Sprintf("%v (retriable)", err.wrapped.Error()) 79 | } 80 | 81 | func (err *retriable) Retriable() bool { 82 | return true 83 | } 84 | 85 | // MarkRetriable marks the given error as retriable 86 | func MarkRetriable(err error) Retriable { 87 | return &retriable{err} 88 | } 89 | 90 | func WithIncludeMemStore(ctx context.Context, includeMemStore bool) context.Context { 91 | return context.WithValue(ctx, keyIncludeMemStore, includeMemStore) 92 | } 93 | 94 | func ShouldIncludeMemStore(ctx context.Context) bool { 95 | include := ctx.Value(keyIncludeMemStore) 96 | return include != nil && include.(bool) 97 | } 98 | 99 | func NanosToMillis(nanos int64) int64 { 100 | return nanos / nanosPerMilli 101 | } 102 | 103 | func TimeToMillis(ts time.Time) int64 { 104 | return NanosToMillis(ts.UnixNano()) 105 | } 106 | 107 | // OffsetsBySource is a map of wal Offsets keyed to source ids 108 | type OffsetsBySource map[int]wal.Offset 109 | 110 | // Advance advances offsetsBySource to the higher of the current offset and the 111 | // new offset from newOffsetsBySource, returning a New OffsetsBySource with the result. 112 | func (offsetsBySource OffsetsBySource) Advance(newOffsetsBySource OffsetsBySource) OffsetsBySource { 113 | if offsetsBySource == nil { 114 | return newOffsetsBySource 115 | } 116 | if newOffsetsBySource == nil { 117 | return offsetsBySource 118 | } 119 | result := make(OffsetsBySource, len(offsetsBySource)) 120 | for source, offset := range offsetsBySource { 121 | result[source] = offset 122 | } 123 | for source, newOffset := range newOffsetsBySource { 124 | oldOffset := result[source] 125 | if newOffset.After(oldOffset) { 126 | result[source] = newOffset 127 | } 128 | } 129 | return result 130 | } 131 | 132 | // LimitAge limits all offsets by source to be no earlier than the given limit 133 | func (offsetsBySource OffsetsBySource) LimitAge(limit wal.Offset) OffsetsBySource { 134 | result := make(OffsetsBySource, len(offsetsBySource)) 135 | for source, offset := range offsetsBySource { 136 | if limit.After(offset) { 137 | result[source] = limit 138 | } else { 139 | result[source] = offset 140 | } 141 | } 142 | return result 143 | } 144 | 145 | // LowestTS returns the lowest TS of any of the offsets 146 | func (offsetsBySource OffsetsBySource) LowestTS() time.Time { 147 | var result time.Time 148 | for _, offset := range offsetsBySource { 149 | ts := offset.TS() 150 | if result.IsZero() || ts.Before(result) { 151 | result = ts 152 | } 153 | } 154 | return result 155 | } 156 | 157 | // HighestTS returns the highest TS of any of the offsets 158 | func (offsetsBySource OffsetsBySource) HighestTS() time.Time { 159 | var result time.Time 160 | for _, offset := range offsetsBySource { 161 | ts := offset.TS() 162 | if result.IsZero() || ts.After(result) { 163 | result = ts 164 | } 165 | } 166 | return result 167 | } 168 | 169 | // TSString returns a string representation of the timestamps by source 170 | func (offsetsBySource OffsetsBySource) TSString() string { 171 | var highWaterMarks []string 172 | for source, offset := range offsetsBySource { 173 | highWaterMarks = append(highWaterMarks, fmt.Sprintf("(%d)%v", source, offset.TS().Format(time.RFC3339))) 174 | } 175 | return strings.Join(highWaterMarks, " , ") 176 | } 177 | 178 | func (offsetsBySource OffsetsBySource) String() string { 179 | var highWaterMarks []string 180 | for source, offset := range offsetsBySource { 181 | highWaterMarks = append(highWaterMarks, fmt.Sprintf("(%d)%v", source, offset)) 182 | } 183 | return strings.Join(highWaterMarks, " , ") 184 | } 185 | -------------------------------------------------------------------------------- /common/common_test.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/getlantern/wal" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestOffsetsBySourceAdvance(t *testing.T) { 12 | now := time.Now() 13 | nowPlusOne := now.Add(1 * time.Second) 14 | 15 | a := OffsetsBySource{ 16 | 0: wal.NewOffsetForTS(now), 17 | 2: wal.NewOffsetForTS(nowPlusOne), 18 | } 19 | b := OffsetsBySource{ 20 | 0: wal.NewOffsetForTS(nowPlusOne), 21 | 1: wal.NewOffsetForTS(now), 22 | 2: wal.NewOffsetForTS(now), 23 | } 24 | expected := OffsetsBySource{ 25 | 0: wal.NewOffsetForTS(nowPlusOne), 26 | 1: wal.NewOffsetForTS(now), 27 | 2: wal.NewOffsetForTS(nowPlusOne), 28 | } 29 | var null OffsetsBySource 30 | assert.EqualValues(t, a, a, "OffsetsBySource should equal itself") 31 | assert.EqualValues(t, a, a.Advance(null), "Advancing by nil should yield the original OffsetsBySource") 32 | assert.EqualValues(t, a, null.Advance(a), "Advancing nil should yield the advanced OffsetsBySource") 33 | assert.EqualValues(t, null, null.Advance(null), "Advancing nil by nil should yield nil") 34 | assert.EqualValues(t, expected, a.Advance(b), "Advancing should yield superset") 35 | assert.EqualValues(t, expected, b.Advance(a), "Advancing should be commutative") 36 | } 37 | 38 | func TestOffsetsBySourceLimitAge(t *testing.T) { 39 | now := time.Now() 40 | nowPlusOne := now.Add(1 * time.Second) 41 | nowPlusTwo := now.Add(2 * time.Second) 42 | 43 | os := OffsetsBySource{ 44 | 0: wal.NewOffsetForTS(now), 45 | 1: wal.NewOffsetForTS(nowPlusOne), 46 | 2: wal.NewOffsetForTS((nowPlusTwo)), 47 | } 48 | 49 | expected := OffsetsBySource{ 50 | 0: wal.NewOffsetForTS(nowPlusOne), 51 | 1: wal.NewOffsetForTS(nowPlusOne), 52 | 2: wal.NewOffsetForTS((nowPlusTwo)), 53 | } 54 | 55 | assert.EqualValues(t, expected, os.LimitAge(wal.NewOffsetForTS(nowPlusOne))) 56 | } 57 | -------------------------------------------------------------------------------- /compression/compression_test.go: -------------------------------------------------------------------------------- 1 | package zenodb 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "math/rand" 7 | "sort" 8 | 9 | "github.com/dustin/go-humanize" 10 | "github.com/getlantern/golog" 11 | "github.com/golang/snappy" 12 | "github.com/jmcvetta/randutil" 13 | 14 | "testing" 15 | ) 16 | 17 | var log = golog.LoggerFor("compression_test") 18 | 19 | const numPeriods = 365 * 24 20 | 21 | func TestSnappyNumberCompression(t *testing.T) { 22 | for i := 0.01; i <= 1; i += 0.01 { 23 | buf := bytes.NewBuffer(make([]byte, 0, numPeriods*8)) 24 | val := rand.Float64() 25 | for j := 0; j < numPeriods; j++ { 26 | delta := float64(rand.Intn(1000)) 27 | if rand.Float64() > 0.5 { 28 | val += delta 29 | } else { 30 | val -= delta 31 | } 32 | if val > i { 33 | val = 0 34 | } 35 | binary.Write(buf, binary.BigEndian, val) 36 | } 37 | b := buf.Bytes() 38 | compressed := snappy.Encode(make([]byte, len(b)), b) 39 | log.Debugf("Fill Rate: %f\tUncompressed: %v\tCompressed: %v\tRatio: %f", i, humanize.Comma(int64(len(b))), humanize.Comma(int64(len(compressed))), float64(len(compressed))/float64(len(b))) 40 | } 41 | } 42 | 43 | func TestSnappyStringCompression(t *testing.T) { 44 | uniques := make([]string, 0, 10000) 45 | for j := 0; j < 10000; j++ { 46 | str, err := randutil.AlphaStringRange(15, 25) 47 | if err != nil { 48 | log.Fatal(err) 49 | } 50 | uniques = append(uniques, str) 51 | } 52 | all := make([]string, 0, 1000000) 53 | for j := 0; j < 1000000; j++ { 54 | all = append(all, uniques[rand.Intn(len(uniques))]) 55 | } 56 | var b bytes.Buffer 57 | for _, str := range all { 58 | b.WriteString(str) 59 | } 60 | unsorted := b.Len() 61 | sort.Strings(all) 62 | var sb bytes.Buffer 63 | for _, str := range all { 64 | sb.WriteString(str) 65 | } 66 | sorted := sb.Len() 67 | compressed := snappy.Encode(make([]byte, b.Len()), b.Bytes()) 68 | compressedSorted := snappy.Encode(make([]byte, sb.Len()), sb.Bytes()) 69 | log.Debugf("Unsorted: %v Sorted: %v Unsorted Compressed: %v Sorted Compressed: %v", humanize.Comma(int64(unsorted)), humanize.Comma(int64(sorted)), humanize.Comma(int64(len(compressed))), humanize.Comma(int64(len(compressedSorted)))) 70 | } 71 | -------------------------------------------------------------------------------- /core/compare.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | func compare(a interface{}, b interface{}) int { 8 | if a == nil { 9 | if b != nil { 10 | return -1 11 | } 12 | return 0 13 | } 14 | if b == nil { 15 | if a != nil { 16 | return 1 17 | } 18 | return 0 19 | } 20 | 21 | switch ta := a.(type) { 22 | case bool: 23 | tvb := b.(bool) 24 | if ta && !tvb { 25 | return 1 26 | } 27 | if !ta && tvb { 28 | return -1 29 | } 30 | case byte: 31 | tvb := b.(byte) 32 | if ta > tvb { 33 | return 1 34 | } 35 | if ta < tvb { 36 | return -1 37 | } 38 | case uint16: 39 | tvb := b.(uint16) 40 | if ta > tvb { 41 | return 1 42 | } 43 | if ta < tvb { 44 | return -1 45 | } 46 | case uint32: 47 | tvb := b.(uint32) 48 | if ta > tvb { 49 | return 1 50 | } 51 | if ta < tvb { 52 | return -1 53 | } 54 | case uint64: 55 | tvb := b.(uint64) 56 | if ta > tvb { 57 | return 1 58 | } 59 | if ta < tvb { 60 | return -1 61 | } 62 | case uint: 63 | tvb := uint(b.(uint64)) 64 | if ta > tvb { 65 | return 1 66 | } 67 | if ta < tvb { 68 | return -1 69 | } 70 | case int8: 71 | tvb := b.(int8) 72 | if ta > tvb { 73 | return 1 74 | } 75 | if ta < tvb { 76 | return -1 77 | } 78 | case int16: 79 | tvb := b.(int16) 80 | if ta > tvb { 81 | return 1 82 | } 83 | if ta < tvb { 84 | return -1 85 | } 86 | case int32: 87 | tvb := b.(int32) 88 | if ta > tvb { 89 | return 1 90 | } 91 | if ta < tvb { 92 | return -1 93 | } 94 | case int64: 95 | tvb := b.(int64) 96 | if ta > tvb { 97 | return 1 98 | } 99 | if ta < tvb { 100 | return -1 101 | } 102 | case int: 103 | tvb := b.(int) 104 | if ta > tvb { 105 | return 1 106 | } 107 | if ta < tvb { 108 | return -1 109 | } 110 | case float32: 111 | tvb := b.(float32) 112 | if ta > tvb { 113 | return 1 114 | } 115 | if ta < tvb { 116 | return -1 117 | } 118 | case float64: 119 | tvb := b.(float64) 120 | if ta > tvb { 121 | return 1 122 | } 123 | if ta < tvb { 124 | return -1 125 | } 126 | case string: 127 | tvb := b.(string) 128 | if ta > tvb { 129 | return 1 130 | } 131 | if ta < tvb { 132 | return -1 133 | } 134 | case time.Time: 135 | tvb := b.(time.Time) 136 | if ta.After(tvb) { 137 | return 1 138 | } 139 | if ta.Before(tvb) { 140 | return -1 141 | } 142 | } 143 | 144 | return 0 145 | } 146 | -------------------------------------------------------------------------------- /core/filter.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/getlantern/bytemap" 7 | ) 8 | 9 | func RowFilter(source RowSource, label string, include func(ctx context.Context, key bytemap.ByteMap, fields Fields, vals Vals) (bytemap.ByteMap, Vals, error)) RowSource { 10 | return &rowFilter{ 11 | rowTransform{source}, 12 | include, 13 | label, 14 | } 15 | } 16 | 17 | type rowFilter struct { 18 | rowTransform 19 | Include func(ctx context.Context, key bytemap.ByteMap, fields Fields, vals Vals) (bytemap.ByteMap, Vals, error) 20 | Label string 21 | } 22 | 23 | func (f *rowFilter) Iterate(ctx context.Context, onFields OnFields, onRow OnRow) (interface{}, error) { 24 | guard := Guard(ctx) 25 | 26 | var fields Fields 27 | return f.source.Iterate(ctx, func(inFields Fields) error { 28 | fields = inFields 29 | return onFields(inFields) 30 | }, func(key bytemap.ByteMap, vals Vals) (bool, error) { 31 | var err error 32 | key, vals, err = f.Include(ctx, key, fields, vals) 33 | if err != nil { 34 | return false, err 35 | } 36 | if key != nil { 37 | return onRow(key, vals) 38 | } 39 | return guard.Proceed() 40 | }) 41 | } 42 | 43 | func (f *rowFilter) String() string { 44 | return fmt.Sprintf("rowFilter %v", f.Label) 45 | } 46 | 47 | func FlatRowFilter(source FlatRowSource, label string, include func(ctx context.Context, row *FlatRow, fields Fields) (*FlatRow, error)) FlatRowSource { 48 | return &flatRowFilter{ 49 | flatRowTransform{source}, 50 | include, 51 | label, 52 | } 53 | } 54 | 55 | type flatRowFilter struct { 56 | flatRowTransform 57 | Include func(ctx context.Context, row *FlatRow, fields Fields) (*FlatRow, error) 58 | Label string 59 | } 60 | 61 | func (f *flatRowFilter) Iterate(ctx context.Context, onFields OnFields, onRow OnFlatRow) (interface{}, error) { 62 | guard := Guard(ctx) 63 | 64 | var fields Fields 65 | return f.source.Iterate(ctx, func(inFields Fields) error { 66 | fields = inFields 67 | return onFields(inFields) 68 | }, func(row *FlatRow) (bool, error) { 69 | var err error 70 | row, err = f.Include(ctx, row, fields) 71 | if err != nil { 72 | return false, err 73 | } 74 | if row != nil { 75 | return onRow(row) 76 | } 77 | return guard.Proceed() 78 | }) 79 | } 80 | 81 | func (f *flatRowFilter) String() string { 82 | return fmt.Sprintf("flatrowFilter %v", f.Label) 83 | } 84 | -------------------------------------------------------------------------------- /core/flatten.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/getlantern/bytemap" 8 | "github.com/getlantern/zenodb/expr" 9 | ) 10 | 11 | func Flatten(source RowSource) FlatRowSource { 12 | return &flatten{rowTransform{source}} 13 | } 14 | 15 | type flatten struct { 16 | rowTransform 17 | } 18 | 19 | func (f *flatten) Iterate(ctx context.Context, onFields OnFields, onRow OnFlatRow) (interface{}, error) { 20 | guard := Guard(ctx) 21 | 22 | resolution := f.GetResolution() 23 | 24 | var fields Fields 25 | var numFields int 26 | 27 | return f.source.Iterate(ctx, func(inFields Fields) error { 28 | fields = inFields 29 | numFields = len(inFields) 30 | // Transform to flattened version of fields 31 | outFields := make(Fields, 0, len(inFields)) 32 | for _, field := range inFields { 33 | outFields = append(outFields, NewField(field.Name, expr.FIELD(field.Name))) 34 | } 35 | return onFields(outFields) 36 | }, func(key bytemap.ByteMap, vals Vals) (bool, error) { 37 | var until time.Time 38 | var asOf time.Time 39 | // Figure out total time range 40 | for i, field := range fields { 41 | val := vals[i] 42 | e := field.Expr 43 | width := e.EncodedWidth() 44 | if val.NumPeriods(width) == 0 { 45 | continue 46 | } 47 | newUntil := val.Until() 48 | newAsOf := val.AsOf(width, resolution) 49 | if newUntil.After(until) { 50 | until = newUntil 51 | } 52 | if asOf.IsZero() || newAsOf.Before(asOf) { 53 | asOf = newAsOf 54 | } 55 | } 56 | 57 | // Iterate 58 | ts := asOf 59 | for ; !ts.After(until); ts = ts.Add(resolution) { 60 | tsNanos := ts.UnixNano() 61 | row := &FlatRow{ 62 | TS: tsNanos, 63 | Key: key, 64 | Values: make([]float64, numFields), 65 | fields: fields, 66 | } 67 | anyNonConstantValueFound := false 68 | for i, field := range fields { 69 | val, found := vals[i].ValueAtTime(ts, field.Expr, resolution) 70 | if found && !field.Expr.IsConstant() { 71 | anyNonConstantValueFound = true 72 | } 73 | row.Values[i] = val 74 | } 75 | if anyNonConstantValueFound { 76 | more, err := onRow(row) 77 | if !more || err != nil { 78 | return more, err 79 | } 80 | } 81 | } 82 | 83 | return guard.Proceed() 84 | }) 85 | } 86 | 87 | func (f *flatten) String() string { 88 | return "flatten" 89 | } 90 | -------------------------------------------------------------------------------- /core/format.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "bytes" 5 | "strings" 6 | ) 7 | 8 | func FormatSource(source Source) string { 9 | result := &bytes.Buffer{} 10 | doFormatSource(result, "", source) 11 | return result.String() 12 | } 13 | 14 | func doFormatSource(result *bytes.Buffer, indent string, source Source) { 15 | for i, s := range strings.Split(source.String(), "\n") { 16 | result.WriteString(indent) 17 | if i == 0 { 18 | result.WriteString("<- ") 19 | } 20 | result.WriteString(s) 21 | result.WriteByte('\n') 22 | } 23 | t, ok := source.(Transform) 24 | if ok { 25 | indent += " " 26 | s := t.GetSource() 27 | doFormatSource(result, indent, s) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /core/limit.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync/atomic" 7 | ) 8 | 9 | func Limit(source FlatRowSource, lim int) FlatRowSource { 10 | return &limit{ 11 | flatRowTransform{source}, 12 | lim, 13 | } 14 | } 15 | 16 | type limit struct { 17 | flatRowTransform 18 | limit int 19 | } 20 | 21 | func (l *limit) Iterate(ctx context.Context, onFields OnFields, onRow OnFlatRow) (interface{}, error) { 22 | idx := int64(0) 23 | return l.source.Iterate(ctx, onFields, func(row *FlatRow) (bool, error) { 24 | newIdx := atomic.AddInt64(&idx, 1) 25 | oldIdx := int(newIdx - 1) 26 | if oldIdx < l.limit { 27 | return onRow(row) 28 | } 29 | return stop() 30 | }) 31 | } 32 | 33 | func (l *limit) String() string { 34 | return fmt.Sprintf("limit %d", l.limit) 35 | } 36 | -------------------------------------------------------------------------------- /core/offset.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync/atomic" 7 | ) 8 | 9 | func Offset(source FlatRowSource, off int) FlatRowSource { 10 | return &offset{ 11 | flatRowTransform{source}, 12 | off, 13 | } 14 | } 15 | 16 | type offset struct { 17 | flatRowTransform 18 | offset int 19 | } 20 | 21 | func (o *offset) Iterate(ctx context.Context, onFields OnFields, onRow OnFlatRow) (interface{}, error) { 22 | guard := Guard(ctx) 23 | 24 | idx := int64(0) 25 | return o.source.Iterate(ctx, onFields, func(row *FlatRow) (bool, error) { 26 | newIdx := atomic.AddInt64(&idx, 1) 27 | oldIdx := int(newIdx - 1) 28 | // TODO: allow stopping iteration here 29 | if oldIdx >= o.offset { 30 | return onRow(row) 31 | } 32 | return guard.Proceed() 33 | }) 34 | } 35 | 36 | func (o *offset) String() string { 37 | return fmt.Sprintf("offset %d", o.offset) 38 | } 39 | -------------------------------------------------------------------------------- /core/sort.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sort" 7 | ) 8 | 9 | // OrderBy specifies an element by whith to order (element being ither a field 10 | // name or the name of a dimension in the row key). 11 | type OrderBy struct { 12 | Field string 13 | Descending bool 14 | } 15 | 16 | func (o OrderBy) String() string { 17 | ascending := "asc" 18 | if o.Descending { 19 | ascending = "desc" 20 | } 21 | return fmt.Sprintf("%v(%v)", o.Field, ascending) 22 | } 23 | 24 | func NewOrderBy(field string, descending bool) OrderBy { 25 | return OrderBy{ 26 | Field: field, 27 | Descending: descending, 28 | } 29 | } 30 | 31 | // Get implements the interface method from goexpr.Params 32 | func (row *FlatRow) Get(param string) interface{} { 33 | // First look at values 34 | for i, field := range row.fields { 35 | if field.Name == param { 36 | return row.Values[i] 37 | } 38 | } 39 | 40 | // Then look at key 41 | return row.Key.Get(param) 42 | } 43 | 44 | func Sort(source FlatRowSource, by ...OrderBy) FlatRowSource { 45 | return &sorter{ 46 | flatRowTransform{source}, 47 | by, 48 | } 49 | } 50 | 51 | type sorter struct { 52 | flatRowTransform 53 | by []OrderBy 54 | } 55 | 56 | func (s *sorter) Iterate(ctx context.Context, onFields OnFields, onRow OnFlatRow) (interface{}, error) { 57 | guard := Guard(ctx) 58 | 59 | rows := orderedRows{ 60 | orderBy: s.by, 61 | } 62 | 63 | metadata, err := s.source.Iterate(ctx, onFields, func(row *FlatRow) (bool, error) { 64 | rows.rows = append(rows.rows, row) 65 | return guard.Proceed() 66 | }) 67 | 68 | if err != ErrDeadlineExceeded { 69 | sort.Sort(rows) 70 | for _, row := range rows.rows { 71 | if guard.TimedOut() { 72 | return metadata, ErrDeadlineExceeded 73 | } 74 | 75 | more, onRowErr := onRow(row) 76 | if onRowErr != nil { 77 | return metadata, onRowErr 78 | } 79 | if !more { 80 | break 81 | } 82 | } 83 | } 84 | return metadata, err 85 | } 86 | 87 | func (s *sorter) String() string { 88 | return fmt.Sprintf("order by %v", s.by) 89 | } 90 | 91 | type orderedRows struct { 92 | orderBy []OrderBy 93 | rows []*FlatRow 94 | } 95 | 96 | func (r orderedRows) Len() int { return len(r.rows) } 97 | func (r orderedRows) Swap(i, j int) { r.rows[i], r.rows[j] = r.rows[j], r.rows[i] } 98 | func (r orderedRows) Less(i, j int) bool { 99 | a := r.rows[i] 100 | b := r.rows[j] 101 | for _, order := range r.orderBy { 102 | // _time is a special case 103 | if order.Field == "_time" { 104 | ta := a.TS 105 | tb := b.TS 106 | if order.Descending { 107 | ta, tb = tb, ta 108 | } 109 | if ta < tb { 110 | return true 111 | } 112 | continue 113 | } 114 | 115 | // sort by field or dim 116 | va := a.Get(order.Field) 117 | vb := b.Get(order.Field) 118 | if order.Descending { 119 | va, vb = vb, va 120 | } 121 | result := compare(va, vb) 122 | if result < 0 { 123 | return true 124 | } 125 | if result > 0 { 126 | return false 127 | } 128 | } 129 | return false 130 | } 131 | -------------------------------------------------------------------------------- /core/sort_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "sort" 5 | "testing" 6 | 7 | "github.com/getlantern/bytemap" 8 | "github.com/getlantern/zenodb/expr" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestSortTime(t *testing.T) { 13 | rows := sortedRows(false, "_time") 14 | assert.Equal(t, []int64{0, 1, 2, 3, 4, 5}, actualTimes(rows)) 15 | 16 | rows = sortedRows(true, "_time") 17 | assert.Equal(t, []int64{5, 4, 3, 2, 1, 0}, actualTimes(rows)) 18 | } 19 | 20 | func TestSortBools(t *testing.T) { 21 | rows := sortedRows(false, "b") 22 | assert.Equal(t, []bool{false, false, false, true, true, true}, actualBools(rows)) 23 | 24 | rows = sortedRows(true, "b") 25 | assert.Equal(t, []bool{true, true, true, false, false, false}, actualBools(rows)) 26 | } 27 | 28 | func TestSortString(t *testing.T) { 29 | rows := sortedRows(false, "s") 30 | assert.Equal(t, []string{"", "a", "b", "b", "c", "c"}, actualStrings(rows)) 31 | 32 | rows = sortedRows(true, "s") 33 | assert.Equal(t, []string{"c", "c", "b", "b", "a", ""}, actualStrings(rows)) 34 | } 35 | 36 | func TestSortVals(t *testing.T) { 37 | rows := sortedRows(false, "val") 38 | assert.Equal(t, []float64{0, 12, 23, 56, 56, 78}, actualVals(rows)) 39 | 40 | rows = sortedRows(true, "val") 41 | assert.Equal(t, []float64{78, 56, 56, 23, 12, 0}, actualVals(rows)) 42 | } 43 | 44 | func TestSortAll(t *testing.T) { 45 | rows := sortedRows(false, "b", "s", "val", "_time") 46 | assert.Equal(t, []bool{false, false, false, true, true, true}, actualBools(rows)) 47 | assert.Equal(t, []string{"a", "b", "b", "", "c", "c"}, actualStrings(rows)) 48 | assert.Equal(t, []float64{78, 0, 23, 12, 56, 56}, actualVals(rows)) 49 | assert.Equal(t, []int64{1, 5, 2, 4, 0, 3}, actualTimes(rows)) 50 | 51 | rows = sortedRows(true, "b", "s", "val", "_time") 52 | assert.Equal(t, []bool{true, true, true, false, false, false}, actualBools(rows)) 53 | assert.Equal(t, []string{"c", "c", "", "b", "b", "a"}, actualStrings(rows)) 54 | assert.Equal(t, []float64{56, 56, 12, 23, 0, 78}, actualVals(rows)) 55 | assert.Equal(t, []int64{3, 0, 4, 2, 5, 1}, actualTimes(rows)) 56 | } 57 | 58 | func actualTimes(rows []*FlatRow) []int64 { 59 | return []int64{rows[0].TS, rows[1].TS, rows[2].TS, rows[3].TS, rows[4].TS, rows[5].TS} 60 | } 61 | 62 | func actualStrings(rows []*FlatRow) []string { 63 | return []string{rows[0].Key.Get("s").(string), rows[1].Key.Get("s").(string), rows[2].Key.Get("s").(string), rows[3].Key.Get("s").(string), rows[4].Key.Get("s").(string), rows[5].Key.Get("s").(string)} 64 | } 65 | 66 | func actualBools(rows []*FlatRow) []bool { 67 | return []bool{rows[0].Key.Get("b").(bool), rows[1].Key.Get("b").(bool), rows[2].Key.Get("b").(bool), rows[3].Key.Get("b").(bool), rows[4].Key.Get("b").(bool), rows[5].Key.Get("b").(bool)} 68 | } 69 | 70 | func actualVals(rows []*FlatRow) []float64 { 71 | return []float64{rows[0].Values[0], rows[1].Values[0], rows[2].Values[0], rows[3].Values[0], rows[4].Values[0], rows[5].Values[0]} 72 | } 73 | 74 | func sortedRows(descending bool, orderByStrings ...string) []*FlatRow { 75 | rows := buildRows() 76 | 77 | orderBy := make([]OrderBy, 0, len(orderByStrings)) 78 | for _, field := range orderByStrings { 79 | orderBy = append(orderBy, NewOrderBy(field, descending)) 80 | } 81 | sort.Sort(&orderedRows{ 82 | orderBy: orderBy, 83 | rows: rows, 84 | }) 85 | 86 | return rows 87 | } 88 | 89 | func buildRows() []*FlatRow { 90 | strs := []string{"c", "a", "b", "c", "", "b"} 91 | bools := []bool{true, false, false, true, true, false} 92 | vals := []float64{56, 78, 23, 56, 12, 0} 93 | 94 | rows := make([]*FlatRow, 0, len(strs)) 95 | for i, str := range strs { 96 | rows = append(rows, &FlatRow{ 97 | TS: int64(i), 98 | Key: bytemap.New(map[string]interface{}{"s": str, "b": bools[i]}), 99 | Values: []float64{vals[i]}, 100 | fields: []Field{NewField("val", expr.FIELD("val"))}, 101 | }) 102 | } 103 | 104 | return rows 105 | } 106 | -------------------------------------------------------------------------------- /core/unflatten.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/getlantern/zenodb/encoding" 8 | "github.com/getlantern/zenodb/expr" 9 | ) 10 | 11 | func Unflatten(source FlatRowSource, fields FieldSource) RowSource { 12 | return &unflatten{ 13 | flatRowTransform{source}, 14 | fields, 15 | } 16 | } 17 | 18 | func UnflattenOptimized(source FlatRowSource) RowSource { 19 | fl, ok := source.(Transform) 20 | if ok { 21 | rs, ok := fl.GetSource().(RowSource) 22 | if ok { 23 | // We're attempting to unflatten a flatten, just go back to the original source and skip the flatten/unflatten cycle 24 | return rs 25 | } 26 | } 27 | return Unflatten(source, PassthroughFieldSource) 28 | } 29 | 30 | type unflatten struct { 31 | flatRowTransform 32 | fields FieldSource 33 | } 34 | 35 | func (f *unflatten) Iterate(ctx context.Context, onFields OnFields, onRow OnRow) (interface{}, error) { 36 | var inFields, outFields Fields 37 | var numIn, numOut int 38 | 39 | return f.source.Iterate(ctx, func(fields Fields) error { 40 | inFields = fields 41 | var err error 42 | outFields, err = f.fields.Get(nil) 43 | if err != nil { 44 | return err 45 | } 46 | numIn = len(inFields) 47 | numOut = len(outFields) 48 | return onFields(outFields) 49 | }, func(row *FlatRow) (bool, error) { 50 | ts := encoding.TimeFromInt(row.TS) 51 | outRow := make(Vals, numOut) 52 | params := expr.Map(make(map[string]float64, numIn)) 53 | for i, field := range inFields { 54 | name := field.Name 55 | params[name] = row.Values[i] 56 | } 57 | for i, field := range outFields { 58 | outRow[i] = encoding.NewValue(field.Expr, ts, params, row.Key) 59 | } 60 | return onRow(row.Key, outRow) 61 | }) 62 | } 63 | 64 | func (f *unflatten) String() string { 65 | if f.fields == PassthroughFieldSource { 66 | return "unflatten all" 67 | } 68 | return fmt.Sprintf("unflatten to %v", f.fields) 69 | } 70 | -------------------------------------------------------------------------------- /encoding/encoding.go: -------------------------------------------------------------------------------- 1 | // Package encoding handles encoding of zenodb data in binary form. 2 | package encoding 3 | 4 | import ( 5 | "encoding/binary" 6 | 7 | "github.com/getlantern/bytemap" 8 | ) 9 | 10 | const ( 11 | Width16bits = 2 12 | Width32bits = 4 13 | Width64bits = 8 14 | ) 15 | 16 | var ( 17 | // Binary is the standard number encoding for zenodb 18 | Binary = binary.BigEndian 19 | ) 20 | 21 | func ReadInt16(b []byte) (int, []byte) { 22 | i := Binary.Uint16(b) 23 | return int(i), b[Width16bits:] 24 | } 25 | 26 | func WriteInt16(b []byte, i int) []byte { 27 | Binary.PutUint16(b, uint16(i)) 28 | return b[Width16bits:] 29 | } 30 | 31 | func ReadInt32(b []byte) (int, []byte) { 32 | i := Binary.Uint32(b) 33 | return int(i), b[Width32bits:] 34 | } 35 | 36 | func WriteInt32(b []byte, i int) []byte { 37 | Binary.PutUint32(b, uint32(i)) 38 | return b[Width32bits:] 39 | } 40 | 41 | func ReadInt64(b []byte) (int, []byte) { 42 | i := Binary.Uint64(b) 43 | return int(i), b[Width64bits:] 44 | } 45 | 46 | func WriteInt64(b []byte, i int) []byte { 47 | Binary.PutUint64(b, uint64(i)) 48 | return b[Width64bits:] 49 | } 50 | 51 | func ReadByteMap(b []byte, l int) (bytemap.ByteMap, []byte) { 52 | return bytemap.ByteMap(b[:l]), b[l:] 53 | } 54 | 55 | func Read(b []byte, l int) ([]byte, []byte) { 56 | return b[:l], b[l:] 57 | } 58 | 59 | func ReadSequence(b []byte, l int) (Sequence, []byte) { 60 | s, r := Read(b, l) 61 | return Sequence(s), r 62 | } 63 | 64 | func Write(b []byte, d []byte) []byte { 65 | copy(b, d) 66 | return b[len(d):] 67 | } 68 | -------------------------------------------------------------------------------- /encoding/params.go: -------------------------------------------------------------------------------- 1 | package encoding 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/getlantern/bytemap" 8 | "github.com/getlantern/zenodb/expr" 9 | ) 10 | 11 | // TSParams combines a timestamp with a ByteMap. 12 | type TSParams []byte 13 | 14 | // NewTSParams constructs a new TSParams using the given Time and ByteMap. 15 | func NewTSParams(ts time.Time, params bytemap.ByteMap) TSParams { 16 | out := make([]byte, Width64bits) 17 | EncodeTime(out, ts) 18 | return TSParams(append(out, params...)) 19 | } 20 | 21 | // TimeAndParams returns the Time and Params components of this TSParams. 22 | func (tsp TSParams) TimeAndParams() (time.Time, expr.Params) { 23 | ts := TimeFromBytes(tsp) 24 | params := bytemapParams(tsp[Width64bits:]) 25 | return ts, params 26 | } 27 | 28 | func (tsp TSParams) TimeInt() int64 { 29 | return TimeIntFromBytes(tsp) 30 | } 31 | 32 | func (tsp TSParams) String() string { 33 | ts, params := tsp.TimeAndParams() 34 | return fmt.Sprintf("%v: %v", ts, params) 35 | } 36 | 37 | // bytemapParams is an implementation of the expr.Params interface backed by a 38 | // ByteMap. 39 | type bytemapParams bytemap.ByteMap 40 | 41 | func (bmp bytemapParams) Get(field string) (float64, bool) { 42 | var result interface{} 43 | result = bytemap.ByteMap(bmp).Get(field) 44 | if result == nil { 45 | // To support counting points, handle _point magic field specially 46 | if "_point" == field { 47 | result = bytemap.ByteMap(bmp).Get("_points") 48 | if result == nil { 49 | return 1, true 50 | } 51 | } 52 | return 0, false 53 | } 54 | return result.(float64), true 55 | } 56 | 57 | func (bmp bytemapParams) String() string { 58 | return fmt.Sprint(bytemap.ByteMap(bmp).AsMap()) 59 | } 60 | -------------------------------------------------------------------------------- /encoding/time.go: -------------------------------------------------------------------------------- 1 | package encoding 2 | 3 | import ( 4 | "math" 5 | "time" 6 | ) 7 | 8 | const ( 9 | WidthTime = Width64bits 10 | nanosPerMilli = 1000000 11 | ) 12 | 13 | var ( 14 | zeroTime = time.Time{} 15 | ) 16 | 17 | func EncodeTime(b []byte, ts time.Time) { 18 | Binary.PutUint64(b, uint64(ts.UnixNano())) 19 | } 20 | 21 | func TimeFromBytes(b []byte) time.Time { 22 | return TimeFromInt(TimeIntFromBytes(b)) 23 | } 24 | 25 | func TimeIntFromBytes(b []byte) int64 { 26 | return int64(Binary.Uint64(b)) 27 | } 28 | 29 | func TimeFromInt(ts int64) time.Time { 30 | s := ts / int64(time.Second) 31 | ns := ts % int64(time.Second) 32 | return time.Unix(s, ns) 33 | } 34 | 35 | func TimeFromMillis(millis int64) time.Time { 36 | return TimeFromInt(millis * nanosPerMilli) 37 | } 38 | 39 | func RoundTimeUp(ts time.Time, resolution time.Duration) time.Time { 40 | rounded := ts.Round(resolution) 41 | if rounded.Before(ts) { 42 | rounded = rounded.Add(1 * resolution) 43 | } 44 | return rounded 45 | } 46 | 47 | func RoundTimeDown(ts time.Time, resolution time.Duration) time.Time { 48 | rounded := ts.Round(resolution) 49 | if rounded.After(ts) { 50 | rounded = rounded.Add(-1 * resolution) 51 | } 52 | return rounded 53 | } 54 | 55 | func RoundTimeUntilUp(ts time.Time, resolution time.Duration, until time.Time) time.Time { 56 | if ts.IsZero() { 57 | return ts 58 | } 59 | if until.IsZero() { 60 | return RoundTimeUp(ts, resolution) 61 | } 62 | delta := until.Sub(ts) 63 | periods := -1 * time.Duration(math.Floor(float64(delta)/float64(resolution))) 64 | return until.Add(periods * resolution) 65 | } 66 | 67 | func RoundTimeUntilDown(ts time.Time, resolution time.Duration, until time.Time) time.Time { 68 | if ts.IsZero() { 69 | return ts 70 | } 71 | if until.IsZero() { 72 | return RoundTimeDown(ts, resolution) 73 | } 74 | delta := until.Sub(ts) 75 | periods := -1 * time.Duration(math.Ceil(float64(delta)/float64(resolution))) 76 | return until.Add(periods * resolution) 77 | } 78 | -------------------------------------------------------------------------------- /encoding/time_test.go: -------------------------------------------------------------------------------- 1 | package encoding 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestRoundTimeUp(t *testing.T) { 10 | ti := time.Date(2015, 1, 1, 0, 0, 2, 0, time.UTC) 11 | to := time.Date(2015, 1, 1, 0, 1, 0, 0, time.UTC) 12 | assert.Equal(t, to, RoundTimeUp(ti, time.Minute)) 13 | } 14 | 15 | func TestRoundTimeDown(t *testing.T) { 16 | ti := time.Date(2015, 1, 1, 0, 0, 2, 0, time.UTC) 17 | to := time.Date(2015, 1, 1, 0, 0, 0, 0, time.UTC) 18 | assert.Equal(t, to, RoundTimeDown(ti, time.Minute)) 19 | } 20 | 21 | func TestRoundTimeUntilUp(t *testing.T) { 22 | until := time.Date(2015, 1, 1, 0, 0, 1, 0, time.UTC) 23 | ti := time.Date(2015, 1, 1, 0, 0, 2, 0, time.UTC) 24 | to := time.Date(2015, 1, 1, 0, 1, 1, 0, time.UTC) 25 | assert.Equal(t, to, RoundTimeUntilUp(ti, time.Minute, until)) 26 | } 27 | 28 | func TestRoundTimeUntilDown(t *testing.T) { 29 | until := time.Date(2015, 1, 1, 0, 0, 1, 0, time.UTC) 30 | ti := time.Date(2015, 1, 1, 0, 0, 2, 0, time.UTC) 31 | to := time.Date(2015, 1, 1, 0, 0, 1, 0, time.UTC) 32 | assert.Equal(t, to, RoundTimeUntilDown(ti, time.Minute, until)) 33 | } 34 | 35 | func TestRoundTimeUntilUpZeroUntil(t *testing.T) { 36 | ti := time.Date(2015, 1, 1, 0, 0, 2, 0, time.UTC) 37 | to := RoundTimeUp(ti, time.Minute) 38 | assert.Equal(t, to, RoundTimeUntilUp(ti, time.Minute, time.Time{})) 39 | } 40 | 41 | func TestRoundTimeUntilDownZeroUntil(t *testing.T) { 42 | ti := time.Date(2015, 1, 1, 0, 0, 2, 0, time.UTC) 43 | to := RoundTimeDown(ti, time.Minute) 44 | assert.Equal(t, to, RoundTimeUntilDown(ti, time.Minute, time.Time{})) 45 | } 46 | 47 | func TestRoundTimeUntilUpZeroTime(t *testing.T) { 48 | until := time.Date(2015, 1, 1, 0, 0, 1, 0, time.UTC) 49 | ti := time.Time{} 50 | to := ti 51 | assert.Equal(t, to, RoundTimeUntilUp(ti, 13, until)) 52 | } 53 | 54 | func TestRoundTimeUntilDownZeroTime(t *testing.T) { 55 | until := time.Date(2015, 1, 1, 0, 0, 1, 0, time.UTC) 56 | ti := time.Time{} 57 | to := ti 58 | assert.Equal(t, to, RoundTimeUntilDown(ti, 13, until)) 59 | } 60 | 61 | func TestTimeFromInt(t *testing.T) { 62 | t1 := time.Date(2018, 1, 1, 1, 1, 1, 0, time.UTC) 63 | t2 := TimeFromInt(t1.UnixNano()).In(time.UTC) 64 | assert.Equal(t, t1, t2) 65 | } 66 | 67 | func TestTimeFromMillis(t *testing.T) { 68 | t1 := time.Date(2018, 1, 1, 1, 1, 1, 0, time.UTC) 69 | t2 := TimeFromMillis(t1.UnixNano() / nanosPerMilli).In(time.UTC) 70 | assert.Equal(t, t1, t2) 71 | } 72 | -------------------------------------------------------------------------------- /expr/aggregate.go: -------------------------------------------------------------------------------- 1 | package expr 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "reflect" 7 | "time" 8 | 9 | "github.com/getlantern/goexpr" 10 | "github.com/getlantern/msgpack" 11 | ) 12 | 13 | var aggregates = make(map[string]func(wrapped interface{}) *aggregate) 14 | 15 | func aggregateFor(name string, wrapped interface{}) *aggregate { 16 | ctor, found := aggregates[name] 17 | if !found { 18 | return nil 19 | } 20 | return ctor(wrapped) 21 | } 22 | 23 | func registerAggregate(name string, update updateFN, merge updateFN) { 24 | aggregates[name] = func(wrapped interface{}) *aggregate { 25 | return &aggregate{ 26 | Name: name, 27 | Wrapped: exprFor(wrapped), 28 | update: update, 29 | merge: merge, 30 | } 31 | } 32 | } 33 | 34 | type updateFN func(wasSet bool, current float64, next float64) float64 35 | 36 | type aggregate struct { 37 | Name string 38 | Wrapped Expr 39 | update updateFN 40 | merge updateFN 41 | } 42 | 43 | func (e *aggregate) Validate() error { 44 | return validateWrappedInAggregate(e.Wrapped) 45 | } 46 | 47 | func validateWrappedInAggregate(wrapped Expr) error { 48 | if wrapped == nil { 49 | return fmt.Errorf("Aggregate cannot wrap nil expression") 50 | } 51 | typeOfWrapped := reflect.TypeOf(wrapped) 52 | if typeOfWrapped != fieldType && typeOfWrapped != constType && typeOfWrapped != boundedType { 53 | return fmt.Errorf("Aggregate can only wrap field and constant expressions, not %v", typeOfWrapped) 54 | } 55 | return wrapped.Validate() 56 | } 57 | 58 | func (e *aggregate) EncodedWidth() int { 59 | return 1 + width64bits + e.Wrapped.EncodedWidth() 60 | } 61 | 62 | func (e *aggregate) Shift() time.Duration { 63 | return e.Wrapped.Shift() 64 | } 65 | 66 | func (e *aggregate) Update(b []byte, params Params, metadata goexpr.Params) ([]byte, float64, bool) { 67 | value, wasSet, more := e.load(b) 68 | remain, wrappedValue, updated := e.Wrapped.Update(more, params, metadata) 69 | if updated { 70 | value = e.update(wasSet, value, wrappedValue) 71 | e.save(b, value) 72 | } 73 | return remain, value, updated 74 | } 75 | 76 | func (e *aggregate) Merge(b []byte, x []byte, y []byte) ([]byte, []byte, []byte) { 77 | valueX, xWasSet, remainX := e.load(x) 78 | valueY, yWasSet, remainY := e.load(y) 79 | if !xWasSet { 80 | if yWasSet { 81 | // Use valueY 82 | b = e.save(b, valueY) 83 | } else { 84 | // Nothing to save, just advance 85 | b = b[width64bits+1:] 86 | } 87 | } else { 88 | if yWasSet { 89 | // Update valueX from valueY 90 | valueX = e.merge(true, valueX, valueY) 91 | } 92 | b = e.save(b, valueX) 93 | } 94 | return b, remainX, remainY 95 | } 96 | 97 | func (e *aggregate) SubMergers(subs []Expr) []SubMerge { 98 | result := make([]SubMerge, len(subs)) 99 | for i, sub := range subs { 100 | if e.String() == sub.String() { 101 | result[i] = e.subMerge 102 | } 103 | } 104 | return result 105 | } 106 | 107 | func (e *aggregate) subMerge(data []byte, other []byte, otherRes time.Duration, metadata goexpr.Params) { 108 | e.Merge(data, data, other) 109 | } 110 | 111 | func (e *aggregate) Get(b []byte) (float64, bool, []byte) { 112 | return e.load(b) 113 | } 114 | 115 | func (e *aggregate) load(b []byte) (float64, bool, []byte) { 116 | remain := b[width64bits+1:] 117 | value := float64(0) 118 | wasSet := b[0] == 1 119 | if wasSet { 120 | value = math.Float64frombits(binaryEncoding.Uint64(b[1:])) 121 | } 122 | return value, wasSet, remain 123 | } 124 | 125 | func (e *aggregate) save(b []byte, value float64) []byte { 126 | b[0] = 1 127 | binaryEncoding.PutUint64(b[1:], math.Float64bits(value)) 128 | return b[width64bits+1:] 129 | } 130 | 131 | func (e *aggregate) IsConstant() bool { 132 | return e.Wrapped.IsConstant() 133 | } 134 | 135 | func (e *aggregate) DeAggregate() Expr { 136 | return e.Wrapped.DeAggregate() 137 | } 138 | 139 | func (e *aggregate) String() string { 140 | return fmt.Sprintf("%v(%v)", e.Name, e.Wrapped) 141 | } 142 | 143 | func (e *aggregate) DecodeMsgpack(dec *msgpack.Decoder) error { 144 | m := make(map[string]interface{}) 145 | err := dec.Decode(&m) 146 | if err != nil { 147 | return err 148 | } 149 | e2 := aggregateFor(m["Name"].(string), m["Wrapped"].(Expr)) 150 | if e2 == nil { 151 | return fmt.Errorf("Unknown aggregate %v", m["Name"]) 152 | } 153 | e.Name = e2.Name 154 | e.Wrapped = e2.Wrapped 155 | e.update = e2.update 156 | e.merge = e2.merge 157 | return nil 158 | } 159 | -------------------------------------------------------------------------------- /expr/aggregates.go: -------------------------------------------------------------------------------- 1 | package expr 2 | 3 | func init() { 4 | registerAggregate("SUM", func(wasSet bool, current float64, next float64) float64 { 5 | return current + next 6 | }, func(wasSet bool, current float64, next float64) float64 { 7 | return current + next 8 | }) 9 | 10 | registerAggregate("MIN", func(wasSet bool, current float64, next float64) float64 { 11 | if !wasSet { 12 | return next 13 | } 14 | if next < current { 15 | return next 16 | } 17 | return current 18 | }, func(wasSet bool, current float64, next float64) float64 { 19 | if !wasSet { 20 | return next 21 | } 22 | if next < current { 23 | return next 24 | } 25 | return current 26 | }) 27 | 28 | registerAggregate("MAX", func(wasSet bool, current float64, next float64) float64 { 29 | if !wasSet { 30 | return next 31 | } 32 | if next > current { 33 | return next 34 | } 35 | return current 36 | }, func(wasSet bool, current float64, next float64) float64 { 37 | if !wasSet { 38 | return next 39 | } 40 | if next > current { 41 | return next 42 | } 43 | return current 44 | }) 45 | 46 | registerAggregate("COUNT", func(wasSet bool, current float64, next float64) float64 { 47 | return current + 1 48 | }, func(wasSet bool, current float64, next float64) float64 { 49 | return current + next 50 | }) 51 | } 52 | 53 | // SUM creates an Expr that obtains its value by summing the given expressions 54 | // or fields. 55 | func SUM(expr interface{}) Expr { 56 | return aggregateFor("SUM", expr) 57 | } 58 | 59 | // MIN creates an Expr that keeps track of the minimum value of the wrapped 60 | // expression or field. 61 | func MIN(expr interface{}) Expr { 62 | return aggregateFor("MIN", expr) 63 | } 64 | 65 | // MAX creates an Expr that keeps track of the maximum value of the wrapped 66 | // expression or field. 67 | func MAX(expr interface{}) Expr { 68 | return aggregateFor("MAX", expr) 69 | } 70 | 71 | // COUNT creates an Expr that counts the number of values. 72 | func COUNT(expr interface{}) Expr { 73 | return aggregateFor("COUNT", expr) 74 | } 75 | -------------------------------------------------------------------------------- /expr/aggregates_test.go: -------------------------------------------------------------------------------- 1 | package expr 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/getlantern/goexpr" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestSUM(t *testing.T) { 11 | doTestAggregate(t, SUM(boundedA()), 15.6) 12 | } 13 | 14 | func TestMIN(t *testing.T) { 15 | doTestAggregate(t, MIN(boundedA()), 2.4) 16 | } 17 | 18 | func TestMAX(t *testing.T) { 19 | doTestAggregate(t, MAX(boundedA()), 8.8) 20 | } 21 | 22 | func TestCOUNT(t *testing.T) { 23 | doTestAggregate(t, COUNT("b"), 3) 24 | } 25 | 26 | func TestAVG(t *testing.T) { 27 | doTestAggregate(t, AVG(boundedA()), 5.2) 28 | } 29 | 30 | func TestWAVG(t *testing.T) { 31 | doTestAggregate(t, WAVG(boundedA(), "b"), 7.52) 32 | } 33 | 34 | func TestSUMConditional(t *testing.T) { 35 | ex := IF(goexpr.Param("i"), SUM("b")) 36 | doTestAggregate(t, ex, 1) 37 | } 38 | 39 | func TestValidateAggregate(t *testing.T) { 40 | sum := SUM(MULT(CONST(1), CONST(2))) 41 | assert.Error(t, sum.Validate()) 42 | avg := AVG(MULT(CONST(1), CONST(2))) 43 | assert.Error(t, avg.Validate()) 44 | wavg := WAVG(FIELD("b"), SUM(FIELD("c"))) 45 | assert.Error(t, wavg.Validate()) 46 | ok := SUM(CONST(1)) 47 | assert.NoError(t, ok.Validate()) 48 | ok2 := AVG(FIELD("b")) 49 | assert.NoError(t, ok2.Validate()) 50 | } 51 | 52 | func boundedA() Expr { 53 | return BOUNDED("a", 0.1, 8.8) 54 | } 55 | 56 | func doTestAggregate(t *testing.T, e Expr, expected float64) { 57 | e = msgpacked(t, e) 58 | params1 := Map{ 59 | "a": 4.4, 60 | } 61 | md1 := goexpr.MapParams{} 62 | params2 := Map{ 63 | "a": 8.8, 64 | "b": 0.8, 65 | } 66 | md2 := goexpr.MapParams{ 67 | "i": true, 68 | } 69 | params3 := Map{ 70 | "a": 2.4, 71 | "b": 0.2, 72 | } 73 | md3 := goexpr.MapParams{} 74 | params4 := Map{ 75 | "b": 0.2, 76 | "i": 1, 77 | } 78 | md4 := goexpr.MapParams{ 79 | "i": true, 80 | } 81 | // params5 and params6 will be ignored because they fall outside of the bounds 82 | params5 := Map{ 83 | "a": 0.01, 84 | } 85 | params6 := Map{ 86 | "a": 8.9, 87 | } 88 | b := make([]byte, e.EncodedWidth()) 89 | e.Update(b, params1, md1) 90 | e.Update(b, params2, md2) 91 | e.Update(b, params3, md3) 92 | e.Update(b, params4, md4) 93 | e.Update(b, params5, md1) 94 | e.Update(b, params6, md1) 95 | val, wasSet, _ := e.Get(b) 96 | if assert.True(t, wasSet) { 97 | AssertFloatEquals(t, expected, val) 98 | } 99 | 100 | // Test Merging 101 | b1 := make([]byte, e.EncodedWidth()) 102 | e.Update(b1, params1, md1) 103 | e.Update(b1, params2, md2) 104 | b2 := make([]byte, e.EncodedWidth()) 105 | e.Update(b2, params3, md3) 106 | e.Update(b2, params4, md4) 107 | b3 := make([]byte, e.EncodedWidth()) 108 | e.Merge(b3, b1, b2) 109 | val, wasSet, _ = e.Get(b3) 110 | if assert.True(t, wasSet) { 111 | AssertFloatEquals(t, expected, val) 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /expr/avg.go: -------------------------------------------------------------------------------- 1 | package expr 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "time" 7 | 8 | "github.com/getlantern/goexpr" 9 | ) 10 | 11 | // AVG creates an Expr that obtains its value as the arithmetic mean over the 12 | // given value. 13 | func AVG(val interface{}) Expr { 14 | return WAVG(val, CONST(1)) 15 | } 16 | 17 | // WAVG creates an Expr that obtains its value as the weighted arithmetic mean 18 | // over the given value weighted by the given weight. 19 | func WAVG(val interface{}, weight interface{}) Expr { 20 | return &avg{exprFor(val), exprFor(weight)} 21 | } 22 | 23 | type avg struct { 24 | Value Expr 25 | Weight Expr 26 | } 27 | 28 | func (e *avg) Validate() error { 29 | err := validateWrappedInAggregate(e.Value) 30 | if err != nil { 31 | return err 32 | } 33 | if e.Weight.EncodedWidth() > 0 { 34 | return fmt.Errorf("Weight expression %v must be a constant or directly derived from a field", e.Weight) 35 | } 36 | return nil 37 | } 38 | 39 | func (e *avg) EncodedWidth() int { 40 | return width64bits*2 + 1 + e.Value.EncodedWidth() 41 | } 42 | 43 | func (e *avg) Shift() time.Duration { 44 | a := e.Value.Shift() 45 | b := e.Weight.Shift() 46 | if a < b { 47 | return a 48 | } 49 | return b 50 | } 51 | 52 | func (e *avg) Update(b []byte, params Params, metadata goexpr.Params) ([]byte, float64, bool) { 53 | count, total, _, remain := e.load(b) 54 | remain, value, updated := e.Value.Update(remain, params, metadata) 55 | remain, weight, _ := e.Weight.Update(remain, params, metadata) 56 | if updated { 57 | count += weight 58 | total += value * weight 59 | e.save(b, count, total) 60 | } 61 | return remain, e.calc(count, total), updated 62 | } 63 | 64 | func (e *avg) Merge(b []byte, x []byte, y []byte) ([]byte, []byte, []byte) { 65 | countX, totalX, xWasSet, remainX := e.load(x) 66 | countY, totalY, yWasSet, remainY := e.load(y) 67 | if !xWasSet { 68 | if yWasSet { 69 | // Use valueY 70 | b = e.save(b, countY, totalY) 71 | } else { 72 | // Nothing to save, just advance 73 | b = b[width64bits*2+1:] 74 | } 75 | } else { 76 | if yWasSet { 77 | countX += countY 78 | totalX += totalY 79 | } 80 | b = e.save(b, countX, totalX) 81 | } 82 | return b, remainX, remainY 83 | } 84 | 85 | func (e *avg) SubMergers(subs []Expr) []SubMerge { 86 | result := make([]SubMerge, 0, len(subs)) 87 | for _, sub := range subs { 88 | var sm SubMerge 89 | if e.String() == sub.String() { 90 | sm = e.subMerge 91 | } 92 | result = append(result, sm) 93 | } 94 | return result 95 | } 96 | 97 | func (e *avg) subMerge(data []byte, other []byte, otherRes time.Duration, metadata goexpr.Params) { 98 | e.Merge(data, data, other) 99 | } 100 | 101 | func (e *avg) Get(b []byte) (float64, bool, []byte) { 102 | count, total, wasSet, remain := e.load(b) 103 | if !wasSet { 104 | return 0, wasSet, remain 105 | } 106 | return e.calc(count, total), wasSet, remain 107 | } 108 | 109 | func (e *avg) calc(count float64, total float64) float64 { 110 | if count == 0 { 111 | return 0 112 | } 113 | return total / count 114 | } 115 | 116 | func (e *avg) load(b []byte) (float64, float64, bool, []byte) { 117 | remain := b[width64bits*2+1:] 118 | wasSet := b[0] == 1 119 | count := float64(0) 120 | total := float64(0) 121 | if wasSet { 122 | count = math.Float64frombits(binaryEncoding.Uint64(b[1:])) 123 | total = math.Float64frombits(binaryEncoding.Uint64(b[width64bits+1:])) 124 | } 125 | return count, total, wasSet, remain 126 | } 127 | 128 | func (e *avg) save(b []byte, count float64, total float64) []byte { 129 | b[0] = 1 130 | binaryEncoding.PutUint64(b[1:], math.Float64bits(count)) 131 | binaryEncoding.PutUint64(b[width64bits+1:], math.Float64bits(total)) 132 | return b[width64bits*2+1:] 133 | } 134 | 135 | func (e *avg) IsConstant() bool { 136 | return e.Value.IsConstant() 137 | } 138 | 139 | func (e *avg) DeAggregate() Expr { 140 | return e.Value.DeAggregate() 141 | } 142 | 143 | func (e *avg) String() string { 144 | return fmt.Sprintf("AVG(%v)", e.Value) 145 | } 146 | -------------------------------------------------------------------------------- /expr/binary.go: -------------------------------------------------------------------------------- 1 | package expr 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "time" 7 | 8 | "github.com/getlantern/goexpr" 9 | "github.com/getlantern/msgpack" 10 | ) 11 | 12 | var binaryExprs = make(map[string]func(left interface{}, right interface{}) *binaryExpr) 13 | 14 | func binaryExprFor(op string, left interface{}, right interface{}) *binaryExpr { 15 | ctor, found := binaryExprs[op] 16 | if !found { 17 | return nil 18 | } 19 | return ctor(left, right) 20 | } 21 | 22 | func registerBinaryExpr(op string, calc calcFN) { 23 | binaryExprs[op] = func(left interface{}, right interface{}) *binaryExpr { 24 | return &binaryExpr{ 25 | Op: op, 26 | Left: exprFor(left), 27 | Right: exprFor(right), 28 | calc: calc, 29 | } 30 | } 31 | } 32 | 33 | type calcFN func(left float64, right float64) float64 34 | 35 | type binaryExpr struct { 36 | Op string 37 | Left Expr 38 | Right Expr 39 | DeAggregated bool 40 | calc calcFN 41 | } 42 | 43 | func (e *binaryExpr) Validate() error { 44 | err := e.validateWrappedInBinary(e.Left) 45 | if err == nil { 46 | err = e.validateWrappedInBinary(e.Right) 47 | } 48 | return err 49 | } 50 | 51 | func (e *binaryExpr) validateWrappedInBinary(wrapped Expr) error { 52 | if wrapped == nil { 53 | return fmt.Errorf("Binary expression cannot wrap nil expression") 54 | } 55 | typeOfWrapped := reflect.TypeOf(wrapped) 56 | if typeOfWrapped == aggregateType || 57 | typeOfWrapped == ifType || 58 | typeOfWrapped == avgType || 59 | typeOfWrapped == constType || 60 | typeOfWrapped == shiftType || 61 | typeOfWrapped == unaryMathType || 62 | typeOfWrapped == percentileType || 63 | typeOfWrapped == percentileOptimizedType { 64 | return nil 65 | } 66 | if typeOfWrapped == binaryType { 67 | return wrapped.Validate() 68 | } 69 | if e.DeAggregated { 70 | return nil 71 | } 72 | return fmt.Errorf("Binary expression must wrap only aggregate, if, constant or shift expressions, or other binary expressions that wrap only aggregate or constant expressions, not %v of type %v", wrapped, typeOfWrapped) 73 | } 74 | 75 | func (e *binaryExpr) EncodedWidth() int { 76 | return e.Left.EncodedWidth() + e.Right.EncodedWidth() 77 | } 78 | 79 | func (e *binaryExpr) Shift() time.Duration { 80 | a := e.Left.Shift() 81 | b := e.Right.Shift() 82 | if a < b { 83 | return a 84 | } 85 | return b 86 | } 87 | 88 | func (e *binaryExpr) Update(b []byte, params Params, metadata goexpr.Params) ([]byte, float64, bool) { 89 | remain, leftValue, updatedLeft := e.Left.Update(b, params, metadata) 90 | remain, rightValue, updatedRight := e.Right.Update(remain, params, metadata) 91 | updated := updatedLeft || updatedRight 92 | return remain, e.calc(leftValue, rightValue), updated 93 | } 94 | 95 | func (e *binaryExpr) Merge(b []byte, x []byte, y []byte) ([]byte, []byte, []byte) { 96 | remainB, remainX, remainY := e.Left.Merge(b, x, y) 97 | return e.Right.Merge(remainB, remainX, remainY) 98 | } 99 | 100 | func (e *binaryExpr) SubMergers(subs []Expr) []SubMerge { 101 | result := make([]SubMerge, len(subs)) 102 | // See if any of the subexpressions match top level and if so, ignore others 103 | for i, sub := range subs { 104 | if e.String() == sub.String() { 105 | result[i] = e.subMerge 106 | return result 107 | } 108 | } 109 | 110 | // None of sub expressions match top level, build combined ones 111 | left := e.Left.SubMergers(subs) 112 | right := e.Right.SubMergers(subs) 113 | for i := range subs { 114 | result[i] = combinedSubMerge(left[i], e.Left.EncodedWidth(), right[i]) 115 | } 116 | 117 | return result 118 | } 119 | 120 | func (e *binaryExpr) subMerge(data []byte, other []byte, otherRes time.Duration, metadata goexpr.Params) { 121 | e.Merge(data, data, other) 122 | } 123 | 124 | func combinedSubMerge(left SubMerge, width int, right SubMerge) SubMerge { 125 | // Optimization - if left and right are both nil, just return nil 126 | if left == nil && right == nil { 127 | return nil 128 | } 129 | if right == nil { 130 | return left 131 | } 132 | if left == nil { 133 | return func(data []byte, other []byte, otherRes time.Duration, metadata goexpr.Params) { 134 | right(data[width:], other, otherRes, metadata) 135 | } 136 | } 137 | return func(data []byte, other []byte, otherRes time.Duration, metadata goexpr.Params) { 138 | left(data, other, otherRes, metadata) 139 | right(data[width:], other, otherRes, metadata) 140 | } 141 | } 142 | 143 | func (e *binaryExpr) Get(b []byte) (float64, bool, []byte) { 144 | valueLeft, leftWasSet, remain := e.Left.Get(b) 145 | valueRight, rightWasSet, remain := e.Right.Get(remain) 146 | if !leftWasSet && !rightWasSet { 147 | return 0, false, remain 148 | } 149 | return e.calc(valueLeft, valueRight), true, remain 150 | } 151 | 152 | func (e *binaryExpr) IsConstant() bool { 153 | return e.Left.IsConstant() && e.Right.IsConstant() 154 | } 155 | 156 | func (e *binaryExpr) DeAggregate() Expr { 157 | result := binaryExprFor(e.Op, e.Left.DeAggregate(), e.Right.DeAggregate()) 158 | result.DeAggregated = true 159 | return result 160 | } 161 | 162 | func (e *binaryExpr) String() string { 163 | return fmt.Sprintf("(%v %v %v)", e.Left, e.Op, e.Right) 164 | } 165 | 166 | func (e *binaryExpr) DecodeMsgpack(dec *msgpack.Decoder) error { 167 | m := make(map[string]interface{}) 168 | err := dec.Decode(&m) 169 | if err != nil { 170 | return err 171 | } 172 | e2 := binaryExprFor(m["Op"].(string), m["Left"].(Expr), m["Right"].(Expr)) 173 | if e2 == nil { 174 | return fmt.Errorf("Unknown binary expression %v", m["Op"]) 175 | } 176 | e.Op = e2.Op 177 | e.Left = e2.Left 178 | e.Right = e2.Right 179 | e.calc = e2.calc 180 | return nil 181 | } 182 | -------------------------------------------------------------------------------- /expr/bounded.go: -------------------------------------------------------------------------------- 1 | package expr 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/getlantern/goexpr" 8 | "github.com/getlantern/msgpack" 9 | ) 10 | 11 | // BOUNDED bounds the given expression to min <= val <= max. Any values 12 | // that fall outside of the bounds will appear as unset (i.e. they are 13 | // discarded.) 14 | func BOUNDED(expr interface{}, min float64, max float64) Expr { 15 | wrapped := exprFor(expr) 16 | return &bounded{ 17 | wrapped: wrapped, 18 | min: min, 19 | max: max, 20 | } 21 | } 22 | 23 | type bounded struct { 24 | wrapped Expr 25 | min float64 26 | max float64 27 | } 28 | 29 | func (e *bounded) Validate() error { 30 | return e.wrapped.Validate() 31 | } 32 | 33 | func (e *bounded) EncodedWidth() int { 34 | return e.wrapped.EncodedWidth() 35 | } 36 | 37 | func (e *bounded) Shift() time.Duration { 38 | return e.wrapped.Shift() 39 | } 40 | 41 | func (e *bounded) Update(b []byte, params Params, metadata goexpr.Params) ([]byte, float64, bool) { 42 | remain, value, updated := e.wrapped.Update(b, params, metadata) 43 | if !e.test(value) { 44 | updated = false 45 | value = 0 46 | } 47 | return remain, value, updated 48 | } 49 | 50 | func (e *bounded) test(val float64) bool { 51 | return val >= e.min && val <= e.max 52 | } 53 | 54 | func (e *bounded) Merge(b []byte, x []byte, y []byte) ([]byte, []byte, []byte) { 55 | return e.wrapped.Merge(b, x, y) 56 | } 57 | 58 | func (e *bounded) SubMergers(subs []Expr) []SubMerge { 59 | return e.wrapped.SubMergers(subs) 60 | } 61 | 62 | func (e *bounded) Get(b []byte) (float64, bool, []byte) { 63 | val, wasSet, remain := e.wrapped.Get(b) 64 | if !wasSet || !e.test(val) { 65 | return 0, false, remain 66 | } 67 | return val, wasSet, remain 68 | } 69 | 70 | func (e *bounded) IsConstant() bool { 71 | return e.wrapped.IsConstant() 72 | } 73 | 74 | func (e *bounded) DeAggregate() Expr { 75 | return BOUNDED(e.wrapped.DeAggregate(), e.min, e.max) 76 | } 77 | 78 | func (e *bounded) String() string { 79 | return fmt.Sprintf("BOUNDED(%v, %v, %v)", e.wrapped, e.min, e.max) 80 | } 81 | 82 | var _ msgpack.CustomEncoder = (*bounded)(nil) 83 | var _ msgpack.CustomDecoder = (*bounded)(nil) 84 | 85 | func (e *bounded) EncodeMsgpack(enc *msgpack.Encoder) error { 86 | return enc.Encode(e.wrapped, e.min, e.max) 87 | } 88 | 89 | func (e *bounded) DecodeMsgpack(dec *msgpack.Decoder) error { 90 | return dec.Decode(&e.wrapped, &e.min, &e.max) 91 | } 92 | -------------------------------------------------------------------------------- /expr/calcs.go: -------------------------------------------------------------------------------- 1 | package expr 2 | 3 | import ( 4 | "math" 5 | ) 6 | 7 | func init() { 8 | registerBinaryExpr("+", func(left float64, right float64) float64 { 9 | return left + right 10 | }) 11 | 12 | registerBinaryExpr("-", func(left float64, right float64) float64 { 13 | return left - right 14 | }) 15 | 16 | registerBinaryExpr("*", func(left float64, right float64) float64 { 17 | return left * right 18 | }) 19 | 20 | registerBinaryExpr("/", func(left float64, right float64) float64 { 21 | if right == 0 { 22 | if left == 0 { 23 | return 0 24 | } 25 | return math.MaxFloat64 26 | } 27 | return left / right 28 | }) 29 | } 30 | 31 | // ADD creates an Expr that obtains its value by adding right and left. 32 | func ADD(left interface{}, right interface{}) Expr { 33 | return binaryExprFor("+", left, right) 34 | } 35 | 36 | // SUB creates an Expr that obtains its value by subtracting right from left. 37 | func SUB(left interface{}, right interface{}) Expr { 38 | return binaryExprFor("-", left, right) 39 | } 40 | 41 | // MULT creates an Expr that obtains its value by multiplying right and left. 42 | func MULT(left interface{}, right interface{}) Expr { 43 | return binaryExprFor("*", left, right) 44 | } 45 | 46 | // DIV creates an Expr that obtains its value by dividing left by right. If 47 | // right is 0, this returns 0. 48 | func DIV(left interface{}, right interface{}) Expr { 49 | return binaryExprFor("/", left, right) 50 | } 51 | -------------------------------------------------------------------------------- /expr/calcs_test.go: -------------------------------------------------------------------------------- 1 | package expr 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "math" 6 | "testing" 7 | ) 8 | 9 | func TestADD(t *testing.T) { 10 | doTestCalc(t, ADD("a", "b"), 13.2) 11 | } 12 | 13 | func TestSUB(t *testing.T) { 14 | doTestCalc(t, SUB("a", "b"), 4.4) 15 | } 16 | 17 | func TestMULT(t *testing.T) { 18 | doTestCalc(t, MULT("a", "b"), 38.72) 19 | } 20 | 21 | func TestDIV(t *testing.T) { 22 | doTestCalc(t, DIV("a", "b"), 2) 23 | } 24 | 25 | func TestDIVZero(t *testing.T) { 26 | doTestCalc(t, DIV("a", "c"), math.MaxFloat64) 27 | } 28 | 29 | func TestDIVZeroZero(t *testing.T) { 30 | doTestCalc(t, DIV("c", "c"), 0) 31 | } 32 | 33 | func TestValidateBinary(t *testing.T) { 34 | bad := MULT(FIELD("a"), FIELD("b")) 35 | assert.Error(t, bad.Validate()) 36 | dok := bad.DeAggregate() 37 | assert.NoError(t, dok.Validate()) 38 | ok := MULT(CONST(1), SUM(FIELD("b"))) 39 | assert.NoError(t, ok.Validate()) 40 | ok2 := MULT(CONST(1), AVG(FIELD("b"))) 41 | assert.NoError(t, ok2.Validate()) 42 | ok3 := MULT(CONST(1), ADD(AVG(FIELD("b")), CONST(3))) 43 | assert.NoError(t, ok3.Validate()) 44 | ok4 := MULT(CONST(1), ADD(AVG(FIELD("b")), GT(CONST(3), SUM("c")))) 45 | assert.NoError(t, ok4.Validate()) 46 | } 47 | 48 | func doTestCalc(t *testing.T, e Expr, expected float64) { 49 | e = msgpacked(t, e) 50 | params := Map{ 51 | "a": 8.8, 52 | "b": 4.4, 53 | "c": 0, 54 | "d": 1.1, 55 | } 56 | 57 | b := make([]byte, e.EncodedWidth()) 58 | _, val, _ := e.Update(b, params, nil) 59 | AssertFloatEquals(t, expected, val) 60 | } 61 | -------------------------------------------------------------------------------- /expr/combined_test.go: -------------------------------------------------------------------------------- 1 | package expr 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/getlantern/goexpr" 7 | "github.com/getlantern/msgpack" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestCombined(t *testing.T) { 12 | avgA := AVG("a") 13 | avgB := AVG("b") 14 | mult := MULT(avgA, avgB) 15 | count := COUNT("b") 16 | ge, err := goexpr.Binary("=", goexpr.Constant(0), goexpr.Constant(0)) 17 | if !assert.NoError(t, err) { 18 | return 19 | } 20 | ie := IF(ge, DIV(mult, count)) 21 | e := msgpacked(t, ie) 22 | params1 := Map{ 23 | "a": 2, 24 | "b": 10, 25 | } 26 | md1 := goexpr.MapParams{} 27 | params2 := Map{ 28 | "a": 4, 29 | "b": 20, 30 | } 31 | md2 := goexpr.MapParams{} 32 | params3 := Map{ 33 | "a": 0, 34 | "b": 3, 35 | } 36 | md3 := goexpr.MapParams{} 37 | 38 | b := make([]byte, e.EncodedWidth()) 39 | e.Update(b, params1, md1) 40 | e.Update(b, params2, md2) 41 | val, _, _ := e.Get(b) 42 | AssertFloatEquals(t, 22.5, val) 43 | 44 | b2 := make([]byte, e.EncodedWidth()) 45 | e.Update(b2, params3, md3) 46 | b3 := make([]byte, e.EncodedWidth()) 47 | e.Merge(b3, b, b2) 48 | val, _, _ = e.Get(b3) 49 | AssertFloatEquals(t, 7.33333333, val) 50 | 51 | // Test SubMerge 52 | bavgA := make([]byte, avgA.EncodedWidth()) 53 | bavgB := make([]byte, avgB.EncodedWidth()) 54 | bmult := make([]byte, mult.EncodedWidth()) 55 | bcount := make([]byte, count.EncodedWidth()) 56 | avgA.Update(bavgA, params1, md1) 57 | avgB.Update(bavgB, params1, md1) 58 | mult.Update(bmult, params1, md1) 59 | count.Update(bcount, params1, md1) 60 | avgA.Update(bavgA, params2, md2) 61 | avgB.Update(bavgB, params2, md2) 62 | mult.Update(bmult, params2, md2) 63 | count.Update(bcount, params2, md2) 64 | avgA.Update(bavgA, params3, md3) 65 | avgB.Update(bavgB, params3, md3) 66 | mult.Update(bmult, params3, md3) 67 | count.Update(bcount, params3, md3) 68 | be := make([]byte, e.EncodedWidth()) 69 | fields := []Expr{avgA, avgB, mult, count} 70 | data := [][]byte{bavgA, bavgB, bmult, bcount} 71 | sms := e.SubMergers(fields) 72 | for i, sm := range sms { 73 | if sm != nil { 74 | sm(be, data[i], 0, nil) 75 | } 76 | } 77 | val, _, _ = e.Get(be) 78 | AssertFloatEquals(t, 7.33333333, val) 79 | } 80 | 81 | func msgpacked(t *testing.T, e Expr) Expr { 82 | b, err := msgpack.Marshal(e) 83 | if !assert.NoError(t, err) { 84 | t.FailNow() 85 | } 86 | var e2 Expr 87 | err = msgpack.Unmarshal(b, &e2) 88 | if !assert.NoError(t, err) { 89 | t.FailNow() 90 | } 91 | return e2 92 | } 93 | -------------------------------------------------------------------------------- /expr/conds.go: -------------------------------------------------------------------------------- 1 | package expr 2 | 3 | func init() { 4 | registerCond("<", func(left float64, right float64) bool { 5 | return left < right 6 | }) 7 | 8 | registerCond("<=", func(left float64, right float64) bool { 9 | return left <= right 10 | }) 11 | 12 | registerCond("=", func(left float64, right float64) bool { 13 | return left == right 14 | }) 15 | 16 | registerCond("<>", func(left float64, right float64) bool { 17 | return left != right 18 | }) 19 | 20 | registerCond(">=", func(left float64, right float64) bool { 21 | return left >= right 22 | }) 23 | 24 | registerCond(">", func(left float64, right float64) bool { 25 | return left > right 26 | }) 27 | 28 | registerCond("AND", func(left float64, right float64) bool { 29 | return left > 0 && right > 0 30 | }) 31 | 32 | registerCond("OR", func(left float64, right float64) bool { 33 | return left > 0 || right > 0 34 | }) 35 | } 36 | 37 | type compareFN func(left float64, right float64) bool 38 | 39 | // registerCond registers a binary expression that performs a comparison using 40 | // the given compareFN and that returns 0 or 1 depending on whether or not the 41 | // comparison evaluates to true. 42 | func registerCond(cond string, compare compareFN) { 43 | registerBinaryExpr(cond, func(left float64, right float64) float64 { 44 | if compare(left, right) { 45 | return 1 46 | } 47 | return 0 48 | }) 49 | } 50 | 51 | // LT tests whether left is less than right 52 | func LT(left interface{}, right interface{}) Expr { 53 | return binaryExprFor("<", left, right) 54 | } 55 | 56 | // LTE tests whether left is less than or equal to the right 57 | func LTE(left interface{}, right interface{}) Expr { 58 | return binaryExprFor("<=", left, right) 59 | } 60 | 61 | // EQ tests whether left equals right 62 | func EQ(left interface{}, right interface{}) Expr { 63 | return binaryExprFor("=", left, right) 64 | } 65 | 66 | // NEQ tests whether left is different from right 67 | func NEQ(left interface{}, right interface{}) Expr { 68 | return binaryExprFor("<>", left, right) 69 | } 70 | 71 | // GTE tests whether left is greater than or equal to right 72 | func GTE(left interface{}, right interface{}) Expr { 73 | return binaryExprFor(">=", left, right) 74 | } 75 | 76 | // GT tests whether left is greater than right 77 | func GT(left interface{}, right interface{}) Expr { 78 | return binaryExprFor(">", left, right) 79 | } 80 | 81 | // AND tests whether left and right is true 82 | func AND(left interface{}, right interface{}) Expr { 83 | return binaryExprFor("AND", left, right) 84 | } 85 | 86 | // OR tests whether left or right is true 87 | func OR(left interface{}, right interface{}) Expr { 88 | return binaryExprFor("OR", left, right) 89 | } 90 | -------------------------------------------------------------------------------- /expr/conds_test.go: -------------------------------------------------------------------------------- 1 | package expr 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestLT(t *testing.T) { 9 | doTestCond(t, LT(SUM("a"), SUM("b")), false) 10 | doTestCond(t, LT(SUM("b"), SUM("a")), true) 11 | } 12 | 13 | func TestLTE(t *testing.T) { 14 | doTestCond(t, LTE(SUM("a"), SUM("b")), false) 15 | doTestCond(t, LTE(SUM("b"), SUM("a")), true) 16 | doTestCond(t, LTE(SUM("b"), SUM("b")), true) 17 | } 18 | 19 | func TestEQ(t *testing.T) { 20 | doTestCond(t, EQ(SUM("a"), SUM("b")), false) 21 | doTestCond(t, EQ(SUM("b"), SUM("a")), false) 22 | doTestCond(t, EQ(SUM("b"), SUM("b")), true) 23 | } 24 | 25 | func TestNEQ(t *testing.T) { 26 | doTestCond(t, NEQ(SUM("a"), SUM("b")), true) 27 | doTestCond(t, NEQ(SUM("b"), SUM("a")), true) 28 | doTestCond(t, NEQ(SUM("b"), SUM("b")), false) 29 | } 30 | 31 | func TestGTE(t *testing.T) { 32 | doTestCond(t, GTE(SUM("a"), SUM("b")), true) 33 | doTestCond(t, GTE(SUM("b"), SUM("a")), false) 34 | doTestCond(t, GTE(SUM("b"), SUM("b")), true) 35 | } 36 | 37 | func TestGT(t *testing.T) { 38 | doTestCond(t, GT(SUM("a"), SUM("b")), true) 39 | doTestCond(t, GT(SUM("b"), SUM("a")), false) 40 | } 41 | 42 | func TestAND(t *testing.T) { 43 | doTestCond(t, AND(GT(SUM("a"), SUM("b")), GT(SUM("b"), SUM("a"))), false) 44 | doTestCond(t, AND(GT(SUM("a"), SUM("b")), GT(SUM("a"), SUM("b"))), true) 45 | } 46 | 47 | func TestOR(t *testing.T) { 48 | doTestCond(t, OR(GT(SUM("a"), SUM("b")), GT(SUM("b"), SUM("a"))), true) 49 | doTestCond(t, OR(GT(SUM("a"), SUM("b")), GT("unknown", SUM("a"))), true) 50 | doTestCond(t, OR(GT(SUM("b"), SUM("a")), GT(SUM("b"), SUM("a"))), false) 51 | } 52 | 53 | func doTestCond(t *testing.T, e Expr, expected bool) { 54 | e = msgpacked(t, e) 55 | params := Map{ 56 | "a": 1.001, 57 | "b": 1.0, 58 | } 59 | 60 | b := make([]byte, e.EncodedWidth()) 61 | _, val, updated := e.Update(b, params, nil) 62 | if !assert.True(t, updated) { 63 | return 64 | } 65 | expectedFloat := float64(0) 66 | if expected { 67 | expectedFloat = 1 68 | } 69 | AssertFloatEquals(t, expectedFloat, val) 70 | readVal, wasSet, _ := e.Get(b) 71 | if assert.True(t, wasSet) { 72 | AssertFloatEquals(t, expectedFloat, readVal) 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /expr/constant.go: -------------------------------------------------------------------------------- 1 | package expr 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/getlantern/goexpr" 8 | ) 9 | 10 | // CONST returns an Accumulator that always has a constant value. 11 | func CONST(value float64) Expr { 12 | return &constant{value} 13 | } 14 | 15 | type constant struct { 16 | Value float64 17 | } 18 | 19 | func (e *constant) Validate() error { 20 | return nil 21 | } 22 | 23 | func (e *constant) EncodedWidth() int { 24 | return 0 25 | } 26 | 27 | func (e *constant) Shift() time.Duration { 28 | return 0 29 | } 30 | 31 | func (e *constant) Update(b []byte, params Params, metadata goexpr.Params) ([]byte, float64, bool) { 32 | return b, e.Value, false 33 | } 34 | 35 | func (e *constant) Merge(b []byte, x []byte, y []byte) ([]byte, []byte, []byte) { 36 | return b, x, y 37 | } 38 | 39 | func (e *constant) SubMergers(subs []Expr) []SubMerge { 40 | return make([]SubMerge, len(subs)) 41 | } 42 | 43 | func (e *constant) Get(b []byte) (float64, bool, []byte) { 44 | return e.Value, true, b 45 | } 46 | 47 | func (e *constant) IsConstant() bool { 48 | return true 49 | } 50 | 51 | func (e *constant) DeAggregate() Expr { 52 | return e 53 | } 54 | 55 | func (e *constant) String() string { 56 | return fmt.Sprintf("%f", e.Value) 57 | } 58 | -------------------------------------------------------------------------------- /expr/constant_test.go: -------------------------------------------------------------------------------- 1 | package expr 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestConstant(t *testing.T) { 8 | e := msgpacked(t, CONST(5.5)) 9 | params := Map{ 10 | "a": 8.8, 11 | "b": 4.4, 12 | } 13 | 14 | b := make([]byte, e.EncodedWidth()) 15 | e.Update(b, params, nil) 16 | val, _, _ := e.Get(b) 17 | AssertFloatEquals(t, 5.5, val) 18 | } 19 | -------------------------------------------------------------------------------- /expr/expr.go: -------------------------------------------------------------------------------- 1 | // Package expr provides a framework for Expressions that evaluate to floating 2 | // point values and allow various functions that can aggregate data, perform 3 | // calculations on that data, evaluate boolean expressions against that data 4 | // and serialize the data to/from bytes for durable storage in the database. 5 | package expr 6 | 7 | import ( 8 | "encoding/binary" 9 | "fmt" 10 | "reflect" 11 | "strconv" 12 | "time" 13 | 14 | "github.com/getlantern/goexpr" 15 | "github.com/getlantern/msgpack" 16 | ) 17 | 18 | const ( 19 | width64bits = 8 20 | ) 21 | 22 | var ( 23 | binaryEncoding = binary.BigEndian 24 | 25 | fieldType = reflect.TypeOf((*field)(nil)) 26 | constType = reflect.TypeOf((*constant)(nil)) 27 | boundedType = reflect.TypeOf((*bounded)(nil)) 28 | aggregateType = reflect.TypeOf((*aggregate)(nil)) 29 | ifType = reflect.TypeOf((*ifExpr)(nil)) 30 | avgType = reflect.TypeOf((*avg)(nil)) 31 | binaryType = reflect.TypeOf((*binaryExpr)(nil)) 32 | shiftType = reflect.TypeOf((*shift)(nil)) 33 | unaryMathType = reflect.TypeOf((*unaryMathExpr)(nil)) 34 | percentileType = reflect.TypeOf((*ptile)(nil)) 35 | percentileOptimizedType = reflect.TypeOf((*ptileOptimized)(nil)) 36 | ) 37 | 38 | func init() { 39 | msgpack.RegisterExt(50, &field{}) 40 | msgpack.RegisterExt(51, &constant{}) 41 | msgpack.RegisterExt(52, &bounded{}) 42 | msgpack.RegisterExt(53, &aggregate{}) 43 | msgpack.RegisterExt(54, &ifExpr{}) 44 | msgpack.RegisterExt(55, &avg{}) 45 | msgpack.RegisterExt(56, &binaryExpr{}) 46 | msgpack.RegisterExt(57, &shift{}) 47 | msgpack.RegisterExt(58, &unaryMathExpr{}) 48 | msgpack.RegisterExt(59, &ptile{}) 49 | msgpack.RegisterExt(60, &ptileOptimized{}) 50 | } 51 | 52 | // Params is an interface for data structures that can contain named values. 53 | type Params interface { 54 | // Get returns the named value. Found should be false if nothing was found for 55 | // the given name. 56 | Get(name string) (val float64, found bool) 57 | } 58 | 59 | // Map is an implementation of the Params interface using a map. 60 | type Map map[string]float64 61 | 62 | // Get implements the method from the Params interface 63 | func (p Map) Get(name string) (val float64, found bool) { 64 | val, found = p[name] 65 | return val, found 66 | } 67 | 68 | // FloatParams is an implementation of Params that always returns the same 69 | // float64 value. 70 | type FloatParams float64 71 | 72 | func (p FloatParams) Get(name string) (val float64, found bool) { 73 | return float64(p), true 74 | } 75 | 76 | // SubMerge is a function that merges other into data for a given Expr, 77 | // potentially taking into account the supplied metadata. otherRes is the amount 78 | // of time represented by each period in other. 79 | type SubMerge func(data []byte, other []byte, otherRes time.Duration, metadata goexpr.Params) 80 | 81 | // An Expr is expression that stores its value in a byte array and that 82 | // evaluates to a float64. 83 | type Expr interface { 84 | // Validate makes sure that this expression is valid and returns an error if 85 | // it is not. 86 | Validate() error 87 | 88 | // EncodedWidth returns the number of bytes needed to represent the internal 89 | // state of this Expr. 90 | EncodedWidth() int 91 | 92 | // Shift returns the total cumulative shift in time, including 93 | // subexpressions. 94 | Shift() time.Duration 95 | 96 | // Update updates the value in buf by applying the given Params. Metadata 97 | // provides additional metadata that can be used in evaluating how to apply 98 | // the update. 99 | Update(b []byte, params Params, metadata goexpr.Params) (remain []byte, value float64, updated bool) 100 | 101 | // Merge merges x and y, writing the result to b. It returns the remaining 102 | // portions of x and y. 103 | Merge(b []byte, x []byte, y []byte) (remainB []byte, remainX []byte, remainY []byte) 104 | 105 | // SubMergers returns a list of functions that merge values of the given 106 | // subexpressions into this Expr. The list is the same length as the number of 107 | // sub expressions. For any subexpression that is not represented in our 108 | // Expression, the corresponding function in the list is nil. 109 | SubMergers(subs []Expr) []SubMerge 110 | 111 | // Get gets the value in buf, returning the value, a boolean indicating 112 | // whether or not the value was actually set, and the remaining byte array 113 | // after consuming the underlying data. 114 | Get(b []byte) (value float64, ok bool, remain []byte) 115 | 116 | // IsConstant indicates whether or not this is a constant expression 117 | IsConstant() bool 118 | 119 | // DeAggregate strips aggregates from this expression and returns the result 120 | DeAggregate() Expr 121 | 122 | String() string 123 | } 124 | 125 | func exprFor(expr interface{}) Expr { 126 | switch e := expr.(type) { 127 | case Expr: 128 | return e 129 | case string: 130 | v, err := strconv.ParseFloat(e, 64) 131 | if err == nil { 132 | return CONST(v) 133 | } 134 | return FIELD(e) 135 | case int: 136 | return CONST(float64(e)) 137 | case int64: 138 | return CONST(float64(e)) 139 | case int32: 140 | return CONST(float64(e)) 141 | case int16: 142 | return CONST(float64(e)) 143 | case byte: 144 | return CONST(float64(e)) 145 | case float32: 146 | return CONST(float64(e)) 147 | case float64: 148 | return CONST(e) 149 | default: 150 | panic(fmt.Sprintf("Got a %v, please specify an Expr, string, float64 or integer", reflect.TypeOf(expr))) 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /expr/field.go: -------------------------------------------------------------------------------- 1 | package expr 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/getlantern/goexpr" 7 | ) 8 | 9 | // IsField checks whether the given expression is a field expression and if so, 10 | // returns the name of the field. 11 | func IsField(e Expr) (string, bool) { 12 | f, ok := e.(*field) 13 | if !ok { 14 | return "", false 15 | } 16 | return f.Name, true 17 | } 18 | 19 | // FIELD creates an Expr that obtains its value from a named field. 20 | func FIELD(name string) Expr { 21 | return &field{name} 22 | } 23 | 24 | type fieldAccumulator struct { 25 | name string 26 | value float64 27 | } 28 | 29 | type field struct { 30 | Name string 31 | } 32 | 33 | func (e *field) Validate() error { 34 | return nil 35 | } 36 | 37 | func (e *field) EncodedWidth() int { 38 | return 0 39 | } 40 | 41 | func (e *field) Shift() time.Duration { 42 | return 0 43 | } 44 | 45 | func (e *field) Update(b []byte, params Params, metadata goexpr.Params) ([]byte, float64, bool) { 46 | val, ok := params.Get(e.Name) 47 | return b, val, ok 48 | } 49 | 50 | func (e *field) Merge(b []byte, x []byte, y []byte) ([]byte, []byte, []byte) { 51 | return b, x, y 52 | } 53 | 54 | func (e *field) SubMergers(subs []Expr) []SubMerge { 55 | return make([]SubMerge, len(subs)) 56 | } 57 | 58 | func (e *field) Get(b []byte) (float64, bool, []byte) { 59 | return 0, false, b 60 | } 61 | 62 | func (e *field) IsConstant() bool { 63 | return false 64 | } 65 | 66 | func (e *field) DeAggregate() Expr { 67 | return e 68 | } 69 | 70 | func (e *field) String() string { 71 | return e.Name 72 | } 73 | -------------------------------------------------------------------------------- /expr/field_test.go: -------------------------------------------------------------------------------- 1 | package expr 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestField(t *testing.T) { 9 | params := Map{ 10 | "a": 4.4, 11 | } 12 | f := msgpacked(t, FIELD("a")) 13 | b := make([]byte, f.EncodedWidth()) 14 | _, val, _ := f.Update(b, params, nil) 15 | assert.EqualValues(t, 4.4, val) 16 | } 17 | -------------------------------------------------------------------------------- /expr/floatequals.go: -------------------------------------------------------------------------------- 1 | package expr 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | const epsilon float64 = 0.00001 11 | 12 | // AssertFloatEquals does a fuzzy comparison of floats. 13 | func AssertFloatEquals(t *testing.T, a, b float64) bool { 14 | t.Helper() 15 | return assert.True(t, FuzzyEquals(epsilon, a, b), fmt.Sprintf("Floats did not match. Expected: %f Actual: %f", a, b)) 16 | } 17 | 18 | // AssertFloatWithin checks whether a given float is within e error (decimal) of 19 | // another float 20 | func AssertFloatWithin(t *testing.T, e, expected float64, actual float64, msg string) bool { 21 | t.Helper() 22 | return assert.True(t, FuzzyEquals(e, expected, actual), fmt.Sprintf("%v -- Floats not within %f of each other. Expected: %f Actual: %f", msg, e, expected, actual)) 23 | } 24 | 25 | // courtesy of https://gist.github.com/cevaris/bc331cbe970b03816c6b 26 | func FuzzyEquals(e, a, b float64) bool { 27 | if a == b { 28 | return true 29 | } 30 | d := b - a 31 | q := d / ((a + b) / 2) 32 | return q >= -1*e && q <= e 33 | } 34 | -------------------------------------------------------------------------------- /expr/if.go: -------------------------------------------------------------------------------- 1 | package expr 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/getlantern/goexpr" 8 | ) 9 | 10 | type ifExpr struct { 11 | Cond goexpr.Expr 12 | Wrapped Expr 13 | Width int 14 | } 15 | 16 | func IF(cond goexpr.Expr, wrapped interface{}) Expr { 17 | _wrapped := exprFor(wrapped) 18 | return &ifExpr{cond, _wrapped, _wrapped.EncodedWidth()} 19 | } 20 | 21 | func (e *ifExpr) Validate() error { 22 | return e.Wrapped.Validate() 23 | } 24 | 25 | func (e *ifExpr) EncodedWidth() int { 26 | return e.Width 27 | } 28 | 29 | func (e *ifExpr) Shift() time.Duration { 30 | return e.Wrapped.Shift() 31 | } 32 | 33 | func (e *ifExpr) Update(b []byte, params Params, metadata goexpr.Params) ([]byte, float64, bool) { 34 | if e.include(metadata) { 35 | return e.Wrapped.Update(b, params, metadata) 36 | } 37 | value, _, remain := e.Wrapped.Get(b) 38 | return remain, value, false 39 | } 40 | 41 | func (e *ifExpr) Merge(b []byte, x []byte, y []byte) ([]byte, []byte, []byte) { 42 | return e.Wrapped.Merge(b, x, y) 43 | } 44 | 45 | func (e *ifExpr) SubMergers(subs []Expr) []SubMerge { 46 | sms := make([]SubMerge, len(subs)) 47 | matched := false 48 | for i, sub := range subs { 49 | if e.String() == sub.String() { 50 | sms[i] = e.subMerge 51 | matched = true 52 | } 53 | } 54 | if matched { 55 | // We have an exact match, use that 56 | return sms 57 | } 58 | 59 | sms = e.Wrapped.SubMergers(subs) 60 | for i, sm := range sms { 61 | sms[i] = e.condSubMerger(sm) 62 | } 63 | return sms 64 | } 65 | 66 | func (e *ifExpr) condSubMerger(wrapped SubMerge) SubMerge { 67 | if wrapped == nil { 68 | return nil 69 | } 70 | return func(data []byte, other []byte, otherRes time.Duration, metadata goexpr.Params) { 71 | if e.include(metadata) { 72 | wrapped(data, other, otherRes, metadata) 73 | } 74 | } 75 | } 76 | 77 | func (e *ifExpr) subMerge(data []byte, other []byte, otherRes time.Duration, metadata goexpr.Params) { 78 | e.Wrapped.Merge(data, data, other) 79 | } 80 | 81 | func (e *ifExpr) include(metadata goexpr.Params) bool { 82 | if metadata == nil || e.Cond == nil { 83 | return true 84 | } 85 | val := e.Cond.Eval(metadata) 86 | if val == nil { 87 | return false 88 | } 89 | valBool, ok := val.(bool) 90 | if !ok { 91 | fmt.Printf("Result %v for expression %v is not a bool!\n", val, e.Cond) 92 | return false 93 | } 94 | return valBool 95 | } 96 | 97 | func (e *ifExpr) Get(b []byte) (float64, bool, []byte) { 98 | return e.Wrapped.Get(b) 99 | } 100 | 101 | func (e *ifExpr) IsConstant() bool { 102 | return e.Wrapped.IsConstant() 103 | } 104 | 105 | func (e *ifExpr) DeAggregate() Expr { 106 | return IF(e.Cond, e.Wrapped.DeAggregate()) 107 | } 108 | 109 | func (e *ifExpr) String() string { 110 | return fmt.Sprintf("IF(%v, %v)", e.Cond, e.Wrapped) 111 | } 112 | -------------------------------------------------------------------------------- /expr/math.go: -------------------------------------------------------------------------------- 1 | package expr 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "time" 7 | 8 | "github.com/getlantern/goexpr" 9 | "github.com/getlantern/msgpack" 10 | ) 11 | 12 | var unaryMathFNs = map[string]func(float64) float64{ 13 | "LN": math.Log, 14 | "LOG2": math.Log2, 15 | "LOG10": math.Log10, 16 | } 17 | 18 | type unaryMathExpr struct { 19 | Name string 20 | fn func(float64) float64 21 | Wrapped Expr 22 | Width int 23 | } 24 | 25 | func UnaryMath(name string, wrapped interface{}) (Expr, error) { 26 | fn := unaryMathFNs[name] 27 | if fn == nil { 28 | return nil, fmt.Errorf("Unknown math function %v", name) 29 | } 30 | _wrapped := exprFor(wrapped) 31 | return &unaryMathExpr{name, fn, _wrapped, _wrapped.EncodedWidth()}, nil 32 | } 33 | 34 | func (e *unaryMathExpr) Validate() error { 35 | return e.Wrapped.Validate() 36 | } 37 | 38 | func (e *unaryMathExpr) EncodedWidth() int { 39 | return e.Width 40 | } 41 | 42 | func (e *unaryMathExpr) Shift() time.Duration { 43 | return e.Wrapped.Shift() 44 | } 45 | 46 | func (e *unaryMathExpr) Update(b []byte, params Params, metadata goexpr.Params) ([]byte, float64, bool) { 47 | return e.Wrapped.Update(b, params, metadata) 48 | } 49 | 50 | func (e *unaryMathExpr) Merge(b []byte, x []byte, y []byte) ([]byte, []byte, []byte) { 51 | return e.Wrapped.Merge(b, x, y) 52 | } 53 | 54 | func (e *unaryMathExpr) SubMergers(subs []Expr) []SubMerge { 55 | ssms := e.Wrapped.SubMergers(subs) 56 | sms := make([]SubMerge, len(subs)) 57 | for i, sub := range subs { 58 | if e.String() == sub.String() { 59 | sms[i] = ssms[i] 60 | } 61 | } 62 | return ssms 63 | } 64 | 65 | func (e *unaryMathExpr) Get(b []byte) (float64, bool, []byte) { 66 | val, found, remain := e.Wrapped.Get(b) 67 | if found { 68 | val = e.fn(val) 69 | } 70 | return val, found, remain 71 | } 72 | 73 | func (e *unaryMathExpr) IsConstant() bool { 74 | return e.Wrapped.IsConstant() 75 | } 76 | 77 | func (e *unaryMathExpr) DeAggregate() Expr { 78 | // We can safely ignore the error because e.Name has already passed validation 79 | result, _ := UnaryMath(e.Name, e.Wrapped.DeAggregate()) 80 | return result 81 | } 82 | 83 | func (e *unaryMathExpr) String() string { 84 | return fmt.Sprintf("%v(%v)", e.Name, e.Wrapped) 85 | } 86 | 87 | func (e *unaryMathExpr) DecodeMsgpack(dec *msgpack.Decoder) error { 88 | m := make(map[string]interface{}) 89 | err := dec.Decode(&m) 90 | if err != nil { 91 | return err 92 | } 93 | e.Name = m["Name"].(string) 94 | e.fn = unaryMathFNs[e.Name] 95 | e.Wrapped = m["Wrapped"].(Expr) 96 | e.Width = int(m["Width"].(uint64)) 97 | return nil 98 | } 99 | -------------------------------------------------------------------------------- /expr/math_test.go: -------------------------------------------------------------------------------- 1 | package expr 2 | 3 | import ( 4 | "math" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestLN(t *testing.T) { 11 | doTestUnaryMath(t, "LN", math.E, 1) 12 | } 13 | 14 | func TestLog2(t *testing.T) { 15 | doTestUnaryMath(t, "LOG2", 2, 1) 16 | } 17 | 18 | func TestLog10(t *testing.T) { 19 | doTestUnaryMath(t, "LOG10", 10, 1) 20 | } 21 | 22 | func doTestUnaryMath(t *testing.T, name string, in float64, expected float64) { 23 | e, err := UnaryMath(name, CONST(in)) 24 | if !assert.NoError(t, err) { 25 | return 26 | } 27 | val, _, _ := msgpacked(t, e).Get(nil) 28 | AssertFloatEquals(t, expected, val) 29 | } 30 | -------------------------------------------------------------------------------- /expr/percentile.go: -------------------------------------------------------------------------------- 1 | package expr 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "time" 7 | 8 | "github.com/HdrHistogram/hdrhistogram-go" 9 | "github.com/getlantern/goexpr" 10 | ) 11 | 12 | // PERCENTILE tracks estimated percentile values for the given expression 13 | // assuming the given min and max possible values, to the given precision 14 | // (where precision is the number of decimal points). 15 | // 16 | // Inputs are automatically bounded to the min/max using BOUNDED, such that 17 | // values falling outside the range are discarded. Percentile is input in 18 | // percent (e.g. 0-100). See https://godoc.org/github.com/codahale/hdrhistogram. 19 | // 20 | // It is possible to wrap an existing PERCENTILE with a new PERCENTILE to reuse 21 | // the original PERCENTILE's storage but look at a different percentile. In this 22 | // case, the min, max and precision parameters are ignored. 23 | // 24 | // WARNING - when PERCENTILEs that wrap existing PERCENTILEs are not stored and 25 | // as such are only suitable for use in querying but not in tables or views 26 | // unless those explicitly include the original PERCENTILE as well. 27 | // 28 | // WARNING - relative to other types of expressions, PERCENTILE types can be 29 | // very large (e.g. on the order of Kilobytes vs 8 or 16 bytes for most other 30 | // expressions) so it is best to keep these relatively low cardinality. 31 | func PERCENTILE(value interface{}, percentile interface{}, min float64, max float64, precision int) Expr { 32 | valueExpr := exprFor(value) 33 | // Remove aggregates 34 | valueExpr = valueExpr.DeAggregate() 35 | // Figure out what precision to use for HDR 36 | hdrPrecision := precision 37 | if hdrPrecision < 1 { 38 | hdrPrecision = 1 39 | } else if hdrPrecision > 5 { 40 | hdrPrecision = 5 41 | } 42 | 43 | sampleHisto := hdrhistogram.New(scaleToInt(min, precision), scaleToInt(max, precision), hdrPrecision) 44 | numCounts := len(sampleHisto.Export().Counts) 45 | return &ptile{ 46 | Value: BOUNDED(valueExpr, min, max), 47 | Percentile: exprFor(percentile), 48 | Min: scaleToInt(min, precision), 49 | Max: scaleToInt(max, precision), 50 | Precision: precision, 51 | HDRPrecision: hdrPrecision, 52 | Width: (1+numCounts)*width64bits + valueExpr.EncodedWidth(), 53 | } 54 | } 55 | 56 | // IsPercentile indicates whether the given expression is a percentile 57 | // expression. 58 | func IsPercentile(e Expr) bool { 59 | switch e.(type) { 60 | case *ptile: 61 | return true 62 | case *ptileOptimized: 63 | return true 64 | default: 65 | return false 66 | } 67 | } 68 | 69 | func scaleToInt(value float64, precision int) int64 { 70 | return int64(value * math.Pow10(precision)) 71 | } 72 | 73 | func scaleFromInt(value int64, precision int) float64 { 74 | return scaleFromFloat(float64(value), precision) 75 | } 76 | 77 | func scaleFromFloat(value float64, precision int) float64 { 78 | return value / float64(math.Pow10(precision)) 79 | } 80 | 81 | type ptile struct { 82 | Value Expr 83 | Percentile Expr 84 | Min int64 85 | Max int64 86 | Precision int 87 | HDRPrecision int 88 | Width int 89 | } 90 | 91 | func (e *ptile) Validate() error { 92 | err := validateWrappedInAggregate(e.Value) 93 | if err != nil { 94 | return err 95 | } 96 | if e.Percentile.EncodedWidth() > 0 { 97 | return fmt.Errorf("Percentile expression %v must be a constant or directly derived from a field", e.Percentile) 98 | } 99 | return nil 100 | } 101 | 102 | func (e *ptile) EncodedWidth() int { 103 | return e.Width 104 | } 105 | 106 | func (e *ptile) Shift() time.Duration { 107 | a := e.Value.Shift() 108 | b := e.Percentile.Shift() 109 | if a < b { 110 | return a 111 | } 112 | return b 113 | } 114 | 115 | func (e *ptile) Update(b []byte, params Params, metadata goexpr.Params) ([]byte, float64, bool) { 116 | histo, _, remain := e.load(b) 117 | remain, value, updated := e.Value.Update(remain, params, metadata) 118 | remain, percentile, _ := e.Percentile.Update(remain, params, metadata) 119 | if updated { 120 | histo.RecordValue(scaleToInt(value, e.Precision)) 121 | e.save(b, histo) 122 | } 123 | return remain, e.calc(histo, percentile), updated 124 | } 125 | 126 | func (e *ptile) Merge(b []byte, x []byte, y []byte) ([]byte, []byte, []byte) { 127 | histoX, xWasSet, remainX := e.load(x) 128 | histoY, yWasSet, remainY := e.load(y) 129 | if !xWasSet { 130 | if yWasSet { 131 | // Use valueY 132 | b = e.save(b, histoY) 133 | } else { 134 | // Nothing to save, just advance 135 | b = b[e.Width:] 136 | } 137 | } else { 138 | if yWasSet { 139 | histoX.Merge(histoY) 140 | } 141 | b = e.save(b, histoX) 142 | } 143 | return b, remainX, remainY 144 | } 145 | 146 | func (e *ptile) SubMergers(subs []Expr) []SubMerge { 147 | result := make([]SubMerge, 0, len(subs)) 148 | for _, sub := range subs { 149 | var sm SubMerge 150 | if e.String() == sub.String() { 151 | sm = e.subMerge 152 | } 153 | result = append(result, sm) 154 | } 155 | return result 156 | } 157 | 158 | func (e *ptile) subMerge(data []byte, other []byte, otherRes time.Duration, metadata goexpr.Params) { 159 | e.Merge(data, data, other) 160 | } 161 | 162 | func (e *ptile) Get(b []byte) (float64, bool, []byte) { 163 | histo, wasSet, remain := e.load(b) 164 | percentile, _, remain := e.Percentile.Get(remain) 165 | if !wasSet { 166 | return 0, wasSet, remain 167 | } 168 | return e.calc(histo, percentile), wasSet, remain 169 | } 170 | 171 | func (e *ptile) calc(histo *hdrhistogram.Histogram, percentile float64) float64 { 172 | return scaleFromInt(histo.ValueAtQuantile(percentile), e.Precision) 173 | } 174 | 175 | func (e *ptile) load(b []byte) (*hdrhistogram.Histogram, bool, []byte) { 176 | remain := b[e.Width:] 177 | numCounts := int(binaryEncoding.Uint64(b)) 178 | wasSet := numCounts > 0 179 | var histo *hdrhistogram.Histogram 180 | if !wasSet { 181 | histo = hdrhistogram.New(e.Min, e.Max, e.HDRPrecision) 182 | } else { 183 | counts := make([]int64, numCounts) 184 | for i := 0; i < numCounts; i++ { 185 | counts[i] = int64(binaryEncoding.Uint64(b[(i+1)*width64bits:])) 186 | } 187 | histo = hdrhistogram.Import(&hdrhistogram.Snapshot{ 188 | LowestTrackableValue: e.Min, 189 | HighestTrackableValue: e.Max, 190 | SignificantFigures: int64(e.HDRPrecision), 191 | Counts: counts, 192 | }) 193 | } 194 | return histo, wasSet, remain 195 | } 196 | 197 | func (e *ptile) save(b []byte, histo *hdrhistogram.Histogram) []byte { 198 | s := histo.Export() 199 | numCounts := len(s.Counts) 200 | binaryEncoding.PutUint64(b, uint64(numCounts)) 201 | for i := 0; i < numCounts; i++ { 202 | binaryEncoding.PutUint64(b[(i+1)*width64bits:], uint64(s.Counts[i])) 203 | } 204 | return b[e.Width:] 205 | } 206 | 207 | func (e *ptile) IsConstant() bool { 208 | return e.Value.IsConstant() 209 | } 210 | 211 | func (e *ptile) DeAggregate() Expr { 212 | return e.Value.(*bounded).wrapped.DeAggregate() 213 | } 214 | 215 | func (e *ptile) String() string { 216 | return fmt.Sprintf("PERCENTILE(%v, %v, %v, %v, %v)", e.Value, e.Percentile, e.Min, e.Max, e.Precision) 217 | } 218 | -------------------------------------------------------------------------------- /expr/percentile_optimized.go: -------------------------------------------------------------------------------- 1 | package expr 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/getlantern/msgpack" 7 | ) 8 | 9 | // PERCENTILEOPT returns an optimized PERCENTILE that wraps an existing 10 | // PERCENTILE. 11 | func PERCENTILEOPT(wrapped interface{}, percentile interface{}) Expr { 12 | var expr Expr 13 | switch t := wrapped.(type) { 14 | case *ptileOptimized: 15 | expr = &t.ptile 16 | default: 17 | expr = wrapped.(*ptile) 18 | } 19 | return &ptileOptimized{Wrapped: expr, ptile: *expr.(*ptile), Percentile: exprFor(percentile)} 20 | } 21 | 22 | type ptileOptimized struct { 23 | ptile 24 | Wrapped Expr 25 | Percentile Expr 26 | } 27 | 28 | func (e *ptileOptimized) Get(b []byte) (float64, bool, []byte) { 29 | histo, wasSet, remain := e.ptile.load(b) 30 | percentile, _, remain := e.Percentile.Get(remain) 31 | if !wasSet { 32 | return 0, wasSet, remain 33 | } 34 | return e.ptile.calc(histo, percentile), wasSet, remain 35 | } 36 | 37 | func (e *ptileOptimized) String() string { 38 | return fmt.Sprintf("PERCENTILE(%v, %v)", e.Wrapped.String(), e.Percentile) 39 | } 40 | 41 | func (e *ptileOptimized) DecodeMsgpack(dec *msgpack.Decoder) error { 42 | m := make(map[string]interface{}) 43 | err := dec.Decode(&m) 44 | if err != nil { 45 | return err 46 | } 47 | wrapped := m["Wrapped"].(*ptile) 48 | percentile := m["Percentile"].(Expr) 49 | e.Wrapped = wrapped 50 | e.ptile = *wrapped 51 | e.Percentile = percentile 52 | return nil 53 | } 54 | -------------------------------------------------------------------------------- /expr/percentile_test.go: -------------------------------------------------------------------------------- 1 | package expr 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/getlantern/goexpr" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestScaleInt(t *testing.T) { 12 | v := 5.126 13 | AssertFloatEquals(t, v, scaleFromInt(scaleToInt(v, 3), 3)) 14 | AssertFloatEquals(t, 5.12, scaleFromInt(scaleToInt(v, 2), 2)) 15 | } 16 | 17 | func TestDeAggregatePercentile(t *testing.T) { 18 | e := msgpacked(t, PERCENTILE("p", 99, 0, 100, 1)) 19 | assert.Equal(t, FIELD("p").String(), e.DeAggregate().String()) 20 | } 21 | 22 | func TestPercentile(t *testing.T) { 23 | e := msgpacked(t, PERCENTILE(SUM("a"), 99, 0, 10, 1)) 24 | expected := float64(9.9) 25 | 26 | eo := msgpacked(t, PERCENTILEOPT(e, 50)) 27 | expectedO := float64(5.1) 28 | 29 | eo2 := msgpacked(t, PERCENTILEOPT(eo, 10)) 30 | expectedO2 := float64(1.0) 31 | 32 | if !assert.True(t, IsPercentile(e)) { 33 | return 34 | } 35 | if !assert.IsType(t, &ptile{}, e) { 36 | return 37 | } 38 | if !assert.IsType(t, &ptileOptimized{}, eo) { 39 | return 40 | } 41 | if !assert.IsType(t, &ptileOptimized{}, eo2) { 42 | return 43 | } 44 | 45 | checkValue := func(e Expr, b []byte, expected float64) { 46 | val, wasSet, _ := e.Get(b) 47 | if assert.True(t, wasSet) { 48 | AssertFloatWithin(t, 0.01, expected, val, "Incorrect percentile") 49 | } 50 | } 51 | 52 | md := goexpr.MapParams{} 53 | 54 | merged := make([]byte, e.EncodedWidth()) 55 | for i := 0; i < 2; i++ { 56 | b := make([]byte, e.EncodedWidth()) 57 | for j := 0; j < 50; j++ { 58 | // Do some direct updates 59 | for k := float64(1); k <= 50; k++ { 60 | e.Update(b, Map{"a": k / 10}, md) 61 | } 62 | 63 | // Do some point merges 64 | for k := float64(51); k <= 100; k++ { 65 | b2 := make([]byte, e.EncodedWidth()) 66 | e.Update(b2, Map{"a": k / 10}, md) 67 | e.Merge(b, b, b2) 68 | } 69 | } 70 | checkValue(e, b, expected) 71 | checkValue(eo, b, expectedO) 72 | checkValue(eo2, b, expectedO2) 73 | e.Merge(merged, merged, b) 74 | } 75 | 76 | checkValue(e, merged, expected) 77 | checkValue(eo, merged, expectedO) 78 | checkValue(eo2, merged, expectedO2) 79 | } 80 | 81 | func TestPercentileSize(t *testing.T) { 82 | p := PERCENTILE("A", 50, 0, 120, 1) 83 | fmt.Println(p.(*ptile).Width) 84 | } 85 | -------------------------------------------------------------------------------- /expr/shift.go: -------------------------------------------------------------------------------- 1 | package expr 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/getlantern/goexpr" 8 | ) 9 | 10 | type shift struct { 11 | Wrapped Expr 12 | Offset time.Duration 13 | Width int 14 | } 15 | 16 | func SHIFT(wrapped interface{}, offset time.Duration) Expr { 17 | _wrapped := exprFor(wrapped) 18 | return &shift{_wrapped, offset, _wrapped.EncodedWidth()} 19 | } 20 | 21 | func (e *shift) Validate() error { 22 | return e.Wrapped.Validate() 23 | } 24 | 25 | func (e *shift) EncodedWidth() int { 26 | return e.Width 27 | } 28 | 29 | func (e *shift) Shift() time.Duration { 30 | return e.Offset + e.Wrapped.Shift() 31 | } 32 | 33 | func (e *shift) Update(b []byte, params Params, metadata goexpr.Params) ([]byte, float64, bool) { 34 | return e.Wrapped.Update(b, params, metadata) 35 | } 36 | 37 | func (e *shift) Merge(b []byte, x []byte, y []byte) ([]byte, []byte, []byte) { 38 | return e.Wrapped.Merge(b, x, y) 39 | } 40 | 41 | func (e *shift) SubMergers(subs []Expr) []SubMerge { 42 | sms := make([]SubMerge, len(subs)) 43 | matched := false 44 | for i, sub := range subs { 45 | if e.String() == sub.String() { 46 | sms[i] = e.subMerge 47 | matched = true 48 | } 49 | } 50 | if matched { 51 | // We have an exact match, use that 52 | return sms 53 | } 54 | 55 | sms = e.Wrapped.SubMergers(subs) 56 | for i, sm := range sms { 57 | sms[i] = e.shiftedSubMerger(sm, subs[i].EncodedWidth()) 58 | } 59 | return sms 60 | } 61 | 62 | func (e *shift) shiftedSubMerger(wrapped SubMerge, subWidth int) SubMerge { 63 | if wrapped == nil { 64 | return nil 65 | } 66 | return func(data []byte, other []byte, otherRes time.Duration, metadata goexpr.Params) { 67 | n := -1 * int(e.Offset/otherRes) * subWidth 68 | if n >= 0 && n < len(other) { 69 | wrapped(data, other[n:], otherRes, metadata) 70 | } 71 | } 72 | } 73 | 74 | func (e *shift) subMerge(data []byte, other []byte, otherRes time.Duration, metadata goexpr.Params) { 75 | e.Wrapped.Merge(data, data, other) 76 | } 77 | 78 | func (e *shift) Get(b []byte) (float64, bool, []byte) { 79 | return e.Wrapped.Get(b) 80 | } 81 | 82 | func (e *shift) IsConstant() bool { 83 | return e.Wrapped.IsConstant() 84 | } 85 | 86 | func (e *shift) DeAggregate() Expr { 87 | return SHIFT(e.Wrapped.DeAggregate(), e.Offset) 88 | } 89 | 90 | func (e *shift) String() string { 91 | return fmt.Sprintf("SHIFT(%v, %v)", e.Wrapped, e.Offset) 92 | } 93 | -------------------------------------------------------------------------------- /expr/shift_test.go: -------------------------------------------------------------------------------- 1 | package expr 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestShiftRegular(t *testing.T) { 11 | params := Map{ 12 | "a": 4.4, 13 | } 14 | s := msgpacked(t, SHIFT(SUM(FIELD("a")), 1*time.Hour)) 15 | b1 := make([]byte, s.EncodedWidth()*2) 16 | b2 := make([]byte, s.EncodedWidth()*2) 17 | b3 := make([]byte, s.EncodedWidth()*2) 18 | _, val, _ := s.Update(b1, params, nil) 19 | assert.EqualValues(t, 4.4, val) 20 | s.Update(b2, params, nil) 21 | val, _, _ = s.Get(b2) 22 | assert.EqualValues(t, 4.4, val) 23 | s.Merge(b3, b1, b2) 24 | val, _, _ = s.Get(b3) 25 | assert.EqualValues(t, 8.8, val) 26 | } 27 | 28 | func TestShiftSubMerge(t *testing.T) { 29 | res := 1 * time.Hour 30 | periods := 10 31 | 32 | fa := msgpacked(t, SUM(FIELD("a"))) 33 | fs := msgpacked(t, SUB(SHIFT(SHIFT(SUM(FIELD("a")), -2*res), -1*res), SUM(FIELD("a")))) 34 | assert.EqualValues(t, -3*res, fs.Shift()) 35 | 36 | a := make([]byte, fa.EncodedWidth()*periods) 37 | s := make([]byte, fs.EncodedWidth()*periods) 38 | 39 | for i := 0; i < periods; i++ { 40 | fa.Update(a[i*fa.EncodedWidth():], Map{"a": float64(i)}, nil) 41 | } 42 | 43 | subs := fs.SubMergers([]Expr{fa}) 44 | for i := 0; i < periods; i++ { 45 | for _, sub := range subs { 46 | sub(s[i*fs.EncodedWidth():], a[i*fa.EncodedWidth():], res, nil) 47 | } 48 | } 49 | for i := 0; i < periods; i++ { 50 | expected := 3 51 | if i >= 7 { 52 | expected = -1 * i 53 | } 54 | actual, _, _ := fs.Get(s[i*fs.EncodedWidth():]) 55 | assert.EqualValues(t, expected, actual, "Wrong value at position %d", i) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/getlantern/zenodb 2 | 3 | go 1.12 4 | 5 | require ( 6 | github.com/HdrHistogram/hdrhistogram-go v1.1.0 7 | github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect 8 | github.com/aristanetworks/goarista v0.0.0-20190502180301-283422fc1708 // indirect 9 | github.com/boltdb/bolt v1.3.1 10 | github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e 11 | github.com/cloudfoundry/gosigar v1.1.0 12 | github.com/davecgh/go-spew v1.1.1 13 | github.com/dustin/go-humanize v1.0.0 14 | github.com/getlantern/appdir v0.0.0-20180320102544-7c0f9d241ea7 15 | github.com/getlantern/bytemap v0.0.0-20210122162547-b07440a617f0 16 | github.com/getlantern/errors v1.0.1 17 | github.com/getlantern/goexpr v0.0.0-20211215215226-4cdd4fd2847b 18 | github.com/getlantern/golog v0.0.0-20210606115803-bce9f9fe5a5f 19 | github.com/getlantern/grtrack v0.0.0-20160824195228-cbf67d3fa0fd 20 | github.com/getlantern/msgpack v3.1.4+incompatible 21 | github.com/getlantern/mtime v0.0.0-20170117193331-ba114e4a82b0 22 | github.com/getlantern/redis-utils v0.0.0-20210823133122-d4f0e525e095 23 | github.com/getlantern/sqlparser v0.0.0-20171012210704-a879d8035f3c 24 | github.com/getlantern/tlsdefaults v0.0.0-20171004213447-cf35cfd0b1b4 25 | github.com/getlantern/uuid v1.2.0 26 | github.com/getlantern/vtime v0.0.0-20160810174823-dc1e573cf991 27 | github.com/getlantern/waitforserver v1.0.1 28 | github.com/getlantern/wal v0.0.0-20220217194315-e4eac848dbd1 29 | github.com/getlantern/withtimeout v0.0.0-20160829163843-511f017cd913 30 | github.com/getlantern/yaml v0.0.0-20190801163808-0c9bb1ebf426 31 | github.com/go-ole/go-ole v1.2.4 // indirect 32 | github.com/go-redis/redis/v8 v8.11.3 33 | github.com/go-stack/stack v1.8.1 // indirect 34 | github.com/golang/snappy v0.0.3 35 | github.com/gorilla/mux v1.7.1 36 | github.com/gorilla/securecookie v1.1.1 37 | github.com/jmcvetta/randutil v0.0.0-20150817122601-2bb1b664bcff 38 | github.com/kylelemons/godebug v1.1.0 39 | github.com/mitchellh/go-homedir v1.1.0 // indirect 40 | github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c 41 | github.com/oxtoacart/emsort v0.0.0-20160911032127-e467347e3354 42 | github.com/pkg/errors v0.8.1 // indirect 43 | github.com/retailnext/hllpp v1.0.0 44 | github.com/rickar/props v0.0.0-20170718221555-0b06aeb2f037 45 | github.com/shirou/gopsutil v2.18.12+incompatible 46 | github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect 47 | github.com/spaolacci/murmur3 v1.1.0 48 | github.com/stretchr/testify v1.7.0 49 | github.com/vharitonsky/iniflags v0.0.0-20180513140207-a33cd0b5f3de 50 | github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 // indirect 51 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 52 | golang.org/x/net v0.0.0-20210610132358-84b48f89b13b 53 | google.golang.org/grpc v1.22.1 54 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect 55 | ) 56 | -------------------------------------------------------------------------------- /math_bench_test.go: -------------------------------------------------------------------------------- 1 | package zenodb 2 | 3 | import ( 4 | "encoding/binary" 5 | "math" 6 | "testing" 7 | ) 8 | 9 | func BenchmarkMathFloatBigEndian(b *testing.B) { 10 | buf := make([]byte, 8) 11 | b.ResetTimer() 12 | for i := 0; i < b.N; i++ { 13 | f := math.Float64frombits(binary.BigEndian.Uint64(buf)) 14 | binary.BigEndian.PutUint64(buf, math.Float64bits(f+1)) 15 | } 16 | } 17 | 18 | func BenchmarkMathFloatLittleEndian(b *testing.B) { 19 | buf := make([]byte, 8) 20 | b.ResetTimer() 21 | for i := 0; i < b.N; i++ { 22 | f := math.Float64frombits(binary.LittleEndian.Uint64(buf)) 23 | binary.LittleEndian.PutUint64(buf, math.Float64bits(f+1)) 24 | } 25 | } 26 | 27 | func BenchmarkMathIntBigEndian(b *testing.B) { 28 | buf := make([]byte, 8) 29 | b.ResetTimer() 30 | for i := 0; i < b.N; i++ { 31 | f := int64(binary.BigEndian.Uint64(buf)) 32 | binary.BigEndian.PutUint64(buf, uint64(f+1)) 33 | } 34 | } 35 | 36 | func BenchmarkMathIntLittleEndian(b *testing.B) { 37 | buf := make([]byte, 8) 38 | b.ResetTimer() 39 | for i := 0; i < b.N; i++ { 40 | f := int64(binary.LittleEndian.Uint64(buf)) 41 | binary.LittleEndian.PutUint64(buf, uint64(f+1)) 42 | } 43 | } 44 | 45 | func BenchmarkMathUintBigEndian(b *testing.B) { 46 | buf := make([]byte, 8) 47 | b.ResetTimer() 48 | for i := 0; i < b.N; i++ { 49 | f := binary.BigEndian.Uint64(buf) 50 | binary.BigEndian.PutUint64(buf, f+1) 51 | } 52 | } 53 | 54 | func BenchmarkMathUintLittleEndian(b *testing.B) { 55 | buf := make([]byte, 8) 56 | b.ResetTimer() 57 | for i := 0; i < b.N; i++ { 58 | f := binary.LittleEndian.Uint64(buf) 59 | binary.LittleEndian.PutUint64(buf, f+1) 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /merge.go: -------------------------------------------------------------------------------- 1 | package zenodb 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "io/ioutil" 7 | "os" 8 | 9 | "github.com/golang/snappy" 10 | 11 | "github.com/getlantern/bytemap" 12 | "github.com/getlantern/errors" 13 | "github.com/getlantern/goexpr" 14 | "github.com/getlantern/golog" 15 | "github.com/getlantern/wal" 16 | 17 | "github.com/getlantern/zenodb/common" 18 | "github.com/getlantern/zenodb/core" 19 | "github.com/getlantern/zenodb/encoding" 20 | "github.com/getlantern/zenodb/sql" 21 | ) 22 | 23 | var ( 24 | emptyOffset = make(wal.Offset, wal.OffsetSize) 25 | ) 26 | 27 | // FilterAndMerge merges the specified inFiles into the given outFile, where 28 | // inFiles and outFiles are all valid filestore files. The schema is based on 29 | // the named table. If whereClause is specified, rows are filtered by comparing 30 | // them to the whereClause. The merge is performed as a disk-based merge in 31 | // order to use minimal memory. If shouldSort is true, the output will be sorted 32 | // by key. 33 | func (db *DB) FilterAndMerge(table string, whereClause string, shouldSort bool, outFile string, inFiles ...string) error { 34 | t := db.getTable(table) 35 | if t == nil { 36 | return errors.New("Table %v not found", table) 37 | } 38 | return t.filterAndMerge(whereClause, shouldSort, outFile, inFiles) 39 | } 40 | 41 | // FileInfo returns information about the given data file 42 | func FileInfo(inFile string) (offsetsBySource common.OffsetsBySource, fieldsString string, fields core.Fields, err error) { 43 | fs := &fileStore{ 44 | filename: inFile, 45 | } 46 | file, err := os.OpenFile(fs.filename, os.O_RDONLY, 0) 47 | if err != nil { 48 | err = errors.New("Unable to open filestore at %v: %v", fs.filename, err) 49 | return 50 | } 51 | defer file.Close() 52 | r := snappy.NewReader(file) 53 | return fs.info(r) 54 | } 55 | 56 | // Check checks all of the given inFiles for readability and returns errors 57 | // for all files that are in error. 58 | func Check(inFiles ...string) map[string]error { 59 | errors := make(map[string]error) 60 | for _, inFile := range inFiles { 61 | fs := &fileStore{ 62 | filename: inFile, 63 | t: &table{ 64 | log: golog.LoggerFor("check"), 65 | }, 66 | } 67 | file, err := os.OpenFile(fs.filename, os.O_RDONLY, 0) 68 | if err != nil { 69 | errors[inFile] = fmt.Errorf("Unable to open filestore at %v: %v", fs.filename, err) 70 | continue 71 | } 72 | defer file.Close() 73 | r := snappy.NewReader(file) 74 | _, _, _, err = fs.info(r) 75 | if err != nil { 76 | errors[inFile] = err 77 | continue 78 | } 79 | n, err := io.Copy(ioutil.Discard, r) 80 | if err != nil { 81 | errors[inFile] = fmt.Errorf("%v after %d bytes read", err, n) 82 | } 83 | fmt.Printf("Read %d uncompressed bytes from %v\n", n, inFile) 84 | } 85 | return errors 86 | } 87 | 88 | // CheckTable checks the given data file for the given table to make sure it's readable 89 | func (db *DB) CheckTable(table string, filename string) error { 90 | t := db.getTable(table) 91 | if t == nil { 92 | return errors.New("Table %v not found", table) 93 | } 94 | fs := &fileStore{ 95 | t: t, 96 | fields: t.fields, 97 | filename: filename, 98 | } 99 | numRows := 0 100 | _, err := fs.iterate(t.fields, nil, true, false, func(key bytemap.ByteMap, columns []encoding.Sequence, raw []byte) (bool, error) { 101 | numRows++ 102 | return true, nil 103 | }) 104 | if err != nil { 105 | return errors.New("Encountered error after reading %d rows: %v", numRows, err) 106 | } 107 | fmt.Printf("Read %d rows\n", numRows) 108 | return nil 109 | } 110 | 111 | func (t *table) filterAndMerge(whereClause string, shouldSort bool, outFile string, inFiles []string) error { 112 | // TODO: make this work with multi-source offsets 113 | // okayToReuseBuffers := false 114 | // rawOkay := false 115 | 116 | // filter, err := whereFor(whereClause) 117 | // if err != nil { 118 | // return err 119 | // } 120 | 121 | // // Find highest offset amongst all infiles 122 | // var offset wal.Offset 123 | // for _, inFile := range inFiles { 124 | // nextOffset, _, offsetErr := readWALOffset(inFile) 125 | // if offsetErr != nil { 126 | // return errors.New("Unable to read WAL offset from %v: %v", inFile, offsetErr) 127 | // } 128 | // if nextOffset.After(offset) { 129 | // offset = nextOffset 130 | // } 131 | // } 132 | 133 | // // Create output file 134 | // out, err := os.OpenFile(outFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) 135 | // if err != nil { 136 | // return errors.New("Unable to create outFile at %v: %v", outFile, err) 137 | // } 138 | // defer out.Close() 139 | 140 | // fso := &fileStore{ 141 | // t: t, 142 | // fields: t.fields, 143 | // } 144 | // cout, err := fso.createOutWriter(out, t.fields, offset, shouldSort) 145 | // if err != nil { 146 | // return errors.New("Unable to create out writer for %v: %v", outFile, err) 147 | // } 148 | // defer cout.Close() 149 | 150 | // truncateBefore := t.truncateBefore() 151 | // for _, inFile := range inFiles { 152 | // fs := &fileStore{ 153 | // t: t, 154 | // fields: t.fields, 155 | // filename: inFile, 156 | // } 157 | // _, err = fs.iterate(t.fields, nil, okayToReuseBuffers, rawOkay, func(key bytemap.ByteMap, columns []encoding.Sequence, raw []byte) (bool, error) { 158 | // _, writeErr := fs.doWrite(cout, t.fields, filter, truncateBefore, shouldSort, key, columns, raw) 159 | // return true, writeErr 160 | // }) 161 | // if err != nil { 162 | // return errors.New("Error iterating on %v: %v", inFile, err) 163 | // } 164 | // } 165 | 166 | return nil 167 | } 168 | 169 | func whereFor(whereClause string) (goexpr.Expr, error) { 170 | if whereClause == "" { 171 | return nil, nil 172 | } 173 | 174 | sqlString := fmt.Sprintf("SELECT * FROM thetable WHERE %v", whereClause) 175 | query, err := sql.Parse(sqlString) 176 | if err != nil { 177 | return nil, fmt.Errorf("Unable to process where clause %v: %v", whereClause, err) 178 | } 179 | 180 | return query.Where, nil 181 | } 182 | -------------------------------------------------------------------------------- /metrics/metrics.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "sort" 5 | "sync" 6 | "time" 7 | 8 | "github.com/getlantern/wal" 9 | "github.com/getlantern/zenodb/common" 10 | ) 11 | 12 | var ( 13 | leaderStats *LeaderStats 14 | followerStats map[common.FollowerID]*FollowerStats 15 | partitionStats map[int]*PartitionStats 16 | 17 | mx sync.RWMutex 18 | ) 19 | 20 | func init() { 21 | reset() 22 | } 23 | 24 | func reset() { 25 | leaderStats = &LeaderStats{} 26 | followerStats = make(map[common.FollowerID]*FollowerStats, 0) 27 | partitionStats = make(map[int]*PartitionStats, 0) 28 | } 29 | 30 | // Stats are the overall stats 31 | type Stats struct { 32 | Leader *LeaderStats 33 | Followers sortedFollowerStats 34 | Partitions sortedPartitionStats 35 | } 36 | 37 | // LeaderStats provides stats for the cluster leader 38 | type LeaderStats struct { 39 | NumPartitions int 40 | ConnectedPartitions int 41 | ConnectedFollowers int 42 | CurrentlyReadingWAL string 43 | } 44 | 45 | // FollowerStats provides stats for a single follower 46 | type FollowerStats struct { 47 | FollowerID common.FollowerID 48 | Queued int 49 | } 50 | 51 | // PartitionStats provides stats for a single partition 52 | type PartitionStats struct { 53 | Partition int 54 | NumFollowers int 55 | } 56 | 57 | type sortedFollowerStats []*FollowerStats 58 | 59 | func (s sortedFollowerStats) Len() int { return len(s) } 60 | func (s sortedFollowerStats) Swap(i, j int) { s[i], s[j] = s[j], s[i] } 61 | func (s sortedFollowerStats) Less(i, j int) bool { 62 | return s[i].FollowerID.Partition < s[j].FollowerID.Partition || s[i].FollowerID.ID < s[j].FollowerID.ID 63 | } 64 | 65 | type sortedPartitionStats []*PartitionStats 66 | 67 | func (s sortedPartitionStats) Len() int { return len(s) } 68 | func (s sortedPartitionStats) Swap(i, j int) { s[i], s[j] = s[j], s[i] } 69 | func (s sortedPartitionStats) Less(i, j int) bool { 70 | return s[i].Partition < s[j].Partition 71 | } 72 | 73 | // SetNumPartitions sets the number of partitions in the cluster 74 | func SetNumPartitions(numPartitions int) { 75 | mx.Lock() 76 | leaderStats.NumPartitions = numPartitions 77 | mx.Unlock() 78 | } 79 | 80 | // CurrentlyReadingWAL indicates that we're currently reading the WAL at a given offset 81 | func CurrentlyReadingWAL(offset wal.Offset) { 82 | ts := offset.TS() 83 | mx.Lock() 84 | leaderStats.CurrentlyReadingWAL = ts.Format(time.RFC3339) 85 | mx.Unlock() 86 | } 87 | 88 | // FollowerJoined records the fact that a follower joined the leader 89 | func FollowerJoined(followerID common.FollowerID) { 90 | mx.Lock() 91 | defer mx.Unlock() 92 | getFollowerStats(followerID) // get follower stats to lazily initialize 93 | ps := partitionStats[followerID.Partition] 94 | if ps == nil { 95 | ps = &PartitionStats{Partition: followerID.Partition} 96 | partitionStats[followerID.Partition] = ps 97 | } 98 | ps.NumFollowers++ 99 | } 100 | 101 | // FollowerFailed records the fact that a follower failed (which is analogous to leaving) 102 | func FollowerFailed(followerID common.FollowerID) { 103 | mx.Lock() 104 | defer mx.Unlock() 105 | fs, found := followerStats[followerID] 106 | if found { 107 | delete(followerStats, followerID) 108 | partitionStats[fs.FollowerID.Partition].NumFollowers-- 109 | if partitionStats[fs.FollowerID.Partition].NumFollowers == 0 { 110 | delete(partitionStats, fs.FollowerID.Partition) 111 | } 112 | } 113 | } 114 | 115 | // QueuedForFollower records how many measurements are queued for a given Follower 116 | func QueuedForFollower(followerID common.FollowerID, queued int) { 117 | mx.Lock() 118 | defer mx.Unlock() 119 | fs, found := followerStats[followerID] 120 | if found { 121 | fs.Queued = queued 122 | } 123 | } 124 | 125 | func getFollowerStats(followerID common.FollowerID) *FollowerStats { 126 | fs, found := followerStats[followerID] 127 | if !found { 128 | fs = &FollowerStats{ 129 | FollowerID: followerID, 130 | Queued: 0, 131 | } 132 | followerStats[followerID] = fs 133 | } 134 | return fs 135 | } 136 | 137 | func GetStats() *Stats { 138 | mx.RLock() 139 | s := &Stats{ 140 | Leader: leaderStats, 141 | Followers: make(sortedFollowerStats, 0, len(followerStats)), 142 | Partitions: make(sortedPartitionStats, 0, len(partitionStats)), 143 | } 144 | 145 | for _, fs := range followerStats { 146 | s.Followers = append(s.Followers, fs) 147 | } 148 | for _, ps := range partitionStats { 149 | s.Partitions = append(s.Partitions, ps) 150 | } 151 | mx.RUnlock() 152 | 153 | sort.Sort(s.Followers) 154 | sort.Sort(s.Partitions) 155 | s.Leader.ConnectedPartitions = len(partitionStats) 156 | s.Leader.ConnectedFollowers = len(followerStats) 157 | return s 158 | } 159 | -------------------------------------------------------------------------------- /metrics/metrics_test.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/getlantern/wal" 8 | "github.com/getlantern/zenodb/common" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestMetrics(t *testing.T) { 14 | reset() 15 | 16 | ts := time.Now() 17 | FollowerJoined(common.FollowerID{1, 1}) 18 | FollowerJoined(common.FollowerID{1, 2}) 19 | FollowerJoined(common.FollowerID{2, 3}) 20 | FollowerJoined(common.FollowerID{2, 4}) 21 | CurrentlyReadingWAL(wal.NewOffsetForTS(ts)) 22 | QueuedForFollower(common.FollowerID{1, 1}, 11) 23 | QueuedForFollower(common.FollowerID{1, 2}, 22) 24 | QueuedForFollower(common.FollowerID{2, 3}, 33) 25 | QueuedForFollower(common.FollowerID{2, 4}, 44) 26 | 27 | s := GetStats() 28 | assert.Equal(t, 4, s.Leader.ConnectedFollowers) 29 | assert.Equal(t, ts.Format(time.RFC3339), s.Leader.CurrentlyReadingWAL) 30 | 31 | assert.Equal(t, 1, s.Followers[0].FollowerID.Partition) 32 | assert.Equal(t, 11, s.Followers[0].Queued) 33 | assert.Equal(t, 1, s.Followers[1].FollowerID.Partition) 34 | assert.Equal(t, 22, s.Followers[1].Queued) 35 | assert.Equal(t, 2, s.Followers[2].FollowerID.Partition) 36 | assert.Equal(t, 33, s.Followers[2].Queued) 37 | assert.Equal(t, 2, s.Followers[3].FollowerID.Partition) 38 | assert.Equal(t, 44, s.Followers[3].Queued) 39 | 40 | assert.Equal(t, 2, s.Leader.ConnectedPartitions) 41 | assert.Equal(t, 1, s.Partitions[0].Partition) 42 | assert.Equal(t, 2, s.Partitions[0].NumFollowers) 43 | assert.Equal(t, 2, s.Partitions[1].Partition) 44 | assert.Equal(t, 2, s.Partitions[1].NumFollowers) 45 | 46 | // Fail a couple of followers. Fail each twice to make sure we don't double- 47 | // subtract. 48 | FollowerFailed(common.FollowerID{1, 2}) 49 | FollowerFailed(common.FollowerID{1, 2}) 50 | FollowerFailed(common.FollowerID{2, 3}) 51 | FollowerFailed(common.FollowerID{2, 3}) 52 | 53 | s = GetStats() 54 | assert.Equal(t, 2, s.Leader.ConnectedFollowers) 55 | assert.Equal(t, 2, s.Leader.ConnectedPartitions) 56 | assert.Equal(t, 1, s.Partitions[0].Partition) 57 | assert.Equal(t, 1, s.Partitions[0].NumFollowers) 58 | assert.Equal(t, 2, s.Partitions[1].Partition) 59 | assert.Equal(t, 1, s.Partitions[1].NumFollowers) 60 | 61 | // Fail remaining followers. 62 | FollowerFailed(common.FollowerID{1, 1}) 63 | FollowerFailed(common.FollowerID{1, 1}) 64 | FollowerFailed(common.FollowerID{2, 4}) 65 | FollowerFailed(common.FollowerID{2, 4}) 66 | 67 | s = GetStats() 68 | assert.Zero(t, s.Leader.ConnectedFollowers) 69 | assert.Zero(t, s.Leader.ConnectedPartitions) 70 | 71 | // Re-join 72 | FollowerJoined(common.FollowerID{1, 1}) 73 | FollowerJoined(common.FollowerID{1, 2}) 74 | 75 | s = GetStats() 76 | assert.Equal(t, 2, s.Leader.ConnectedFollowers) 77 | assert.Equal(t, 1, s.Leader.ConnectedPartitions) 78 | } 79 | -------------------------------------------------------------------------------- /planner/having.go: -------------------------------------------------------------------------------- 1 | package planner 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/getlantern/zenodb/core" 8 | "github.com/getlantern/zenodb/sql" 9 | ) 10 | 11 | func addHaving(flat core.FlatRowSource, query *sql.Query) core.FlatRowSource { 12 | base := core.FlatRowFilter(flat, core.HavingFieldName, func(ctx context.Context, row *core.FlatRow, fields core.Fields) (*core.FlatRow, error) { 13 | havingIdx := len(fields) - 1 14 | include := row.Values[havingIdx] 15 | if include == 1 { 16 | // Removing having field 17 | row.Values = row.Values[:havingIdx] 18 | return row, nil 19 | } 20 | return nil, nil 21 | }) 22 | return &havingFilter{base} 23 | } 24 | 25 | type havingFilter struct { 26 | base core.FlatRowSource 27 | } 28 | 29 | func (f *havingFilter) Iterate(ctx context.Context, onFields core.OnFields, onRow core.OnFlatRow) (interface{}, error) { 30 | return f.base.Iterate(ctx, func(fields core.Fields) error { 31 | // Remove the "_having" field 32 | cleanedFields := make(core.Fields, 0, len(fields)) 33 | for _, field := range fields { 34 | if field.Name != core.HavingFieldName { 35 | cleanedFields = append(cleanedFields, field) 36 | } 37 | } 38 | return onFields(cleanedFields) 39 | }, onRow) 40 | } 41 | 42 | func (f *havingFilter) GetGroupBy() []core.GroupBy { 43 | return f.base.GetGroupBy() 44 | } 45 | 46 | func (f *havingFilter) GetResolution() time.Duration { 47 | return f.base.GetResolution() 48 | } 49 | 50 | func (f *havingFilter) GetAsOf() time.Time { 51 | return f.base.GetAsOf() 52 | } 53 | 54 | func (f *havingFilter) GetUntil() time.Time { 55 | return f.base.GetUntil() 56 | } 57 | 58 | func (f *havingFilter) GetSource() core.Source { 59 | switch t := f.base.(type) { 60 | case core.Transform: 61 | return t.GetSource() 62 | } 63 | return nil 64 | } 65 | 66 | func (f *havingFilter) String() string { 67 | return f.base.String() 68 | } 69 | -------------------------------------------------------------------------------- /planner/local.go: -------------------------------------------------------------------------------- 1 | package planner 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync/atomic" 7 | "time" 8 | 9 | "github.com/getlantern/bytemap" 10 | "github.com/getlantern/zenodb/core" 11 | "github.com/getlantern/zenodb/encoding" 12 | "github.com/getlantern/zenodb/sql" 13 | ) 14 | 15 | func planLocal(query *sql.Query, opts *Opts) (core.FlatRowSource, error) { 16 | fixupSubQuery(query, opts) 17 | 18 | var source core.RowSource 19 | var err error 20 | if query.FromSubQuery != nil { 21 | source, err = sourceForSubQuery(query, opts) 22 | if err != nil { 23 | return nil, err 24 | } 25 | } else { 26 | source, err = sourceForTable(query, opts) 27 | if err != nil { 28 | return nil, err 29 | } 30 | } 31 | 32 | now := opts.Now(query.From) 33 | asOf, asOfChanged, until, untilChanged := asOfUntilFor(query, opts, source, now) 34 | sourceAsOf := source.GetAsOf() 35 | if asOf.Before(sourceAsOf) { 36 | return nil, fmt.Errorf("Query asOf of %v is before table asOf of %v", asOf, sourceAsOf) 37 | } 38 | 39 | resolution, strideSlice, resolutionChanged, resolutionTruncated, err := resolutionFor(query, opts, source, asOf, until) 40 | if err != nil { 41 | return nil, err 42 | } 43 | 44 | if query.Where != nil { 45 | source, err = applySubQueryFilters(query, opts, source) 46 | if err != nil { 47 | return nil, err 48 | } 49 | } 50 | 51 | needsGroupBy := asOfChanged || untilChanged || resolutionChanged || 52 | !query.GroupByAll || query.HasSpecificFields || query.HasHaving || 53 | query.Crosstab != nil || strideSlice > 0 54 | if needsGroupBy { 55 | source = addGroupBy(source, query, resolutionTruncated || resolutionChanged, resolution, strideSlice) 56 | } 57 | 58 | flat := core.Flatten(source) 59 | 60 | if query.HasHaving { 61 | flat = addHaving(flat, query) 62 | } 63 | 64 | return addOrderLimitOffset(flat, query), nil 65 | } 66 | 67 | func sourceForSubQuery(query *sql.Query, opts *Opts) (core.RowSource, error) { 68 | subSource, err := Plan(query.FromSubQuery.SQL, opts) 69 | if err != nil { 70 | return nil, err 71 | } 72 | return core.Unflatten(subSource, query.FieldsNoHaving), nil 73 | } 74 | 75 | func sourceForTable(query *sql.Query, opts *Opts) (core.RowSource, error) { 76 | return opts.GetTable(query.From, func(tableFields core.Fields) (core.Fields, error) { 77 | if query.HasSelectAll { 78 | // For SELECT *, include all table fields 79 | return tableFields, nil 80 | } 81 | 82 | tableExprs := tableFields.Exprs() 83 | 84 | // Otherwise, figure out minimum set of fields needed by query 85 | includedFields := make([]bool, len(tableFields)) 86 | fields, err := query.Fields.Get(tableFields) 87 | if err != nil { 88 | return nil, err 89 | } 90 | for _, field := range fields { 91 | sms := field.Expr.SubMergers(tableExprs) 92 | for i, sm := range sms { 93 | if sm != nil { 94 | includedFields[i] = true 95 | } 96 | } 97 | } 98 | 99 | result := make(core.Fields, 0, len(tableFields)) 100 | for i, included := range includedFields { 101 | if included { 102 | result = append(result, tableFields[i]) 103 | } 104 | } 105 | 106 | return result, nil 107 | }) 108 | } 109 | 110 | func asOfUntilFor(query *sql.Query, opts *Opts, source core.RowSource, now time.Time) (time.Time, bool, time.Time, bool) { 111 | if query.AsOfOffset != 0 { 112 | query.AsOf = now.Add(query.AsOfOffset) 113 | } 114 | if query.UntilOffset != 0 { 115 | query.Until = now.Add(query.UntilOffset) 116 | } 117 | 118 | // Round asOf and until 119 | query.AsOf = encoding.RoundTimeUp(query.AsOf, source.GetResolution()) 120 | query.Until = encoding.RoundTimeUp(query.Until, source.GetResolution()) 121 | 122 | asOf := source.GetAsOf() 123 | asOfChanged := !query.AsOf.IsZero() && query.AsOf.UnixNano() != source.GetAsOf().UnixNano() 124 | if asOfChanged { 125 | asOf = query.AsOf 126 | } 127 | 128 | until := source.GetUntil() 129 | untilChanged := !query.Until.IsZero() && query.Until.UnixNano() != source.GetUntil().UnixNano() 130 | if untilChanged { 131 | until = query.Until 132 | } 133 | 134 | return asOf, asOfChanged, until, untilChanged 135 | } 136 | 137 | func resolutionFor(query *sql.Query, opts *Opts, source core.RowSource, asOf time.Time, until time.Time) (time.Duration, time.Duration, bool, bool, error) { 138 | resolution := query.Resolution 139 | var strideSlice time.Duration 140 | if resolution == 0 { 141 | resolution = source.GetResolution() 142 | } 143 | 144 | if query.Stride > 0 { 145 | if query.Stride%source.GetResolution() != 0 { 146 | return 0, 0, false, false, fmt.Errorf("Query stride '%v' is not an even multiple of table resolution '%v'", query.Stride, source.GetResolution()) 147 | } 148 | strideSlice = resolution 149 | resolution = query.Stride 150 | } 151 | 152 | resolutionTruncated := false 153 | window := until.Sub(asOf) 154 | if resolution > window { 155 | resolution = window 156 | resolutionTruncated = true 157 | } 158 | 159 | resolutionChanged := resolution != source.GetResolution() 160 | if resolutionChanged { 161 | if resolution < source.GetResolution() { 162 | return 0, 0, false, false, fmt.Errorf("Query resolution '%v' is higher than table resolution '%v'", resolution, source.GetResolution()) 163 | } 164 | if resolution%source.GetResolution() != 0 { 165 | return 0, 0, false, false, fmt.Errorf("Query resolution '%v' is not an even multiple of table resolution '%v'", resolution, source.GetResolution()) 166 | } 167 | } 168 | 169 | return resolution, strideSlice, resolutionChanged, resolutionTruncated, nil 170 | } 171 | 172 | func applySubQueryFilters(query *sql.Query, opts *Opts, source core.RowSource) (core.RowSource, error) { 173 | runSubQueries, subQueryPlanErr := planSubQueries(opts, query) 174 | if subQueryPlanErr != nil { 175 | return nil, subQueryPlanErr 176 | } 177 | 178 | hasRunSubqueries := int32(0) 179 | return core.RowFilter(source, query.WhereSQL, func(ctx context.Context, key bytemap.ByteMap, fields core.Fields, vals core.Vals) (bytemap.ByteMap, core.Vals, error) { 180 | if atomic.CompareAndSwapInt32(&hasRunSubqueries, 0, 1) { 181 | _, err := runSubQueries(ctx) 182 | if err != nil && err != core.ErrDeadlineExceeded { 183 | return nil, nil, err 184 | } 185 | } 186 | result := query.Where.Eval(key) 187 | if result != nil && result.(bool) { 188 | return key, vals, nil 189 | } 190 | return nil, nil, nil 191 | }), nil 192 | } 193 | -------------------------------------------------------------------------------- /planner/planner.go: -------------------------------------------------------------------------------- 1 | // Package planner provides functionality for planning the execution of queries. 2 | package planner 3 | 4 | import ( 5 | "time" 6 | 7 | "github.com/getlantern/golog" 8 | "github.com/getlantern/zenodb/core" 9 | "github.com/getlantern/zenodb/sql" 10 | ) 11 | 12 | var ( 13 | log = golog.LoggerFor("planner") 14 | ) 15 | 16 | type Table interface { 17 | core.RowSource 18 | GetPartitionBy() []string 19 | } 20 | 21 | type Opts struct { 22 | GetTable func(table string, includedFields func(tableFields core.Fields) (core.Fields, error)) (Table, error) 23 | Now func(table string) time.Time 24 | IsSubQuery bool 25 | SubQueryResults [][]interface{} 26 | QueryCluster QueryClusterFN 27 | } 28 | 29 | func Plan(sqlString string, opts *Opts) (core.FlatRowSource, error) { 30 | query, err := sql.Parse(sqlString) 31 | if err != nil { 32 | return nil, err 33 | } 34 | 35 | fixupSubQuery(query, opts) 36 | 37 | if opts.QueryCluster != nil { 38 | allowPushdown, err := pushdownAllowed(opts, query) 39 | if err != nil { 40 | return nil, err 41 | } 42 | if allowPushdown { 43 | return planClusterPushdown(opts, query) 44 | } 45 | if query.FromSubQuery == nil { 46 | return planClusterNonPushdown(opts, query) 47 | } 48 | } 49 | 50 | return planLocal(query, opts) 51 | } 52 | 53 | func addGroupBy(source core.RowSource, query *sql.Query, applyResolution bool, resolution time.Duration, strideSlice time.Duration) core.RowSource { 54 | opts := core.GroupOpts{ 55 | By: query.GroupBy, 56 | Crosstab: query.Crosstab, 57 | CrosstabIncludesTotal: query.CrosstabIncludesTotal, 58 | Fields: query.Fields, 59 | AsOf: query.AsOf, 60 | Until: query.Until, 61 | StrideSlice: strideSlice, 62 | } 63 | if applyResolution { 64 | opts.Resolution = resolution 65 | } 66 | return core.Group(source, opts) 67 | } 68 | 69 | func addOrderLimitOffset(flat core.FlatRowSource, query *sql.Query) core.FlatRowSource { 70 | if len(query.OrderBy) > 0 { 71 | flat = core.Sort(flat, query.OrderBy...) 72 | } 73 | 74 | if query.Offset > 0 { 75 | flat = core.Offset(flat, query.Offset) 76 | } 77 | 78 | if query.Limit > 0 { 79 | flat = core.Limit(flat, query.Limit) 80 | } 81 | 82 | return flat 83 | } 84 | -------------------------------------------------------------------------------- /planner/subquery.go: -------------------------------------------------------------------------------- 1 | package planner 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | 8 | "github.com/getlantern/goexpr" 9 | "github.com/getlantern/zenodb/core" 10 | "github.com/getlantern/zenodb/sql" 11 | ) 12 | 13 | func planSubQueries(opts *Opts, query *sql.Query) (func(ctx context.Context) ([][]interface{}, error), error) { 14 | var subQueries []*sql.SubQuery 15 | query.Where.WalkLists(func(list goexpr.List) { 16 | sq, ok := list.(*sql.SubQuery) 17 | if ok { 18 | subQueries = append(subQueries, sq) 19 | } 20 | }) 21 | if len(opts.SubQueryResults) == len(subQueries) { 22 | for i, sq := range subQueries { 23 | sq.SetResult(opts.SubQueryResults[i]) 24 | } 25 | return noopSubQueries, nil 26 | } 27 | 28 | var subQueryPlans []core.FlatRowSource 29 | sqOpts := &Opts{} 30 | *sqOpts = *opts 31 | sqOpts.IsSubQuery = true 32 | for _, sq := range subQueries { 33 | sqPlan, sqPlanErr := Plan(sq.SQL, sqOpts) 34 | if sqPlanErr != nil { 35 | return nil, sqPlanErr 36 | } 37 | subQueryPlans = append(subQueryPlans, sqPlan) 38 | } 39 | 40 | if len(subQueries) == 0 { 41 | return noopSubQueries, nil 42 | } 43 | 44 | return func(ctx context.Context) ([][]interface{}, error) { 45 | // Run subqueries in parallel 46 | // TODO: respect ctx deadline 47 | sqResultChs := make(chan chan *sqResult, len(subQueries)) 48 | for _i, _sq := range subQueries { 49 | i := _i 50 | sq := _sq 51 | sqResultCh := make(chan *sqResult) 52 | sqResultChs <- sqResultCh 53 | go func() { 54 | var mx sync.Mutex 55 | uniques := make(map[interface{}]bool, 0) 56 | sqPlan := subQueryPlans[i] 57 | onRow := func(row *core.FlatRow) (bool, error) { 58 | dim := row.Key.Get(sq.Dim) 59 | mx.Lock() 60 | uniques[dim] = true 61 | mx.Unlock() 62 | return true, nil 63 | } 64 | _, err := sqPlan.Iterate(ctx, core.FieldsIgnored, onRow) 65 | 66 | dims := make([]interface{}, 0, len(uniques)) 67 | if err == nil || err == core.ErrDeadlineExceeded { 68 | for dim := range uniques { 69 | dims = append(dims, dim) 70 | } 71 | sq.SetResult(dims) 72 | } 73 | 74 | sqResultCh <- &sqResult{dims, err} 75 | }() 76 | } 77 | 78 | subQueryResults := make([][]interface{}, 0, len(subQueries)) 79 | var finalErr error 80 | for i := 0; i < len(subQueries); i++ { 81 | sqResultCh := <-sqResultChs 82 | result := <-sqResultCh 83 | err := result.err 84 | if err != nil && (finalErr == nil || finalErr == core.ErrDeadlineExceeded) { 85 | finalErr = err 86 | } 87 | subQueryResults = append(subQueryResults, result.dims) 88 | } 89 | return subQueryResults, finalErr 90 | }, nil 91 | } 92 | 93 | type sqResult struct { 94 | dims []interface{} 95 | err error 96 | } 97 | 98 | func noopSubQueries(ctx context.Context) ([][]interface{}, error) { 99 | return nil, nil 100 | } 101 | 102 | func fixupSubQuery(query *sql.Query, opts *Opts) { 103 | if opts.IsSubQuery { 104 | // Change field to _points field 105 | query.Fields = &pointsAndHavingFieldSource{query.Fields} 106 | } 107 | } 108 | 109 | // pointsAndHavingFieldSource is a FieldSource that wraps an existing 110 | // FieldSource and returns only the _having field (if present) and the _points 111 | // field. 112 | type pointsAndHavingFieldSource struct { 113 | wrapped core.FieldSource 114 | } 115 | 116 | func (phfs pointsAndHavingFieldSource) Get(known core.Fields) (core.Fields, error) { 117 | var result core.Fields 118 | origFields, err := phfs.wrapped.Get(known) 119 | if err != nil { 120 | return result, err 121 | } 122 | result = append(result, core.PointsField) 123 | for _, field := range origFields { 124 | if field.Name == core.HavingFieldName { 125 | result = append(result, field) 126 | } 127 | } 128 | return result, nil 129 | } 130 | 131 | func (phfs pointsAndHavingFieldSource) String() string { 132 | return fmt.Sprintf("pointsAndHaving(%s)", phfs.wrapped) 133 | } 134 | -------------------------------------------------------------------------------- /query.go: -------------------------------------------------------------------------------- 1 | package zenodb 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "time" 8 | 9 | "github.com/getlantern/bytemap" 10 | 11 | "github.com/getlantern/zenodb/common" 12 | "github.com/getlantern/zenodb/core" 13 | "github.com/getlantern/zenodb/encoding" 14 | "github.com/getlantern/zenodb/planner" 15 | "github.com/getlantern/zenodb/sql" 16 | ) 17 | 18 | var ( 19 | ErrOutOfMemory = errors.New("out of memory") 20 | ) 21 | 22 | func (db *DB) Query(sqlString string, isSubQuery bool, subQueryResults [][]interface{}, includeMemStore bool) (core.FlatRowSource, error) { 23 | q, err := sql.Parse(sqlString) 24 | if err != nil { 25 | return nil, err 26 | } 27 | 28 | if q.ForceFresh { 29 | db.log.Debug("Query requires fresh results, including mem store") 30 | includeMemStore = true 31 | } 32 | 33 | opts := &planner.Opts{ 34 | GetTable: func(table string, outFields func(tableFields core.Fields) (core.Fields, error)) (planner.Table, error) { 35 | return db.getQueryable(table, outFields, includeMemStore) 36 | }, 37 | Now: db.now, 38 | IsSubQuery: isSubQuery, 39 | SubQueryResults: subQueryResults, 40 | } 41 | if db.opts.Passthrough { 42 | opts.QueryCluster = func(ctx context.Context, sqlString string, isSubQuery bool, subQueryResults [][]interface{}, unflat bool, onFields core.OnFields, onRow core.OnRow, onFlatRow core.OnFlatRow) (interface{}, error) { 43 | return db.queryCluster(ctx, sqlString, isSubQuery, subQueryResults, includeMemStore, unflat, onFields, onRow, onFlatRow) 44 | } 45 | } 46 | plan, err := planner.Plan(sqlString, opts) 47 | if err != nil { 48 | return nil, err 49 | } 50 | db.log.Debugf("\n------------ Query Plan ------------\n\n%v\n\n%v\n----------- End Query Plan ----------", sqlString, core.FormatSource(plan)) 51 | return plan, nil 52 | } 53 | 54 | func (db *DB) getQueryable(table string, outFields func(tableFields core.Fields) (core.Fields, error), includeMemStore bool) (*queryable, error) { 55 | t := db.getTable(table) 56 | if t == nil { 57 | return nil, fmt.Errorf("Table %v not found", table) 58 | } 59 | if t.Virtual { 60 | return nil, fmt.Errorf("Table %v is virtual and cannot be queried", table) 61 | } 62 | until := encoding.RoundTimeUp(db.clock.Now(), t.Resolution) 63 | asOf := encoding.RoundTimeUp(until.Add(-1*t.RetentionPeriod), t.Resolution) 64 | fields := t.getFields() 65 | out, err := outFields(fields) 66 | if err != nil { 67 | return nil, err 68 | } 69 | if out == nil { 70 | out = t.getFields() 71 | } 72 | return &queryable{db, t, out, asOf, until, includeMemStore}, nil 73 | } 74 | 75 | func MetaDataFor(source core.FlatRowSource, fields core.Fields) *common.QueryMetaData { 76 | return &common.QueryMetaData{ 77 | FieldNames: fields.Names(), 78 | AsOf: source.GetAsOf(), 79 | Until: source.GetUntil(), 80 | Resolution: source.GetResolution(), 81 | Plan: core.FormatSource(source), 82 | } 83 | } 84 | 85 | type queryable struct { 86 | db *DB 87 | t *table 88 | fields core.Fields 89 | asOf time.Time 90 | until time.Time 91 | includeMemStore bool 92 | } 93 | 94 | func (q *queryable) GetGroupBy() []core.GroupBy { 95 | return q.t.GroupBy 96 | } 97 | 98 | func (q *queryable) GetResolution() time.Duration { 99 | return q.t.Resolution 100 | } 101 | 102 | func (q *queryable) GetAsOf() time.Time { 103 | return q.asOf 104 | } 105 | 106 | func (q *queryable) GetUntil() time.Time { 107 | return q.until 108 | } 109 | 110 | func (q *queryable) GetPartitionBy() []string { 111 | return q.t.PartitionBy 112 | } 113 | 114 | func (q *queryable) String() string { 115 | return q.t.Name 116 | } 117 | 118 | func (q *queryable) Iterate(ctx context.Context, onFields core.OnFields, onRow core.OnRow) (interface{}, error) { 119 | // We report all fields from the table 120 | err := onFields(q.fields) 121 | if err != nil { 122 | return nil, err 123 | } 124 | 125 | if len(q.fields) == 0 { 126 | return nil, errors.New("No fields found!") 127 | } 128 | 129 | i := 1 130 | // When iterating, as an optimization, we read only the needed fields (not 131 | // all table fields). 132 | highWaterMarks, err := q.t.iterate(ctx, q.fields, q.includeMemStore, func(key bytemap.ByteMap, vals []encoding.Sequence) (bool, error) { 133 | if i%1000 == 0 { 134 | // every 1000 rows, check and cap memory size 135 | if !q.db.capMemorySize(false) { 136 | q.t.log.Error("Returning ErrOutOfMemory") 137 | return false, ErrOutOfMemory 138 | } 139 | } 140 | i++ 141 | return onRow(key, vals) 142 | }) 143 | if err != nil { 144 | q.t.log.Errorf("Error on iterating: %v", err) 145 | } 146 | numSuccessfulPartitions := 0 147 | if err == nil { 148 | numSuccessfulPartitions = 1 149 | } 150 | return &common.QueryStats{ 151 | NumPartitions: 1, 152 | NumSuccessfulPartitions: numSuccessfulPartitions, 153 | LowestHighWaterMark: common.TimeToMillis(highWaterMarks.LowestTS()), 154 | HighestHighWaterMark: common.TimeToMillis(highWaterMarks.HighestTS()), 155 | }, err 156 | } 157 | -------------------------------------------------------------------------------- /quickstart_aliases.props: -------------------------------------------------------------------------------- 1 | IS_SUCCESS = %v = 200 2 | PERCENT = %v * 100 3 | -------------------------------------------------------------------------------- /quickstart_schema.yaml: -------------------------------------------------------------------------------- 1 | base: 2 | virtual: true 3 | sql: > 4 | SELECT 5 | requests, 6 | AVG(load_avg) AS load_avg 7 | FROM inbound 8 | GROUP BY *, period(15s) 9 | 10 | combined: 11 | retentionperiod: 1h 12 | maxflushlatency: 30s 13 | view: true 14 | sql: > 15 | SELECT * 16 | FROM base 17 | WHERE exclude IS NULL 18 | 19 | derived: 20 | retentionperiod: 1h 21 | maxflushlatency: 30s 22 | view: true 23 | sql: > 24 | SELECT requests + load_avg AS rpla 25 | FROM combined 26 | WHERE exclude IS NULL 27 | 28 | partitioned: 29 | retentionperiod: 1h 30 | maxflushlatency: 30s 31 | view: true 32 | partitionby: [server,path] 33 | sql: > 34 | SELECT * 35 | FROM combined 36 | 37 | newish: 38 | retentionperiod: 1h 39 | backfill: 1s 40 | maxflushlatency: 30s 41 | view: true 42 | partitionby: [server,path] 43 | sql: > 44 | SELECT * 45 | FROM combined 46 | 47 | backfilled: 48 | retentionperiod: 1h 49 | backfill: 30m 50 | maxflushlatency: 30s 51 | view: true 52 | partitionby: [server,path] 53 | sql: > 54 | SELECT * 55 | FROM combined 56 | 57 | notbackfilled: 58 | retentionperiod: 1h 59 | backfill: 1s 60 | maxflushlatency: 30s 61 | view: true 62 | partitionby: [server,path] 63 | sql: > 64 | SELECT * 65 | FROM combined 66 | 67 | notbackfilled2: 68 | retentionperiod: 1h 69 | backfill: 1s 70 | maxflushlatency: 30s 71 | view: true 72 | partitionby: [server,path] 73 | sql: > 74 | SELECT * 75 | FROM combined 76 | -------------------------------------------------------------------------------- /row_store_test.go: -------------------------------------------------------------------------------- 1 | package zenodb 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "testing" 7 | 8 | "github.com/getlantern/golog" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestStorage(t *testing.T) { 13 | tmpDir, err := ioutil.TempDir("", "zenodbtest") 14 | if !assert.NoError(t, err, "Unable to create temp directory") { 15 | return 16 | } 17 | defer os.RemoveAll(tmpDir) 18 | 19 | tb := &table{ 20 | log: golog.LoggerFor("storagetest"), 21 | db: &DB{}, 22 | } 23 | cs, _, err := tb.openRowStore(&rowStoreOptions{ 24 | dir: tmpDir, 25 | }) 26 | if !assert.NoError(t, err) { 27 | return 28 | } 29 | 30 | if false { 31 | cs.insert(&insert{}) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /rpc/msgpack_codec.go: -------------------------------------------------------------------------------- 1 | package rpc 2 | 3 | import ( 4 | "github.com/getlantern/msgpack" 5 | ) 6 | 7 | type MsgPackCodec struct { 8 | } 9 | 10 | func (c *MsgPackCodec) Marshal(v interface{}) ([]byte, error) { 11 | return msgpack.Marshal(v) 12 | } 13 | 14 | func (c *MsgPackCodec) Unmarshal(data []byte, v interface{}) error { 15 | return msgpack.Unmarshal(data, v) 16 | } 17 | 18 | func (c *MsgPackCodec) String() string { 19 | return "MsgPackCodec" 20 | } 21 | -------------------------------------------------------------------------------- /rpc/rpc.go: -------------------------------------------------------------------------------- 1 | package rpc 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/getlantern/bytemap" 8 | "github.com/getlantern/golog" 9 | "github.com/getlantern/wal" 10 | "github.com/getlantern/zenodb/common" 11 | "github.com/getlantern/zenodb/core" 12 | "github.com/getlantern/zenodb/planner" 13 | "google.golang.org/grpc" 14 | ) 15 | 16 | const ( 17 | PasswordKey = "pwd" 18 | ) 19 | 20 | var ( 21 | log = golog.LoggerFor("zenodb.rpc") 22 | 23 | Codec = &MsgPackCodec{} 24 | ) 25 | 26 | type Insert struct { 27 | Stream string // note, only the first Insert in a batch needs to include the Stream 28 | TS int64 29 | Dims []byte 30 | Vals []byte 31 | EndOfInserts bool 32 | } 33 | 34 | type InsertReport struct { 35 | Received int 36 | Succeeded int 37 | Errors map[int]string 38 | } 39 | 40 | type Query struct { 41 | SQLString string 42 | IsSubQuery bool 43 | SubQueryResults [][]interface{} 44 | IncludeMemStore bool 45 | Unflat bool 46 | Deadline time.Time 47 | HasDeadline bool 48 | } 49 | 50 | type Point struct { 51 | Data []byte 52 | Offset wal.Offset 53 | } 54 | 55 | type SourceInfo struct { 56 | ID int 57 | } 58 | 59 | type RemoteQueryResult struct { 60 | Fields core.Fields 61 | Key bytemap.ByteMap 62 | Vals core.Vals 63 | Row *core.FlatRow 64 | Stats *common.QueryStats 65 | Error string 66 | EndOfResults bool 67 | } 68 | 69 | type RegisterQueryHandler struct { 70 | Partition int 71 | } 72 | 73 | type Client interface { 74 | NewInserter(ctx context.Context, stream string, opts ...grpc.CallOption) (Inserter, error) 75 | 76 | Query(ctx context.Context, sqlString string, includeMemStore bool, opts ...grpc.CallOption) (*common.QueryMetaData, func(onRow core.OnFlatRow) (*common.QueryStats, error), error) 77 | 78 | Follow(ctx context.Context, in *common.Follow, opts ...grpc.CallOption) (int, func() (data []byte, newOffset wal.Offset, err error), error) 79 | 80 | ProcessRemoteQuery(ctx context.Context, partition int, query planner.QueryClusterFN, timeout time.Duration, opts ...grpc.CallOption) error 81 | 82 | Close() error 83 | } 84 | 85 | type Server interface { 86 | Insert(stream grpc.ServerStream) error 87 | 88 | Query(*Query, grpc.ServerStream) error 89 | 90 | Follow(*common.Follow, grpc.ServerStream) error 91 | 92 | HandleRemoteQueries(r *RegisterQueryHandler, stream grpc.ServerStream) error 93 | } 94 | 95 | var ServiceDesc = grpc.ServiceDesc{ 96 | ServiceName: "zenodb", 97 | HandlerType: (*Server)(nil), 98 | Methods: []grpc.MethodDesc{}, 99 | Streams: []grpc.StreamDesc{ 100 | { 101 | StreamName: "query", 102 | Handler: queryHandler, 103 | ServerStreams: true, 104 | }, 105 | { 106 | StreamName: "follow", 107 | Handler: followHandler, 108 | ServerStreams: true, 109 | }, 110 | { 111 | StreamName: "remoteQuery", 112 | Handler: remoteQueryHandler, 113 | ServerStreams: true, 114 | ClientStreams: true, 115 | }, 116 | { 117 | StreamName: "insert", 118 | Handler: insertHandler, 119 | ClientStreams: true, 120 | }, 121 | }, 122 | } 123 | 124 | func insertHandler(srv interface{}, stream grpc.ServerStream) error { 125 | return srv.(Server).Insert(stream) 126 | } 127 | 128 | func queryHandler(srv interface{}, stream grpc.ServerStream) error { 129 | q := new(Query) 130 | if err := stream.RecvMsg(q); err != nil { 131 | return err 132 | } 133 | return srv.(Server).Query(q, stream) 134 | } 135 | 136 | func followHandler(srv interface{}, stream grpc.ServerStream) error { 137 | f := new(common.Follow) 138 | if err := stream.RecvMsg(f); err != nil { 139 | return err 140 | } 141 | return srv.(Server).Follow(f, stream) 142 | } 143 | 144 | func remoteQueryHandler(srv interface{}, stream grpc.ServerStream) error { 145 | r := new(RegisterQueryHandler) 146 | if err := stream.RecvMsg(r); err != nil { 147 | return err 148 | } 149 | return srv.(Server).HandleRemoteQueries(r, stream) 150 | } 151 | -------------------------------------------------------------------------------- /rpc/rpc_client.go: -------------------------------------------------------------------------------- 1 | package rpc 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "net" 8 | "time" 9 | 10 | "github.com/getlantern/bytemap" 11 | "github.com/getlantern/errors" 12 | "github.com/getlantern/mtime" 13 | "github.com/getlantern/wal" 14 | "github.com/getlantern/zenodb/common" 15 | "github.com/getlantern/zenodb/core" 16 | "github.com/getlantern/zenodb/planner" 17 | "google.golang.org/grpc" 18 | "google.golang.org/grpc/metadata" 19 | ) 20 | 21 | type ClientOpts struct { 22 | // Password, if specified, is the password that client will present to server 23 | // in order to gain access. 24 | Password string 25 | 26 | Dialer func(string, time.Duration) (net.Conn, error) 27 | } 28 | 29 | type Inserter interface { 30 | Insert(ts time.Time, dims map[string]interface{}, vals func(func(string, interface{}))) error 31 | 32 | Close() (*InsertReport, error) 33 | } 34 | 35 | func Dial(addr string, opts *ClientOpts) (Client, error) { 36 | if opts.Dialer == nil { 37 | // Use default dialer 38 | opts.Dialer = func(addr string, timeout time.Duration) (net.Conn, error) { 39 | return net.DialTimeout("tcp", addr, timeout) 40 | } 41 | } 42 | 43 | opts.Dialer = snappyDialer(opts.Dialer) 44 | 45 | conn, err := grpc.Dial(addr, 46 | grpc.WithInsecure(), 47 | grpc.WithDialer(opts.Dialer), 48 | grpc.WithCodec(Codec), 49 | grpc.WithBackoffMaxDelay(1*time.Minute)) 50 | if err != nil { 51 | return nil, err 52 | } 53 | return &client{conn, opts.Password}, nil 54 | } 55 | 56 | type client struct { 57 | cc *grpc.ClientConn 58 | password string 59 | } 60 | 61 | type inserter struct { 62 | clientStream grpc.ClientStream 63 | streamName string 64 | } 65 | 66 | func (c *client) NewInserter(ctx context.Context, streamName string, opts ...grpc.CallOption) (Inserter, error) { 67 | clientStream, err := grpc.NewClientStream(ctx, &ServiceDesc.Streams[3], c.cc, "/zenodb/insert", opts...) 68 | if err != nil { 69 | return nil, err 70 | } 71 | 72 | return &inserter{ 73 | clientStream: clientStream, 74 | streamName: streamName, 75 | }, nil 76 | } 77 | 78 | func (i *inserter) Insert(ts time.Time, dims map[string]interface{}, vals func(func(string, interface{}))) error { 79 | iterate := func(cb func(string, interface{})) { 80 | vals(func(field string, val interface{}) { 81 | cb(field, val) 82 | }) 83 | } 84 | insert := &Insert{ 85 | Stream: i.streamName, 86 | TS: ts.UnixNano(), 87 | Dims: bytemap.New(dims), 88 | Vals: bytemap.Build(iterate, nil, true), 89 | } 90 | // Set streamName to "" to prevent sending it unnecessarily in subsequent inserts 91 | i.streamName = "" 92 | return i.clientStream.SendMsg(insert) 93 | } 94 | 95 | func (i *inserter) Close() (*InsertReport, error) { 96 | err := i.clientStream.SendMsg(&Insert{EndOfInserts: true}) 97 | if err != nil { 98 | return nil, fmt.Errorf("Unable to send closing message: %v", err) 99 | } 100 | err = i.clientStream.CloseSend() 101 | if err != nil { 102 | return nil, fmt.Errorf("Unable to close send: %v", err) 103 | } 104 | report := &InsertReport{} 105 | err = i.clientStream.RecvMsg(&report) 106 | if err != nil { 107 | return nil, fmt.Errorf("Error from server: %v", err) 108 | } 109 | return report, nil 110 | } 111 | 112 | func (c *client) Query(ctx context.Context, sqlString string, includeMemStore bool, opts ...grpc.CallOption) (*common.QueryMetaData, func(onRow core.OnFlatRow) (*common.QueryStats, error), error) { 113 | stream, err := grpc.NewClientStream(c.authenticated(ctx), &ServiceDesc.Streams[0], c.cc, "/zenodb/query", opts...) 114 | if err != nil { 115 | return nil, nil, err 116 | } 117 | if err = stream.SendMsg(&Query{SQLString: sqlString, IncludeMemStore: includeMemStore}); err != nil { 118 | return nil, nil, err 119 | } 120 | if err = stream.CloseSend(); err != nil { 121 | return nil, nil, err 122 | } 123 | 124 | md := &common.QueryMetaData{} 125 | err = stream.RecvMsg(md) 126 | if err != nil { 127 | return nil, nil, err 128 | } 129 | 130 | iterate := func(onRow core.OnFlatRow) (*common.QueryStats, error) { 131 | for { 132 | result := &RemoteQueryResult{} 133 | rowErr := stream.RecvMsg(result) 134 | if rowErr != nil { 135 | return nil, rowErr 136 | } 137 | if result.EndOfResults { 138 | return result.Stats, nil 139 | } 140 | more, rowErr := onRow(result.Row) 141 | if !more || rowErr != nil { 142 | return nil, rowErr 143 | } 144 | } 145 | } 146 | 147 | return md, iterate, nil 148 | } 149 | 150 | func (c *client) Follow(ctx context.Context, f *common.Follow, opts ...grpc.CallOption) (int, func() (data []byte, newOffset wal.Offset, err error), error) { 151 | stream, err := grpc.NewClientStream(c.authenticated(ctx), &ServiceDesc.Streams[1], c.cc, "/zenodb/follow", opts...) 152 | if err != nil { 153 | return 0, nil, err 154 | } 155 | if err := stream.SendMsg(f); err != nil { 156 | return 0, nil, err 157 | } 158 | if err := stream.CloseSend(); err != nil { 159 | return 0, nil, err 160 | } 161 | 162 | sourceInfo := &SourceInfo{} 163 | if err := stream.RecvMsg(sourceInfo); err != nil { 164 | return 0, nil, err 165 | } 166 | 167 | log.Debugf("Following leader %d", sourceInfo.ID) 168 | 169 | next := func() ([]byte, wal.Offset, error) { 170 | point := &Point{} 171 | err := stream.RecvMsg(point) 172 | if err != nil { 173 | return nil, nil, err 174 | } 175 | return point.Data, point.Offset, nil 176 | } 177 | 178 | return sourceInfo.ID, next, nil 179 | } 180 | 181 | func (c *client) ProcessRemoteQuery(ctx context.Context, partition int, query planner.QueryClusterFN, timeout time.Duration, opts ...grpc.CallOption) error { 182 | elapsed := mtime.Stopwatch() 183 | 184 | stream, err := grpc.NewClientStream(c.authenticated(ctx), &ServiceDesc.Streams[2], c.cc, "/zenodb/remoteQuery", opts...) 185 | if err != nil { 186 | return errors.New("Unable to obtain client stream: %v", err) 187 | } 188 | defer stream.CloseSend() 189 | 190 | if err := stream.SendMsg(&RegisterQueryHandler{partition}); err != nil { 191 | return errors.New("Unable to send registration message: %v", err) 192 | } 193 | 194 | queryCh := make(chan *Query) 195 | queryErrCh := make(chan error) 196 | 197 | go func() { 198 | q := &Query{} 199 | recvErr := stream.RecvMsg(q) 200 | if recvErr != nil { 201 | queryErrCh <- recvErr 202 | } 203 | queryCh <- q 204 | }() 205 | 206 | var q *Query 207 | select { 208 | case q = <-queryCh: 209 | log.Debug("Got query!") 210 | if q.SQLString == "" { 211 | log.Debug("Query was a noop, ignoring") 212 | return nil 213 | } 214 | case queryErr := <-queryErrCh: 215 | return errors.New("Unable to read query: %v", queryErr) 216 | case <-time.After(timeout): 217 | return errors.New("Didn't receive query within %v, closing connection", timeout) 218 | } 219 | 220 | defer func() { 221 | log.Debugf("Finished processing query in %v", elapsed()) 222 | }() 223 | 224 | onFields := func(fields core.Fields) error { 225 | return stream.SendMsg(&RemoteQueryResult{Fields: fields}) 226 | } 227 | var onRow core.OnRow 228 | var onFlatRow core.OnFlatRow 229 | 230 | if q.Unflat { 231 | onRow = func(key bytemap.ByteMap, vals core.Vals) (bool, error) { 232 | err := stream.SendMsg(&RemoteQueryResult{Key: key, Vals: vals}) 233 | return true, err 234 | } 235 | } else { 236 | onFlatRow = func(row *core.FlatRow) (bool, error) { 237 | err := stream.SendMsg(&RemoteQueryResult{Row: row}) 238 | return true, err 239 | } 240 | } 241 | 242 | streamCtx := stream.Context() 243 | if q.HasDeadline { 244 | var cancel context.CancelFunc 245 | streamCtx, cancel = context.WithDeadline(streamCtx, q.Deadline) 246 | defer cancel() 247 | } 248 | streamCtx = common.WithIncludeMemStore(streamCtx, q.IncludeMemStore) 249 | 250 | _stats, queryErr := query(streamCtx, q.SQLString, q.IsSubQuery, q.SubQueryResults, q.Unflat, onFields, onRow, onFlatRow) 251 | var stats *common.QueryStats 252 | if _stats != nil { 253 | stats = _stats.(*common.QueryStats) 254 | } 255 | result := &RemoteQueryResult{Stats: stats, EndOfResults: true} 256 | if queryErr != nil && queryErr != io.EOF { 257 | log.Debugf("Error on querying: %v", queryErr) 258 | result.Error = queryErr.Error() 259 | } 260 | stream.SendMsg(result) 261 | 262 | return nil 263 | } 264 | 265 | func (c *client) Close() error { 266 | return c.cc.Close() 267 | } 268 | 269 | func (c *client) authenticated(ctx context.Context) context.Context { 270 | if c.password == "" { 271 | return ctx 272 | } 273 | md := metadata.New(map[string]string{PasswordKey: c.password}) 274 | return metadata.NewOutgoingContext(ctx, md) 275 | } 276 | -------------------------------------------------------------------------------- /rpc/server/rpc_server.go: -------------------------------------------------------------------------------- 1 | package rpcserver 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net" 7 | "time" 8 | 9 | "github.com/getlantern/bytemap" 10 | "github.com/getlantern/errors" 11 | "github.com/getlantern/golog" 12 | "github.com/getlantern/wal" 13 | "github.com/getlantern/zenodb" 14 | "github.com/getlantern/zenodb/common" 15 | "github.com/getlantern/zenodb/core" 16 | "github.com/getlantern/zenodb/encoding" 17 | "github.com/getlantern/zenodb/planner" 18 | "github.com/getlantern/zenodb/rpc" 19 | "google.golang.org/grpc" 20 | "google.golang.org/grpc/metadata" 21 | ) 22 | 23 | type Opts struct { 24 | // ID uniquely identifies this server 25 | ID int 26 | 27 | // Password, if specified, is the password that clients must present in order 28 | // to access the server. 29 | Password string 30 | } 31 | 32 | // DB is an interface for database-like things (implemented by common.DB). 33 | type DB interface { 34 | InsertRaw(stream string, ts time.Time, dims bytemap.ByteMap, vals bytemap.ByteMap) error 35 | 36 | Query(sqlString string, isSubQuery bool, subQueryResults [][]interface{}, includeMemStore bool) (core.FlatRowSource, error) 37 | 38 | Follow(f *common.Follow, cb func([]byte, wal.Offset) error) 39 | 40 | RegisterQueryHandler(partition int, query planner.QueryClusterFN) 41 | } 42 | 43 | func PrepareServer(db DB, l net.Listener, opts *Opts) (func() error, func()) { 44 | l = &rpc.SnappyListener{l} 45 | gs := grpc.NewServer(grpc.CustomCodec(rpc.Codec)) 46 | gs.RegisterService(&rpc.ServiceDesc, &server{golog.LoggerFor(fmt.Sprintf("zenodb.rpc (%d)", opts.ID)), db, opts.ID, opts.Password}) 47 | return func() error { return gs.Serve(l) }, gs.Stop 48 | } 49 | 50 | type server struct { 51 | log golog.Logger 52 | db DB 53 | id int 54 | password string 55 | } 56 | 57 | func (s *server) Insert(stream grpc.ServerStream) error { 58 | // No need to authorize, anyone can insert 59 | 60 | now := time.Now() 61 | streamName := "" 62 | 63 | report := &rpc.InsertReport{ 64 | Errors: make(map[int]string), 65 | } 66 | 67 | i := -1 68 | for { 69 | i++ 70 | insert := &rpc.Insert{} 71 | err := stream.RecvMsg(insert) 72 | if err != nil { 73 | return fmt.Errorf("Error reading insert: %v", err) 74 | } 75 | if insert.EndOfInserts { 76 | // We're done inserting 77 | return stream.SendMsg(report) 78 | } 79 | report.Received++ 80 | 81 | if streamName == "" { 82 | streamName = insert.Stream 83 | if streamName == "" { 84 | return fmt.Errorf("Please specify a stream") 85 | } 86 | } 87 | 88 | if len(insert.Dims) == 0 { 89 | report.Errors[i] = fmt.Sprintf("Need at least one dim") 90 | continue 91 | } 92 | if len(insert.Vals) == 0 { 93 | report.Errors[i] = fmt.Sprintf("Need at least one val") 94 | continue 95 | } 96 | var ts time.Time 97 | if insert.TS == 0 { 98 | ts = now 99 | } else { 100 | ts = encoding.TimeFromInt(insert.TS) 101 | } 102 | 103 | // TODO: make sure we don't barf on invalid bytemaps here 104 | insertErr := s.db.InsertRaw(streamName, ts, bytemap.ByteMap(insert.Dims), bytemap.ByteMap(insert.Vals)) 105 | if insertErr != nil { 106 | report.Errors[i] = fmt.Sprintf("Unable to insert: %v", insertErr) 107 | continue 108 | } 109 | report.Succeeded++ 110 | } 111 | } 112 | 113 | func (s *server) Query(q *rpc.Query, stream grpc.ServerStream) error { 114 | authorizeErr := s.authorize(stream) 115 | if authorizeErr != nil { 116 | return authorizeErr 117 | } 118 | 119 | source, err := s.db.Query(q.SQLString, q.IsSubQuery, q.SubQueryResults, q.IncludeMemStore) 120 | if err != nil { 121 | return err 122 | } 123 | 124 | rr := &rpc.RemoteQueryResult{} 125 | stats, err := source.Iterate(stream.Context(), func(fields core.Fields) error { 126 | // Send query metadata 127 | md := zenodb.MetaDataFor(source, fields) 128 | return stream.SendMsg(md) 129 | }, func(row *core.FlatRow) (bool, error) { 130 | rr.Row = row 131 | return true, stream.SendMsg(rr) 132 | }) 133 | if err != nil { 134 | return err 135 | } 136 | 137 | // Send end of results 138 | rr.Row = nil 139 | if stats != nil { 140 | rr.Stats = stats.(*common.QueryStats) 141 | } 142 | rr.EndOfResults = true 143 | return stream.SendMsg(rr) 144 | } 145 | 146 | func (s *server) Follow(f *common.Follow, stream grpc.ServerStream) error { 147 | if authorizeErr := s.authorize(stream); authorizeErr != nil { 148 | return authorizeErr 149 | } 150 | 151 | s.log.Debugf("Follower %v joined", f.FollowerID) 152 | defer s.log.Debugf("Follower %v left", f.FollowerID) 153 | 154 | if err := stream.SendMsg(&rpc.SourceInfo{ID: s.id}); err != nil { 155 | return err 156 | } 157 | 158 | s.db.Follow(f, func(data []byte, newOffset wal.Offset) error { 159 | return stream.SendMsg(&rpc.Point{data, newOffset}) 160 | }) 161 | return nil 162 | } 163 | 164 | func (s *server) HandleRemoteQueries(r *rpc.RegisterQueryHandler, stream grpc.ServerStream) error { 165 | initialResultCh := make(chan *rpc.RemoteQueryResult) 166 | initialErrCh := make(chan error, 1) 167 | finalErrCh := make(chan error, 1) 168 | 169 | finish := func(err error) { 170 | select { 171 | case finalErrCh <- err: 172 | // ok 173 | default: 174 | // ignore 175 | } 176 | } 177 | 178 | s.db.RegisterQueryHandler(r.Partition, func(ctx context.Context, sqlString string, isSubQuery bool, subQueryResults [][]interface{}, unflat bool, onFields core.OnFields, onRow core.OnRow, onFlatRow core.OnFlatRow) (interface{}, error) { 179 | q := &rpc.Query{ 180 | SQLString: sqlString, 181 | IsSubQuery: isSubQuery, 182 | SubQueryResults: subQueryResults, 183 | Unflat: unflat, 184 | IncludeMemStore: common.ShouldIncludeMemStore(ctx), 185 | } 186 | q.Deadline, q.HasDeadline = ctx.Deadline() 187 | sendErr := stream.SendMsg(q) 188 | 189 | m, recvErr := <-initialResultCh, <-initialErrCh 190 | 191 | // Check send error after reading initial result to avoid blocking 192 | // unnecessarily 193 | if sendErr != nil { 194 | err := errors.New("Unable to send query: %v", sendErr) 195 | finish(err) 196 | return nil, common.MarkRetriable(err) 197 | } 198 | 199 | var finalErr error 200 | 201 | first := true 202 | receiveLoop: 203 | for { 204 | // Process current result 205 | if recvErr != nil { 206 | m.Error = recvErr.Error() 207 | finalErr = errors.New("Unable to receive result: %v", recvErr) 208 | if first { 209 | finalErr = common.MarkRetriable(finalErr) 210 | } 211 | break 212 | } 213 | 214 | if first { 215 | // First message contains only fields information 216 | onFields(m.Fields) 217 | first = false 218 | } else { 219 | if m.Error != "" { 220 | finalErr = errors.New(m.Error) 221 | } 222 | // Subsequent messages contain data 223 | if m.EndOfResults { 224 | break 225 | } 226 | var more bool 227 | var err error 228 | if unflat { 229 | more, err = onRow(m.Key, m.Vals) 230 | } else { 231 | more, err = onFlatRow(m.Row) 232 | } 233 | if !more || err != nil { 234 | finalErr = err 235 | break receiveLoop 236 | } 237 | } 238 | 239 | // Read next result 240 | m = &rpc.RemoteQueryResult{} 241 | recvErr = stream.RecvMsg(m) 242 | } 243 | 244 | s.log.Debugf("Finishing partition %d with finalErr: %v", r.Partition, finalErr) 245 | finish(finalErr) 246 | return m.Stats, finalErr 247 | }) 248 | 249 | // Block on reading initial result to keep connection open 250 | m := &rpc.RemoteQueryResult{} 251 | err := stream.RecvMsg(m) 252 | initialResultCh <- m 253 | initialErrCh <- err 254 | 255 | if err == nil { 256 | // Wait for final error so we don't close the connection prematurely 257 | err = <-finalErrCh 258 | } 259 | return err 260 | } 261 | 262 | func (s *server) authorize(stream grpc.ServerStream) error { 263 | if s.password == "" { 264 | s.log.Debug("No password specified, allowing access to world") 265 | return nil 266 | } 267 | md, ok := metadata.FromIncomingContext(stream.Context()) 268 | if !ok { 269 | return s.log.Error("No metadata provided, unable to authenticate") 270 | } 271 | passwords := md[rpc.PasswordKey] 272 | for _, password := range passwords { 273 | if password == s.password { 274 | // authorized 275 | return nil 276 | } 277 | } 278 | return s.log.Error("None of the provided passwords matched, not authorized!") 279 | } 280 | -------------------------------------------------------------------------------- /rpc/server/rpc_test.go: -------------------------------------------------------------------------------- 1 | package rpcserver 2 | 3 | import ( 4 | "context" 5 | "net" 6 | "sync/atomic" 7 | "testing" 8 | "time" 9 | 10 | "github.com/getlantern/bytemap" 11 | "github.com/getlantern/wal" 12 | "github.com/getlantern/zenodb/common" 13 | "github.com/getlantern/zenodb/core" 14 | "github.com/getlantern/zenodb/planner" 15 | "github.com/getlantern/zenodb/rpc" 16 | "github.com/stretchr/testify/assert" 17 | ) 18 | 19 | func TestInsert(t *testing.T) { 20 | l, err := net.Listen("tcp", ":0") 21 | if !assert.NoError(t, err) { 22 | return 23 | } 24 | defer l.Close() 25 | 26 | db := &mockDB{} 27 | start, _ := PrepareServer(db, l, &Opts{ 28 | Password: "password", 29 | }) 30 | go start() 31 | time.Sleep(1 * time.Second) 32 | 33 | client, err := rpc.Dial(l.Addr().String(), &rpc.ClientOpts{ 34 | Password: "password", 35 | Dialer: func(addr string, timeout time.Duration) (net.Conn, error) { 36 | return net.DialTimeout("tcp", addr, timeout) 37 | }, 38 | }) 39 | if !assert.NoError(t, err) { 40 | return 41 | } 42 | defer client.Close() 43 | 44 | inserter, err := client.NewInserter(context.Background(), "thestream") 45 | if !assert.NoError(t, err) { 46 | return 47 | } 48 | 49 | for i := 0; i < 10; i++ { 50 | dims := map[string]interface{}{"dim": "dimval"} 51 | if i > 1 && i < 7 { 52 | dims = nil 53 | } 54 | err = inserter.Insert(time.Time{}, dims, func(cb func(key string, value interface{})) { 55 | if i < 7 { 56 | cb("val", float64(i)) 57 | } 58 | }) 59 | if !assert.NoError(t, err, "Error on iteration %d", i) { 60 | return 61 | } 62 | } 63 | 64 | report, err := inserter.Close() 65 | if !assert.NoError(t, err) { 66 | return 67 | } 68 | 69 | assert.Equal(t, 10, report.Received) 70 | assert.Equal(t, 2, report.Succeeded) 71 | assert.Equal(t, 2, db.NumInserts()) 72 | for i := 2; i < 10; i++ { 73 | if i < 7 { 74 | assert.Equal(t, "Need at least one dim", report.Errors[i]) 75 | } else { 76 | assert.Equal(t, "Need at least one val", report.Errors[i]) 77 | } 78 | } 79 | } 80 | 81 | type mockDB struct { 82 | numInserts int64 83 | } 84 | 85 | func (db *mockDB) InsertRaw(stream string, ts time.Time, dims bytemap.ByteMap, vals bytemap.ByteMap) error { 86 | atomic.AddInt64(&db.numInserts, 1) 87 | return nil 88 | } 89 | 90 | func (db *mockDB) NumInserts() int { 91 | return int(atomic.LoadInt64(&db.numInserts)) 92 | } 93 | 94 | func (db *mockDB) Query(sqlString string, isSubQuery bool, subQueryResults [][]interface{}, includeMemStore bool) (core.FlatRowSource, error) { 95 | return nil, nil 96 | } 97 | 98 | func (db *mockDB) Follow(f *common.Follow, cb func([]byte, wal.Offset) error) { 99 | } 100 | 101 | func (db *mockDB) RegisterQueryHandler(partition int, query planner.QueryClusterFN) { 102 | 103 | } 104 | -------------------------------------------------------------------------------- /rpc/snappyconn.go: -------------------------------------------------------------------------------- 1 | package rpc 2 | 3 | import ( 4 | "net" 5 | "sync" 6 | "time" 7 | 8 | "github.com/golang/snappy" 9 | ) 10 | 11 | func snappyDialer(d func(string, time.Duration) (net.Conn, error)) func(addr string, timeout time.Duration) (net.Conn, error) { 12 | return func(addr string, timeout time.Duration) (net.Conn, error) { 13 | return snappyWrap(d(addr, timeout)) 14 | } 15 | } 16 | 17 | type SnappyListener struct { 18 | net.Listener 19 | } 20 | 21 | func (sl *SnappyListener) Accept() (net.Conn, error) { 22 | return snappyWrap(sl.Listener.Accept()) 23 | } 24 | 25 | func snappyWrap(conn net.Conn, err error) (net.Conn, error) { 26 | if err != nil { 27 | return nil, err 28 | } 29 | r := snappy.NewReader(conn) 30 | // Note we don't use a buffered writer here as it doesn't seem to work well 31 | // with gRPC for some reason. In particular, without explicitly flushing, it 32 | // sometimes doesn't transmit messages completely, and with explicit flushing 33 | // the throughput seems low. TODO: figure out if we can buffer here. 34 | w := snappy.NewWriter(conn) 35 | sc := &snappyConn{Conn: conn, r: r, w: w} 36 | return sc, nil 37 | } 38 | 39 | type snappyConn struct { 40 | net.Conn 41 | r *snappy.Reader 42 | w *snappy.Writer 43 | mx sync.Mutex 44 | } 45 | 46 | func (sc *snappyConn) Read(p []byte) (int, error) { 47 | return sc.r.Read(p) 48 | } 49 | 50 | func (sc *snappyConn) Write(p []byte) (int, error) { 51 | sc.mx.Lock() 52 | n, err := sc.w.Write(p) 53 | sc.mx.Unlock() 54 | return n, err 55 | } 56 | 57 | func (sc *snappyConn) Close() error { 58 | sc.mx.Lock() 59 | sc.w.Close() 60 | sc.mx.Unlock() 61 | return sc.Conn.Close() 62 | } 63 | -------------------------------------------------------------------------------- /schema.go: -------------------------------------------------------------------------------- 1 | package zenodb 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "strings" 8 | "time" 9 | 10 | "github.com/getlantern/yaml" 11 | "github.com/getlantern/zenodb/sql" 12 | ) 13 | 14 | type Schema map[string]*TableOpts 15 | 16 | func (db *DB) pollForSchema(filename string) error { 17 | stat, err := os.Stat(filename) 18 | if err != nil { 19 | return err 20 | } 21 | 22 | err = db.ApplySchemaFromFile(filename) 23 | if err != nil { 24 | db.log.Error(err) 25 | return err 26 | } 27 | 28 | db.Go(func(stop <-chan interface{}) { 29 | db.log.Debug("Polling for schema changes") 30 | 31 | ticker := time.NewTicker(1 * time.Second) 32 | defer ticker.Stop() 33 | 34 | for { 35 | select { 36 | case <-stop: 37 | return 38 | case <-ticker.C: 39 | newStat, err := os.Stat(filename) 40 | if err != nil { 41 | db.log.Errorf("Unable to stat schema: %v", err) 42 | } else if newStat.ModTime().After(stat.ModTime()) || newStat.Size() != stat.Size() { 43 | db.log.Debug("Schema file changed, applying") 44 | applyErr := db.ApplySchemaFromFile(filename) 45 | if applyErr != nil { 46 | db.log.Error(applyErr) 47 | } 48 | stat = newStat 49 | } 50 | } 51 | } 52 | }) 53 | 54 | return nil 55 | } 56 | 57 | func (db *DB) ApplySchemaFromFile(filename string) error { 58 | b, err := ioutil.ReadFile(filename) 59 | if err != nil { 60 | return err 61 | } 62 | var schema Schema 63 | err = yaml.Unmarshal(b, &schema) 64 | if err != nil { 65 | db.log.Errorf("Error applying schema: %v", err) 66 | db.log.Debug(string(b)) 67 | return err 68 | } 69 | return db.ApplySchema(schema) 70 | } 71 | 72 | func (db *DB) ApplySchema(_schema Schema) error { 73 | schema := make(Schema, len(_schema)) 74 | // Convert all names in schema to lowercase 75 | for name, opts := range _schema { 76 | opts.Name = strings.ToLower(name) 77 | schema[opts.Name] = opts 78 | } 79 | 80 | // Identify dependencies 81 | var tables []*TableOpts 82 | for name, opts := range schema { 83 | if !opts.View { 84 | tables = append(tables, opts) 85 | } else { 86 | dependsOn, err := sql.TableFor(opts.SQL) 87 | if err != nil { 88 | return fmt.Errorf("Unable to determine underlying table for view %v: %v", name, err) 89 | } 90 | table, found := schema[dependsOn] 91 | if !found { 92 | return fmt.Errorf("Table %v needed by view %v not found", name, dependsOn) 93 | } 94 | table.dependencyOf = append(table.dependencyOf, opts) 95 | } 96 | } 97 | // Apply tables in order of dependencies 98 | bd := &byDependency{} 99 | for _, opts := range tables { 100 | bd.add(opts) 101 | } 102 | db.log.Debugf("Applying tables in order: %v", strings.Join(bd.names, ", ")) 103 | for _, opts := range bd.opts { 104 | name := opts.Name 105 | t := db.getTable(name) 106 | tableType := "table" 107 | if opts.View { 108 | tableType = "view" 109 | } 110 | if t == nil { 111 | db.log.Debugf("Creating %v '%v' as\n%v", tableType, name, opts.SQL) 112 | db.log.Debugf("MaxFlushLatency: %v MinFlushLatency: %v", opts.MaxFlushLatency, opts.MinFlushLatency) 113 | err := db.CreateTable(opts) 114 | if err != nil { 115 | return fmt.Errorf("Error creating table %v: %v", name, err) 116 | } 117 | db.log.Debugf("Created %v %v", tableType, name) 118 | } else { 119 | db.log.Debugf("Altering %v '%v' as \n%v", tableType, name, opts.SQL) 120 | err := t.Alter(opts) 121 | if err != nil { 122 | return err 123 | } 124 | } 125 | } 126 | 127 | return nil 128 | } 129 | 130 | type byDependency struct { 131 | opts []*TableOpts 132 | names []string 133 | } 134 | 135 | func (bd *byDependency) add(opts *TableOpts) { 136 | bd.opts = append(bd.opts, opts) 137 | bd.names = append(bd.names, opts.Name) 138 | for _, dep := range opts.dependencyOf { 139 | bd.add(dep) 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /server/signal.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "os" 5 | "os/signal" 6 | "syscall" 7 | ) 8 | 9 | func (s *Server) HandleShutdownSignal() { 10 | c := make(chan os.Signal, 1) 11 | signal.Notify(c, 12 | syscall.SIGHUP, 13 | syscall.SIGINT, 14 | syscall.SIGTERM, 15 | syscall.SIGQUIT) 16 | go func() { 17 | sig := <-c 18 | s.log.Debugf("Got signal \"%s\", closing db and exiting...", sig) 19 | s.Close() 20 | os.Exit(0) 21 | }() 22 | } 23 | -------------------------------------------------------------------------------- /sql/duration.go: -------------------------------------------------------------------------------- 1 | package sql 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "time" 7 | ) 8 | 9 | const ( 10 | day = 24 * time.Hour 11 | week = 7 * day 12 | ) 13 | 14 | var unitMap = map[string]int64{ 15 | "ns": int64(time.Nanosecond), 16 | "us": int64(time.Microsecond), 17 | "µs": int64(time.Microsecond), // U+00B5 = micro symbol 18 | "μs": int64(time.Microsecond), // U+03BC = Greek letter mu 19 | "ms": int64(time.Millisecond), 20 | "s": int64(time.Second), 21 | "m": int64(time.Minute), 22 | "h": int64(time.Hour), 23 | "d": int64(day), 24 | "w": int64(week), 25 | } 26 | 27 | var errLeadingInt = errors.New("time: bad [0-9]*") 28 | 29 | // leadingInt consumes the leading [0-9]* from s. 30 | func leadingInt(s string) (x int64, rem string, err error) { 31 | i := 0 32 | for ; i < len(s); i++ { 33 | c := s[i] 34 | if c < '0' || c > '9' { 35 | break 36 | } 37 | if x > (1<<63-1)/10 { 38 | // overflow 39 | return 0, "", errLeadingInt 40 | } 41 | x = x*10 + int64(c) - '0' 42 | if x < 0 { 43 | // overflow 44 | return 0, "", errLeadingInt 45 | } 46 | } 47 | return x, s[i:], nil 48 | } 49 | 50 | // leadingFraction consumes the leading [0-9]* from s. 51 | // It is used only for fractions, so does not return an error on overflow, 52 | // it just stops accumulating precision. 53 | func leadingFraction(s string) (x int64, scale float64, rem string) { 54 | i := 0 55 | scale = 1 56 | overflow := false 57 | for ; i < len(s); i++ { 58 | c := s[i] 59 | if c < '0' || c > '9' { 60 | break 61 | } 62 | if overflow { 63 | continue 64 | } 65 | if x > (1<<63-1)/10 { 66 | // It's possible for overflow to give a positive number, so take care. 67 | overflow = true 68 | continue 69 | } 70 | y := x*10 + int64(c) - '0' 71 | if y < 0 { 72 | overflow = true 73 | continue 74 | } 75 | x = y 76 | scale *= 10 77 | } 78 | return x, scale, s[i:] 79 | } 80 | 81 | // ParseDuration parses a duration string. 82 | // A duration string is a possibly signed sequence of 83 | // decimal numbers, each with optional fraction and a unit suffix, 84 | // such as "300ms", "-1.5h" or "2h45m". 85 | // Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". 86 | func ParseDuration(s string) (time.Duration, error) { 87 | // [-+]?([0-9]*(\.[0-9]*)?[a-z]+)+ 88 | orig := s 89 | var d int64 90 | neg := false 91 | 92 | // Consume [-+]? 93 | if s != "" { 94 | c := s[0] 95 | if c == '-' || c == '+' { 96 | neg = c == '-' 97 | s = s[1:] 98 | } 99 | } 100 | // Special case: if all that is left is "0", this is zero. 101 | if s == "0" { 102 | return 0, nil 103 | } 104 | if s == "" { 105 | return 0, errors.New("time: invalid duration " + orig) 106 | } 107 | for s != "" { 108 | var ( 109 | v, f int64 // integers before, after decimal point 110 | scale float64 = 1 // value = v + f/scale 111 | ) 112 | 113 | var err error 114 | 115 | // The next character must be [0-9.] 116 | if !(s[0] == '.' || '0' <= s[0] && s[0] <= '9') { 117 | return 0, errors.New("time: invalid duration " + orig) 118 | } 119 | // Consume [0-9]* 120 | pl := len(s) 121 | v, s, err = leadingInt(s) 122 | if err != nil { 123 | return 0, errors.New("time: invalid duration " + orig) 124 | } 125 | pre := pl != len(s) // whether we consumed anything before a period 126 | 127 | // Consume (\.[0-9]*)? 128 | post := false 129 | if s != "" && s[0] == '.' { 130 | s = s[1:] 131 | pl := len(s) 132 | f, scale, s = leadingFraction(s) 133 | post = pl != len(s) 134 | } 135 | if !pre && !post { 136 | // no digits (e.g. ".s" or "-.s") 137 | return 0, errors.New("time: invalid duration " + orig) 138 | } 139 | 140 | // Consume unit. 141 | i := 0 142 | for ; i < len(s); i++ { 143 | c := s[i] 144 | if c == '.' || '0' <= c && c <= '9' { 145 | break 146 | } 147 | } 148 | if i == 0 { 149 | return 0, errors.New("time: missing unit in duration " + orig) 150 | } 151 | u := s[:i] 152 | s = s[i:] 153 | unit, ok := unitMap[u] 154 | if !ok { 155 | return 0, errors.New("time: unknown unit " + u + " in duration " + orig) 156 | } 157 | if v > (1<<63-1)/unit { 158 | // overflow 159 | return 0, errors.New("time: invalid duration " + orig) 160 | } 161 | v *= unit 162 | if f > 0 { 163 | // float64 is needed to be nanosecond accurate for fractions of hours. 164 | // v >= 0 && (f*unit/scale) <= 3.6e+12 (ns/h, h is the largest unit) 165 | v += int64(float64(f) * (float64(unit) / scale)) 166 | if v < 0 { 167 | // overflow 168 | return 0, errors.New("time: invalid duration " + orig) 169 | } 170 | } 171 | d += v 172 | if d < 0 { 173 | // overflow 174 | return 0, errors.New("time: invalid duration " + orig) 175 | } 176 | } 177 | 178 | if neg { 179 | d = -d 180 | } 181 | return time.Duration(d), nil 182 | } 183 | 184 | func durationToString(dur time.Duration) string { 185 | result := "" 186 | weeks := dur / week 187 | dur = dur % week 188 | if weeks > 0 { 189 | result = fmt.Sprintf("%dw", weeks) 190 | } 191 | days := dur / day 192 | dur = dur % day 193 | if days > 0 || (weeks > 0 && dur > 0) { 194 | result = fmt.Sprintf("%v%dd", result, days) 195 | } 196 | if dur > 0 { 197 | result = fmt.Sprintf("%v%v", result, dur) 198 | } 199 | return result 200 | } 201 | -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env sh 2 | echo "mode: count" > profile.cov 3 | TP=$(go list -f '{{if len .GoFiles}}{{.ImportPath}}{{end}}' ./... | grep -v "/vendor/" | grep -v "zenodb/zeno" | grep -v "zenodb/zeno-cli") 4 | CP=$(echo $TP | tr ' ', ',') 5 | set -x 6 | for pkg in $TP; do \ 7 | GO111MODULE=on go test -v -covermode=atomic -coverprofile=profile_tmp.cov -coverpkg "$CP" $pkg || exit 1; \ 8 | tail -n +2 profile_tmp.cov >> profile.cov; \ 9 | done 10 | exit $? 11 | -------------------------------------------------------------------------------- /testsupport/expectedresult.go: -------------------------------------------------------------------------------- 1 | package testsupport 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/getlantern/golog" 8 | "github.com/getlantern/zenodb/common" 9 | "github.com/getlantern/zenodb/core" 10 | "github.com/getlantern/zenodb/encoding" 11 | . "github.com/getlantern/zenodb/expr" 12 | 13 | "testing" 14 | 15 | "github.com/stretchr/testify/assert" 16 | ) 17 | 18 | var ( 19 | log = golog.LoggerFor("expectedresult") 20 | ) 21 | 22 | type ExpectedResult []ExpectedRow 23 | 24 | func (er ExpectedResult) Assert(t *testing.T, label string, md *common.QueryMetaData, rows []*core.FlatRow) bool { 25 | t.Helper() 26 | if !assert.Len(t, rows, len(er), label+" | Wrong number of rows") { 27 | return false 28 | } 29 | ok := true 30 | for i, erow := range er { 31 | row := rows[i] 32 | if !erow.Assert(t, label, md.FieldNames, row, i+1) { 33 | ok = false 34 | } 35 | } 36 | return ok 37 | } 38 | 39 | func (er ExpectedResult) TryAssert(md *common.QueryMetaData, rows []*core.FlatRow) bool { 40 | if len(rows) != len(er) { 41 | return false 42 | } 43 | for i, erow := range er { 44 | row := rows[i] 45 | if !erow.TryAssert(md.FieldNames, row, i+1) { 46 | return false 47 | } 48 | } 49 | return true 50 | } 51 | 52 | type ExpectedRow struct { 53 | TS time.Time 54 | Dims map[string]interface{} 55 | Vals map[string]float64 56 | } 57 | 58 | func (erow ExpectedRow) Assert(t *testing.T, label string, fieldNames []string, row *core.FlatRow, idx int) bool { 59 | t.Helper() 60 | if !assert.Equal(t, erow.TS.In(time.UTC), encoding.TimeFromInt(row.TS).In(time.UTC), label+" | Row %d - wrong timestamp", idx) { 61 | return false 62 | } 63 | 64 | dims := row.Key.AsMap() 65 | if !assert.Len(t, dims, len(erow.Dims), label+" | Row %d - wrong number of dimensions in result", idx) { 66 | return false 67 | } 68 | for k, v := range erow.Dims { 69 | if !assert.Equal(t, v, dims[k], label+" | Row %d - mismatch on dimension %v", idx, k) { 70 | return false 71 | } 72 | } 73 | 74 | if !assert.Len(t, row.Values, len(erow.Vals), label+" | Row %d - wrong number of values in result. %v", idx, erow.FieldDiff(fieldNames)) { 75 | return false 76 | } 77 | 78 | ok := true 79 | for i, v := range row.Values { 80 | fieldName := fieldNames[i] 81 | if !AssertFloatWithin(t, 0.01, erow.Vals[fieldName], v, fmt.Sprintf(label+" | Row %d - mismatch on field %v", idx, fieldName)) { 82 | ok = false 83 | } 84 | } 85 | return ok 86 | } 87 | 88 | func (erow ExpectedRow) TryAssert(fieldNames []string, row *core.FlatRow, idx int) bool { 89 | if !erow.TS.In(time.UTC).Equal(encoding.TimeFromInt(row.TS).In(time.UTC)) { 90 | return false 91 | } 92 | 93 | dims := row.Key.AsMap() 94 | if len(dims) != len(erow.Dims) { 95 | return false 96 | } 97 | for k, v := range erow.Dims { 98 | if v != dims[k] { 99 | return false 100 | } 101 | } 102 | 103 | if len(row.Values) != len(erow.Vals) { 104 | return false 105 | } 106 | 107 | for i, v := range row.Values { 108 | fieldName := fieldNames[i] 109 | if !FuzzyEquals(0.01, erow.Vals[fieldName], v) { 110 | return false 111 | } 112 | } 113 | 114 | return true 115 | } 116 | 117 | func (erow ExpectedRow) FieldDiff(fieldNames []string) string { 118 | diff := "" 119 | first := true 120 | for expected := range erow.Vals { 121 | found := false 122 | for _, name := range fieldNames { 123 | if name == expected { 124 | found = true 125 | break 126 | } 127 | } 128 | if !found { 129 | if !first { 130 | diff += " " 131 | } 132 | first = false 133 | diff += expected + "(m)" 134 | } 135 | } 136 | for _, name := range fieldNames { 137 | _, found := erow.Vals[name] 138 | if !found { 139 | if !first { 140 | diff += " " 141 | } 142 | first = false 143 | diff += name + "(e)" 144 | } 145 | } 146 | return diff 147 | } 148 | -------------------------------------------------------------------------------- /testsupport/testwriter.go: -------------------------------------------------------------------------------- 1 | package testsupport 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/getlantern/golog" 8 | ) 9 | 10 | type testWriter struct { 11 | t *testing.T 12 | start time.Time 13 | } 14 | 15 | func (tw *testWriter) Write(b []byte) (int, error) { 16 | tw.t.Logf("(+%dms) %v", time.Now().Sub(tw.start).Nanoseconds()/1000000, string(b)) 17 | return len(b), nil 18 | } 19 | 20 | // RedirectLogsToTest redirects golog log statements to t.Log. Call the returned cancel function 21 | // to start sending logs back to stdout and stderr. 22 | func RedirectLogsToTest(t *testing.T) (cancel func()) { 23 | tw := &testWriter{t, time.Now()} 24 | golog.SetOutputs(tw, tw) 25 | return func() { 26 | golog.ResetOutputs() 27 | time.Sleep(1 * time.Second) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /web/auth.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "net/http" 8 | "time" 9 | ) 10 | 11 | const ( 12 | authcookie = "authcookie" 13 | xsrftoken = "xsrftoken" 14 | authheader = "X-Zeno-Auth-Token" 15 | 16 | randomKeyLength = 32 17 | ) 18 | 19 | var ( 20 | sessionTimeout = 1 * time.Hour 21 | ) 22 | 23 | type AuthData struct { 24 | AccessToken string 25 | Expiration time.Time 26 | } 27 | 28 | func (h *handler) authenticate(resp http.ResponseWriter, req *http.Request) bool { 29 | if h.Opts.OAuthClientID == "" || h.Opts.OAuthClientSecret == "" { 30 | log.Debug("OAuth not configured, not authenticating!") 31 | return true 32 | } 33 | 34 | // First check for static auth token 35 | if h.Opts.Password != "" { 36 | password := req.Header.Get(authheader) 37 | if password != "" { 38 | result := password == h.Opts.Password 39 | return result 40 | } 41 | } 42 | 43 | // Then check for GitHub credentials 44 | cookie, err := req.Cookie(authcookie) 45 | if err == nil { 46 | ad := &AuthData{} 47 | err = h.sc.Decode(authcookie, cookie.Value, ad) 48 | if err == nil { 49 | if ad.Expiration.Before(time.Now()) { 50 | return true 51 | } 52 | inOrg, err := h.userInOrg(ad.AccessToken) 53 | if err != nil { 54 | log.Errorf("Unable to check if user is in org: %v", err) 55 | } else if inOrg { 56 | ad.Expiration = time.Now().Add(sessionTimeout) 57 | return true 58 | } 59 | } 60 | } 61 | 62 | // User not logged in, request authorization from OAuth provider 63 | h.requestAuthorization(resp, req) 64 | 65 | return false 66 | } 67 | 68 | func (h *handler) requestAuthorization(resp http.ResponseWriter, req *http.Request) { 69 | xsrfExpiration := time.Now().Add(1 * time.Minute) 70 | state, err := h.sc.Encode(xsrftoken, xsrfExpiration) 71 | if err != nil { 72 | log.Errorf("Unable to encode xsrf token: %v", err) 73 | // TODO: figure out how to handle this 74 | return 75 | } 76 | 77 | u, err := buildURL("https://github.com/login/oauth/authorize", map[string]string{ 78 | "client_id": h.OAuthClientID, 79 | "state": state, 80 | "scope": "read:org", 81 | }) 82 | if err != nil { 83 | h.db.Panic(err) 84 | } 85 | 86 | log.Debugf("Redirecting to: %v", u.String()) 87 | 88 | resp.Header().Set("Location", u.String()) 89 | resp.WriteHeader(http.StatusTemporaryRedirect) 90 | } 91 | 92 | func (h *handler) oauthCode(resp http.ResponseWriter, req *http.Request) { 93 | code := req.URL.Query().Get("code") 94 | state := req.URL.Query().Get("state") 95 | var xsrfExpiration time.Time 96 | err := h.sc.Decode(xsrftoken, state, &xsrfExpiration) 97 | if err != nil { 98 | log.Errorf("Unable to decode xsrf token, may indicate attempted attack, re-authorizing: %v", err) 99 | h.requestAuthorization(resp, req) 100 | } 101 | if time.Now().After(xsrfExpiration) { 102 | log.Error("XSRF Token expired, re-authorizing") 103 | h.requestAuthorization(resp, req) 104 | return 105 | } 106 | 107 | u, err := buildURL("https://github.com/login/oauth/access_token", map[string]string{ 108 | "client_id": h.OAuthClientID, 109 | "client_secret": h.OAuthClientSecret, 110 | "code": code, 111 | "state": state, 112 | }) 113 | if err != nil { 114 | h.db.Panic(err) 115 | } 116 | 117 | post, _ := http.NewRequest(http.MethodPost, u.String(), nil) 118 | post.Header.Set("Accept", "application/json") 119 | tokenResp, err := h.client.Do(post) 120 | if err != nil { 121 | log.Errorf("Error requesting access token, re-authorizing: %v", err) 122 | h.requestAuthorization(resp, req) 123 | return 124 | } 125 | 126 | defer tokenResp.Body.Close() 127 | body, err := ioutil.ReadAll(tokenResp.Body) 128 | if err != nil { 129 | log.Errorf("Error reading access token, re-authorizing: %v", err) 130 | h.requestAuthorization(resp, req) 131 | return 132 | } 133 | 134 | tokenData := make(map[string]string) 135 | err = json.Unmarshal(body, &tokenData) 136 | if err != nil { 137 | log.Errorf("Error unmarshalling access token, re-authorizing: %v", err) 138 | h.requestAuthorization(resp, req) 139 | return 140 | } 141 | 142 | accessToken := tokenData["access_token"] 143 | inOrg, err := h.userInOrg(accessToken) 144 | if err != nil { 145 | log.Errorf("Unable to check if user is in org, re-authorizing: %v", err) 146 | } else if !inOrg { 147 | log.Errorf("User not in needed org") 148 | // TODO: figure out what to do 149 | return 150 | } 151 | 152 | ad := &AuthData{ 153 | AccessToken: accessToken, 154 | Expiration: time.Now().Add(sessionTimeout), 155 | } 156 | cookieData, err := h.sc.Encode(authcookie, ad) 157 | if err != nil { 158 | log.Errorf("Unable to encode authcookie: %v", err) 159 | // TODO: figure out what to handle here 160 | return 161 | } 162 | http.SetCookie(resp, &http.Cookie{ 163 | Path: "/", 164 | Secure: true, 165 | Name: authcookie, 166 | Value: cookieData, 167 | Expires: time.Now().Add(365 * 24 * time.Hour), 168 | }) 169 | 170 | log.Debug("User logged in!") 171 | resp.Header().Set("Location", "/") 172 | resp.WriteHeader(http.StatusTemporaryRedirect) 173 | } 174 | 175 | func (h *handler) userInOrg(accessToken string) (bool, error) { 176 | req, _ := http.NewRequest(http.MethodGet, "https://api.github.com/user/orgs", nil) 177 | req.Header.Set("Authorization", fmt.Sprintf("token %v", accessToken)) 178 | resp, err := h.client.Do(req) 179 | if err != nil { 180 | return false, fmt.Errorf("Unable to get user orgs from GitHub: %v", err) 181 | } 182 | defer resp.Body.Close() 183 | body, err := ioutil.ReadAll(resp.Body) 184 | if err != nil { 185 | return false, fmt.Errorf("Unable to read user orgs from GitHub: %v", err) 186 | } 187 | if resp.StatusCode > 299 { 188 | return false, fmt.Errorf("Got response status %d: %v", resp.StatusCode, string(body)) 189 | } 190 | orgs := make([]map[string]interface{}, 0) 191 | err = json.Unmarshal(body, &orgs) 192 | if err != nil { 193 | if err != nil { 194 | return false, fmt.Errorf("Unable to unmarshal user orgs from GitHub: %v", err) 195 | } 196 | } 197 | 198 | for _, org := range orgs { 199 | if org["login"] == h.GitHubOrg { 200 | return true, nil 201 | } 202 | } 203 | 204 | log.Debugf("User not in org %v", h.GitHubOrg) 205 | return false, nil 206 | } 207 | -------------------------------------------------------------------------------- /web/cache.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "time" 7 | 8 | "github.com/boltdb/bolt" 9 | "github.com/getlantern/errors" 10 | "github.com/getlantern/uuid" 11 | "github.com/getlantern/zenodb/encoding" 12 | ) 13 | 14 | const ( 15 | statusPending = 0 16 | statusSuccess = 1 17 | statusError = 2 18 | 19 | widthPermalink = 16 20 | idxStatus = 0 21 | idxPermalink = 1 22 | idxTime = idxPermalink + widthPermalink 23 | idxData = idxTime + encoding.WidthTime 24 | ) 25 | 26 | var ( 27 | cacheBucket = []byte("cache") 28 | permalinkBucket = []byte("permalink") 29 | ) 30 | 31 | type cache struct { 32 | db *bolt.DB 33 | ttl time.Duration 34 | size int 35 | } 36 | 37 | type cacheEntry []byte 38 | 39 | func (c *cache) newCacheEntry() cacheEntry { 40 | ce := make([]byte, widthPermalink+encoding.WidthTime+1) 41 | permalink := uuid.New() 42 | copy(ce[idxPermalink:], permalink[:]) 43 | encoding.EncodeTime(ce[idxTime:], time.Now().Add(c.ttl)) 44 | return ce 45 | } 46 | 47 | func (ce cacheEntry) permalink() string { 48 | return ce.uuid().String() 49 | } 50 | 51 | func (ce cacheEntry) permalinkBytes() []byte { 52 | return ce[idxPermalink : idxPermalink+widthPermalink] 53 | } 54 | 55 | func (ce cacheEntry) uuid() uuid.UUID { 56 | var arr [16]byte 57 | copy(arr[:], ce.permalinkBytes()) 58 | return uuid.UUID(arr) 59 | } 60 | 61 | func (ce cacheEntry) expires() time.Time { 62 | return encoding.TimeFromBytes(ce[idxTime:]) 63 | } 64 | 65 | func (ce cacheEntry) expired() bool { 66 | return ce.expires().Before(time.Now()) 67 | } 68 | 69 | func (ce cacheEntry) status() byte { 70 | return ce[idxStatus] 71 | } 72 | 73 | func (ce cacheEntry) pending() { 74 | ce[idxStatus] = statusPending 75 | } 76 | 77 | func (ce cacheEntry) data() []byte { 78 | return ce[idxData:] 79 | } 80 | 81 | func (ce cacheEntry) error() []byte { 82 | return ce.data() 83 | } 84 | 85 | func (ce cacheEntry) succeed(data []byte) cacheEntry { 86 | return ce.update(statusSuccess, data) 87 | } 88 | 89 | func (ce cacheEntry) fail(err error) cacheEntry { 90 | return ce.update(statusError, []byte(err.Error())) 91 | } 92 | 93 | func (ce cacheEntry) update(status byte, data []byte) cacheEntry { 94 | result := make(cacheEntry, 0, 1+widthPermalink+encoding.WidthTime+len(data)) 95 | result = append(result, status) // mark as succeeded 96 | result = append(result, ce[idxPermalink:idxTime]...) 97 | result = append(result, ce[idxTime:idxData]...) 98 | result = append(result, data...) 99 | return result 100 | } 101 | 102 | func (ce cacheEntry) copy() cacheEntry { 103 | if ce == nil { 104 | return nil 105 | } 106 | result := make(cacheEntry, len(ce)) 107 | copy(result, ce) 108 | return result 109 | } 110 | 111 | func newCache(cacheDir string, ttl time.Duration) (*cache, error) { 112 | err := os.MkdirAll(cacheDir, 0700) 113 | if err != nil { 114 | return nil, errors.New("Unable to create cacheDir at %v: %v", cacheDir, err) 115 | } 116 | return newCacheFromFile(filepath.Join(cacheDir, "webcache.db"), ttl, nil) 117 | } 118 | 119 | func newCacheFromFile(cacheFile string, ttl time.Duration, options *bolt.Options) (*cache, error) { 120 | db, err := bolt.Open(cacheFile, 0600, options) 121 | if err != nil { 122 | return nil, errors.New("Unable to open cache database: %v", err) 123 | } 124 | 125 | if options == nil || !options.ReadOnly { 126 | err = db.Update(func(tx *bolt.Tx) error { 127 | _, bucketErr := tx.CreateBucketIfNotExists(cacheBucket) 128 | if bucketErr != nil { 129 | return bucketErr 130 | } 131 | _, bucketErr = tx.CreateBucketIfNotExists(permalinkBucket) 132 | return bucketErr 133 | }) 134 | if err != nil { 135 | return nil, errors.New("Unable to initialize cache database: %v", err) 136 | } 137 | } 138 | 139 | return &cache{ 140 | db: db, 141 | ttl: ttl, 142 | }, nil 143 | } 144 | 145 | func (c *cache) getOrBegin(sql string) (ce cacheEntry, created bool, err error) { 146 | key := []byte(sql) 147 | err = c.db.Update(func(tx *bolt.Tx) error { 148 | cb := tx.Bucket(cacheBucket) 149 | pb := tx.Bucket(permalinkBucket) 150 | ce = cacheEntry(cb.Get(key)).copy() 151 | expired := ce != nil && ce.expired() 152 | if ce == nil || expired { 153 | created = true 154 | if expired { 155 | ce = nil 156 | err = cb.Delete(key) 157 | if err != nil { 158 | return err 159 | } 160 | } 161 | ce = c.newCacheEntry() 162 | cb.Put(key, ce) 163 | pb.Put(ce.permalinkBytes(), ce) 164 | return nil 165 | } 166 | return nil 167 | }) 168 | return 169 | } 170 | 171 | func (c *cache) begin(sql string) (ce cacheEntry, err error) { 172 | key := []byte(sql) 173 | err = c.db.Update(func(tx *bolt.Tx) error { 174 | cb := tx.Bucket(cacheBucket) 175 | pb := tx.Bucket(permalinkBucket) 176 | ce = c.newCacheEntry() 177 | cb.Put(key, ce) 178 | pb.Put(ce.permalinkBytes(), ce) 179 | return nil 180 | }) 181 | return 182 | } 183 | 184 | func (c *cache) getByPermalink(permalink string) (ce cacheEntry, err error) { 185 | key := uuid.MustParse(permalink) 186 | err = c.db.View(func(tx *bolt.Tx) error { 187 | pb := tx.Bucket(permalinkBucket) 188 | ce = cacheEntry(pb.Get(key[:])).copy() 189 | return nil 190 | }) 191 | return 192 | } 193 | 194 | func (c *cache) put(sql string, ce cacheEntry) error { 195 | key := []byte(sql) 196 | 197 | return c.db.Update(func(tx *bolt.Tx) error { 198 | cb := tx.Bucket(cacheBucket) 199 | pb := tx.Bucket(permalinkBucket) 200 | pb.Put(ce.permalinkBytes(), ce) 201 | if ce.expired() { 202 | cb.Delete(key) 203 | return errors.New("Finished entry after expiration") 204 | } 205 | cb.Put(key, ce) 206 | return nil 207 | }) 208 | } 209 | 210 | func (c *cache) Close() error { 211 | return c.db.Close() 212 | } 213 | -------------------------------------------------------------------------------- /web/cache_test.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "testing" 7 | "time" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestCache(t *testing.T) { 13 | ttl := 250 * time.Millisecond 14 | 15 | cacheDir, err := ioutil.TempDir("", "") 16 | if !assert.NoError(t, err) { 17 | return 18 | } 19 | defer os.RemoveAll(cacheDir) 20 | 21 | cache, err := newCache(cacheDir, 250*time.Millisecond) 22 | if !assert.NoError(t, err) { 23 | return 24 | } 25 | defer cache.Close() 26 | 27 | ce, created, err := cache.getOrBegin("a") 28 | if !assert.NoError(t, err) { 29 | return 30 | } 31 | assert.Empty(t, ce.data()) 32 | assert.True(t, created) 33 | assert.EqualValues(t, statusPending, ce.status()) 34 | assert.NotEmpty(t, ce.permalink()) 35 | 36 | permalink := ce.permalink() 37 | ce, err = cache.getByPermalink(permalink) 38 | if !assert.NoError(t, err) { 39 | return 40 | } 41 | assert.Empty(t, ce.data()) 42 | assert.EqualValues(t, statusPending, ce.status()) 43 | assert.NotEmpty(t, ce.permalink()) 44 | 45 | ce, created, err = cache.getOrBegin("a") 46 | if !assert.NoError(t, err) { 47 | return 48 | } 49 | assert.False(t, created) 50 | assert.Empty(t, ce.data()) 51 | assert.EqualValues(t, statusPending, ce.status()) 52 | assert.NotEmpty(t, ce.permalink()) 53 | 54 | ce = ce.succeed([]byte("1")) 55 | err = cache.put("a", ce) 56 | if !assert.NoError(t, err) { 57 | return 58 | } 59 | 60 | ce, created, err = cache.getOrBegin("a") 61 | if !assert.NoError(t, err) { 62 | return 63 | } 64 | assert.False(t, created) 65 | assert.EqualValues(t, []byte("1"), ce.data()) 66 | assert.EqualValues(t, statusSuccess, ce.status()) 67 | 68 | time.Sleep(ttl) 69 | ce, created, err = cache.getOrBegin("a") 70 | if !assert.NoError(t, err) { 71 | return 72 | } 73 | assert.True(t, created) 74 | assert.Empty(t, ce.data()) 75 | assert.EqualValues(t, statusPending, ce.status()) 76 | assert.NotEqual(t, string(permalink), string(ce.permalink())) 77 | 78 | ce, err = cache.getByPermalink(permalink) 79 | if !assert.NoError(t, err) { 80 | return 81 | } 82 | assert.EqualValues(t, []byte("1"), ce.data()) 83 | assert.EqualValues(t, statusSuccess, ce.status()) 84 | } 85 | -------------------------------------------------------------------------------- /web/cookie_test.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "crypto/rand" 5 | "github.com/gorilla/securecookie" 6 | "github.com/stretchr/testify/assert" 7 | "testing" 8 | ) 9 | 10 | func TestPendingAuthCookie(t *testing.T) { 11 | hashKey := make([]byte, 64) 12 | blockKey := make([]byte, 32) 13 | 14 | log.Debugf("Generating random hash key") 15 | _, err := rand.Read(hashKey) 16 | if !assert.NoError(t, err) { 17 | return 18 | } 19 | 20 | log.Debugf("Generating random block key") 21 | _, err = rand.Read(blockKey) 22 | if !assert.NoError(t, err) { 23 | return 24 | } 25 | 26 | sc := securecookie.New(hashKey, blockKey) 27 | 28 | randomKey := make([]byte, randomKeyLength) 29 | rand.Read(randomKey) 30 | 31 | encoded, err := sc.Encode(authcookie, randomKey) 32 | if !assert.NoError(t, err) { 33 | return 34 | } 35 | 36 | randomKeyRT := make([]byte, randomKeyLength) 37 | err = sc.Decode(authcookie, encoded, &randomKeyRT) 38 | if !assert.NoError(t, err) { 39 | return 40 | } 41 | 42 | assert.Equal(t, string(randomKey), string(randomKeyRT)) 43 | } 44 | -------------------------------------------------------------------------------- /web/handler.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "crypto/rand" 5 | "errors" 6 | "fmt" 7 | "net/http" 8 | "net/url" 9 | "time" 10 | 11 | "github.com/getlantern/golog" 12 | "github.com/getlantern/zenodb" 13 | "github.com/gorilla/mux" 14 | "github.com/gorilla/securecookie" 15 | ) 16 | 17 | var ( 18 | log = golog.LoggerFor("zenodb.web") 19 | ) 20 | 21 | type Opts struct { 22 | OAuthClientID string 23 | OAuthClientSecret string 24 | GitHubOrg string 25 | HashKey string 26 | BlockKey string 27 | AssetsDir string 28 | CacheDir string 29 | CacheTTL time.Duration 30 | Password string 31 | QueryTimeout time.Duration 32 | QueryConcurrencyLimit int 33 | MaxResponseBytes int 34 | } 35 | 36 | type handler struct { 37 | Opts 38 | db *zenodb.DB 39 | fs http.Handler 40 | sc *securecookie.SecureCookie 41 | client *http.Client 42 | cache *cache 43 | queries chan *query 44 | coalescedQueries chan []*query 45 | } 46 | 47 | func Configure(db *zenodb.DB, router *mux.Router, opts *Opts) (func(), error) { 48 | if opts.OAuthClientID == "" || opts.OAuthClientSecret == "" || opts.GitHubOrg == "" { 49 | log.Errorf("WARNING - Missing OAuthClientID, OAuthClientSecret and/or GitHubOrg, web API will not authenticate!") 50 | } 51 | 52 | if opts.CacheDir == "" { 53 | return nil, errors.New("Unable to start web server, no CacheDir specified") 54 | } 55 | 56 | if opts.CacheTTL <= 0 { 57 | opts.CacheTTL = 2 * time.Hour 58 | } 59 | 60 | if opts.QueryTimeout <= 0 { 61 | opts.QueryTimeout = 30 * time.Minute 62 | } 63 | 64 | if opts.QueryConcurrencyLimit <= 0 { 65 | opts.QueryConcurrencyLimit = 2 66 | } 67 | 68 | if opts.MaxResponseBytes <= 0 { 69 | opts.MaxResponseBytes = 25 * 1024 * 1024 // 25 MB 70 | } 71 | 72 | hashKey := []byte(opts.HashKey) 73 | blockKey := []byte(opts.BlockKey) 74 | 75 | if len(hashKey) != 64 { 76 | log.Debugf("Generating random hash key") 77 | hashKey = make([]byte, 64) 78 | _, err := rand.Read(hashKey) 79 | if err != nil { 80 | return nil, fmt.Errorf("Unable to generate random hash key: %v", err) 81 | } 82 | } 83 | 84 | if len(blockKey) != 32 { 85 | log.Debugf("Generating random block key") 86 | blockKey = make([]byte, 32) 87 | _, err := rand.Read(blockKey) 88 | if err != nil { 89 | return nil, fmt.Errorf("Unable to generate random block key: %v", err) 90 | } 91 | } 92 | 93 | cache, err := newCache(opts.CacheDir, opts.CacheTTL) 94 | if err != nil { 95 | return nil, err 96 | } 97 | 98 | h := &handler{ 99 | Opts: *opts, 100 | db: db, 101 | sc: securecookie.New(hashKey, blockKey), 102 | client: &http.Client{}, 103 | cache: cache, 104 | queries: make(chan *query, opts.QueryConcurrencyLimit*1000), 105 | coalescedQueries: make(chan []*query, opts.QueryConcurrencyLimit), 106 | } 107 | 108 | log.Debugf("Starting %d goroutines to process queries", opts.QueryConcurrencyLimit) 109 | go h.coalesceQueries() 110 | for i := 0; i < opts.QueryConcurrencyLimit; i++ { 111 | go h.processQueries() 112 | } 113 | 114 | router.StrictSlash(true) 115 | router.HandleFunc("/insert/{stream}", h.insert) 116 | router.HandleFunc("/oauth/code", h.oauthCode) 117 | router.PathPrefix("/async").HandlerFunc(h.asyncQuery) 118 | router.PathPrefix("/immediate").HandlerFunc(h.immediateQuery) 119 | router.PathPrefix("/run").HandlerFunc(h.runQuery) 120 | router.PathPrefix("/cached/{permalink}").HandlerFunc(h.cachedQuery) 121 | router.PathPrefix("/favicon").Handler(http.NotFoundHandler()) 122 | router.PathPrefix("/report/{permalink}").HandlerFunc(h.index) 123 | router.PathPrefix("/metrics").HandlerFunc(h.metrics) 124 | router.PathPrefix("/").HandlerFunc(h.index) 125 | 126 | return func() { 127 | close(h.queries) 128 | if err := cache.Close(); err != nil { 129 | log.Errorf("Unable to close cache: %v", err) 130 | } 131 | }, nil 132 | } 133 | 134 | func buildURL(base string, params map[string]string) (*url.URL, error) { 135 | u, err := url.Parse(base) 136 | if err != nil { 137 | return nil, err 138 | } 139 | 140 | q := u.Query() 141 | for key, value := range params { 142 | q.Set(key, value) 143 | } 144 | u.RawQuery = q.Encode() 145 | 146 | return u, nil 147 | } 148 | -------------------------------------------------------------------------------- /web/insert.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "time" 9 | 10 | "github.com/gorilla/mux" 11 | ) 12 | 13 | const ( 14 | // ContentType is the key for the Content-Type header 15 | ContentType = "Content-Type" 16 | 17 | // ContentTypeJSON is the allowed content type 18 | ContentTypeJSON = "application/json" 19 | ) 20 | 21 | type Point struct { 22 | Ts time.Time `json:"ts,omitempty"` 23 | Dims map[string]interface{} `json:"dims,omitempty"` 24 | Vals map[string]interface{} `json:"vals,omitempty"` 25 | } 26 | 27 | func (h *handler) insert(resp http.ResponseWriter, req *http.Request) { 28 | if req.Method != http.MethodPost { 29 | resp.WriteHeader(http.StatusMethodNotAllowed) 30 | fmt.Fprintf(resp, "Method %v not allowed\n", req.Method) 31 | return 32 | } 33 | 34 | contentType := req.Header.Get(ContentType) 35 | if contentType != ContentTypeJSON { 36 | resp.WriteHeader(http.StatusUnsupportedMediaType) 37 | fmt.Fprintf(resp, "Media type %v unsupported\n", contentType) 38 | return 39 | } 40 | 41 | stream := mux.Vars(req)["stream"] 42 | dec := json.NewDecoder(req.Body) 43 | for { 44 | point := &Point{} 45 | err := dec.Decode(point) 46 | if err == io.EOF { 47 | // Done reading points 48 | resp.WriteHeader(http.StatusCreated) 49 | return 50 | } 51 | if err != nil { 52 | badRequest(resp, "Error decoding JSON: %v", err) 53 | return 54 | } 55 | if len(point.Dims) == 0 { 56 | badRequest(resp, "Need at least one dim") 57 | return 58 | } 59 | if len(point.Vals) == 0 { 60 | badRequest(resp, "Need at least one val") 61 | return 62 | } 63 | if point.Ts.IsZero() { 64 | point.Ts = time.Now() 65 | } 66 | 67 | insertErr := h.db.Insert(stream, point.Ts, point.Dims, point.Vals) 68 | if insertErr != nil { 69 | internalServerError(resp, "Error submitting point: %v", insertErr) 70 | } 71 | } 72 | } 73 | 74 | func badRequest(resp http.ResponseWriter, msg string, args ...interface{}) { 75 | resp.WriteHeader(http.StatusBadRequest) 76 | log.Errorf(msg, args...) 77 | fmt.Fprintf(resp, msg+"\n", args...) 78 | } 79 | 80 | func internalServerError(resp http.ResponseWriter, msg string, args ...interface{}) { 81 | resp.WriteHeader(http.StatusInternalServerError) 82 | log.Errorf(msg, args...) 83 | fmt.Fprintf(resp, msg+"\n", args...) 84 | } 85 | -------------------------------------------------------------------------------- /web/metrics.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | 7 | "github.com/getlantern/zenodb/metrics" 8 | ) 9 | 10 | func (h *handler) metrics(resp http.ResponseWriter, req *http.Request) { 11 | if !h.authenticate(resp, req) { 12 | resp.WriteHeader(http.StatusForbidden) 13 | return 14 | } 15 | 16 | json.NewEncoder(resp).Encode(metrics.GetStats()) 17 | } 18 | -------------------------------------------------------------------------------- /web/toolsupport.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "encoding/json" 7 | "fmt" 8 | "os" 9 | "sort" 10 | "time" 11 | 12 | "github.com/boltdb/bolt" 13 | 14 | "github.com/getlantern/errors" 15 | "github.com/getlantern/zenodb/encoding" 16 | ) 17 | 18 | // PermalinkInfo provides info about a stored permalink 19 | type PermalinkInfo struct { 20 | Permalink string 21 | SQL string 22 | Timestamp time.Time 23 | NumRows int 24 | } 25 | 26 | type permalinkInfo struct { 27 | Permalink string 28 | SQL string 29 | TS int64 30 | Rows []*json.RawMessage 31 | } 32 | 33 | type PermalinkInfoRow struct { 34 | TS int64 35 | } 36 | 37 | type byTimestamp []*PermalinkInfo 38 | 39 | func (a byTimestamp) Len() int { return len(a) } 40 | func (a byTimestamp) Swap(i, j int) { a[i], a[j] = a[j], a[i] } 41 | func (a byTimestamp) Less(i, j int) bool { return a[i].Timestamp.After(a[j].Timestamp) } 42 | 43 | // ListPermalinks returns a list of known permalinks and associated info 44 | func ListPermalinks(cacheFile string) ([]*PermalinkInfo, error) { 45 | c, err := newCacheFromFile(cacheFile, 24*time.Hour, &bolt.Options{ 46 | ReadOnly: true, 47 | Timeout: 10 * time.Second, 48 | }) 49 | if err != nil { 50 | return nil, errors.New("Unable to open cache at %v: %v", cacheFile, err) 51 | } 52 | permalinks, err := c.listPermalinks() 53 | if err != nil { 54 | return nil, err 55 | } 56 | 57 | sort.Sort(byTimestamp(permalinks)) 58 | return permalinks, nil 59 | } 60 | 61 | func (c *cache) listPermalinks() ([]*PermalinkInfo, error) { 62 | var permalinks []*PermalinkInfo 63 | err := c.db.View(func(tx *bolt.Tx) error { 64 | defer func() { 65 | fmt.Fprintln(os.Stderr) 66 | }() 67 | 68 | pb := tx.Bucket(permalinkBucket) 69 | stats := pb.Stats() 70 | numKeys := stats.KeyN 71 | fmt.Fprintf(os.Stderr, "Scanning %d permalinks\n", numKeys) 72 | reportEvery := numKeys / 100 73 | i := 0 74 | return pb.ForEach(func(key []byte, value []byte) error { 75 | ce := cacheEntry(value) 76 | if ce.status() != statusSuccess { 77 | // ignore unsuccessful cache entries 78 | return nil 79 | } 80 | pli := &permalinkInfo{} 81 | gzr, gzErr := gzip.NewReader(bytes.NewReader(ce.data())) 82 | if gzErr != nil { 83 | return errors.New("Unable to decompress cached data: %v", gzErr) 84 | } 85 | dec := json.NewDecoder(gzr) 86 | parseErr := dec.Decode(pli) 87 | if parseErr != nil { 88 | return errors.New("Unable to parse cached data: %v", parseErr) 89 | } 90 | permalinks = append(permalinks, &PermalinkInfo{ 91 | Permalink: pli.Permalink, 92 | SQL: pli.SQL, 93 | Timestamp: encoding.TimeFromInt(pli.TS * 1000000), 94 | NumRows: len(pli.Rows), 95 | }) 96 | if i > 0 && i%reportEvery == 0 { 97 | fmt.Fprint(os.Stderr, ".") 98 | } 99 | i++ 100 | return nil 101 | }) 102 | }) 103 | return permalinks, err 104 | } 105 | --------------------------------------------------------------------------------