├── .gitignore ├── .travis.yml ├── CONTRIBUTORS ├── LICENSE ├── MAINTAINERS ├── Makefile ├── README.md ├── src ├── config │ ├── bake_default_config.sh │ ├── config.go │ └── default.toml ├── datamodel │ ├── constants.go │ ├── info.go │ ├── protobuf │ │ ├── skizze.pb.go │ │ └── skizze.proto │ └── sketcher.go ├── manager │ ├── domain.go │ ├── info.go │ ├── manager.go │ ├── manager_test.go │ └── sketch.go ├── server │ ├── domain.go │ ├── global.go │ ├── server.go │ ├── server_test.go │ ├── sketch.go │ ├── sketch_test.go │ └── todo.go ├── sketches │ ├── bloom.go │ ├── bloom_test.go │ ├── cml.go │ ├── cml_test.go │ ├── hllpp.go │ ├── hllpp_test.go │ ├── proxy.go │ ├── topk.go │ └── topk_test.go ├── skizze-cli │ ├── bridge │ │ ├── domain.go │ │ ├── loop.go │ │ └── sketch.go │ └── main.go ├── skizze │ └── main.go ├── storage │ ├── aof.go │ ├── aof_test.go │ └── entry.go ├── testutils │ └── testutils.go └── utils │ ├── test_utils.go │ ├── utils.go │ └── utils_test.go └── vendor └── manifest /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | .DS_Store 26 | 27 | # gb 28 | bin 29 | pkg 30 | vendor/src 31 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - 1.5.3 4 | 5 | install: 6 | - go get github.com/constabulary/gb/... 7 | - go get github.com/alecthomas/gometalinter 8 | 9 | before_script: 10 | - make dist 11 | - gometalinter --install --update 12 | - export GOPATH=$HOME/gopath:$HOME/gopath/src/github.com/skizzehq/skizze/:$HOME/gopath/src/github.com/skizzehq/skizze/vendor 13 | 14 | script: 15 | - gometalinter ./src/* -D gocyclo -D gotype -D dupl --deadline=120s 16 | - make test 17 | -------------------------------------------------------------------------------- /CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | Martin Pinto-Bazurco 2 | Manuel Barkhau 3 | Keith Ballinger 4 | Arne Bahlo 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) Xamarin Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. -------------------------------------------------------------------------------- /MAINTAINERS: -------------------------------------------------------------------------------- 1 | Seif Lotfy 2 | Neil Jagdish Patel 3 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | GOPATH=$(CURDIR)/vendor:$(CURDIR) 2 | GOBIN=$(CURDIR)/vendor/bin 3 | GB=$(CURDIR)/vendor/bin/gb 4 | 5 | VERSION = 0.0.2 6 | 7 | all: bake-config skizze skizze-cli 8 | 9 | bake-config: 10 | @GOPATH=$(GOPATH) \ 11 | go generate src/config/config.go > src/config/config.go.tmp 12 | @mv src/config/config.go.tmp src/config/config.go 13 | 14 | skizze: 15 | @GOPATH=$(GOPATH) \ 16 | go build -a -v -ldflags "-w -X main.version=${VERSION}" \ 17 | -o ./bin/skizze ./src/skizze 18 | 19 | skizze-cli: 20 | @GOPATH=$(GOPATH) \ 21 | go build -a -v -ldflags "-w -X skizze-cli/bridge.version=${VERSION}" \ 22 | -o ./bin/skizze-cli ./src/skizze-cli 23 | 24 | build-dep: 25 | @GOBIN=$(GOBIN) go get github.com/constabulary/gb/... 26 | 27 | vendor: 28 | @$(GB) vendor restore 29 | 30 | test: 31 | @GOPATH=$(GOPATH) go test -race -cover ./src/... 32 | 33 | bench: 34 | @GOPATH=$(GOPATH) go test -bench=. ./src/... 35 | 36 | proto: 37 | @protoc --go_out=plugins=grpc:. ./src/datamodel/protobuf/skizze.proto 38 | 39 | dist: build-dep vendor all 40 | 41 | setup-lint: 42 | @GOPATH=$(GOPATH) && \ 43 | go get github.com/alecthomas/gometalinter 44 | @GOPATH=$(GOPATH) && \ 45 | gometalinter --install --update 46 | 47 | lint: 48 | @GOPATH=$(GOPATH) && \ 49 | gometalinter ./src/* -D gocyclo -D gotype -D dupl --deadline=120s 50 | 51 | clean: 52 | @rm ./bin/* 53 | 54 | .PHONY: all build-dep vendor test dist clean 55 | 56 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 |
3 | 4 | [![Build Status](https://travis-ci.org/skizzehq/skizze.svg?branch=master)](https://travis-ci.org/skizzehq/skizze) [![license](http://img.shields.io/badge/license-Apache-blue.svg)](https://raw.githubusercontent.com/skizzehq/skizze/master/LICENSE) 5 | 6 | Skizze ([ˈskɪt͡sə]: german for sketch) is a sketch data store to deal with all problems around counting and sketching using probabilistic data-structures. 7 | 8 | Unlike a Key-Value store, Skizze does not store values, but rather appends values to defined sketches, allowing one to solve frequency and cardinality queries in near O(1) time, with minimal memory footprint. 9 | 10 | Current status ==> Alpha (tagged v0.0.2) 11 | 12 | ## Motivation 13 | 14 | Statistical analysis and mining of huge multi-terabyte data sets is a common task nowadays, especially in areas like web analytics and Internet advertising. Analysis of such large data sets often requires powerful distributed data stores like Hadoop and heavy data processing with techniques like MapReduce. This approach often leads to heavyweight high-latency analytical processes and poor applicability to realtime use cases. On the other hand, when one is interested only in simple additive metrics like total page views or average price of conversion, it is obvious that raw data can be efficiently summarized, for example, on a daily basis or using simple in-stream counters. Computation of more advanced metrics like a number of unique visitor or most frequent items is more challenging and requires a lot of resources if implemented straightforwardly. 15 | 16 | Skizze is a (fire and forget) service that provides a probabilistic data structures (sketches) storage that allows estimation of these and many other metrics, with a trade off in precision of the estimations for the memory consumption. These data structures can be used both as temporary data accumulators in query processing procedures and, perhaps more important, as a compact – sometimes astonishingly compact – replacement of raw data in stream-based computing. 17 | 18 | ## Example use cases (queries) 19 | * How many distinct elements are in the data set (i.e. what is the cardinality of the data set)? 20 | * What are the most frequent elements (the terms “heavy hitters” and “top-k elements” are also used)? 21 | * What are the frequencies of the most frequent elements? 22 | * How many elements belong to the specified range (range query, in SQL it looks like `SELECT count(v) WHERE v >= c1 AND v < c2)`? 23 | * Does the data set contain a particular element (membership query)? 24 | 25 | ## How to build and run 26 | ``` 27 | make dist 28 | ./bin/skizze 29 | ``` 30 | 31 | ## Bindings 32 | 33 | Two bindings are currently available: 34 | 35 | * [Go](https://github.com/skizzehq/goskizze) 36 | * `go get github.com/skizzehq/goskizze/skizze` [Documentation](https://godoc.org/github.com/skizzehq/goskizze/skizze) 37 | 38 | * [Node.js](http://github.com/skizzehq/node-skizze) 39 | * `npm install --save skizze` [Documentation](https://github.com/skizzehq/node-skizze#documentation) 40 | 41 | 42 | ## Example usage: 43 | 44 | Skizze comes with a CLI to help test and explore the server. It can be run via 45 | 46 | ``` 47 | ./bin/skizze-cli 48 | ``` 49 | 50 | ### Commands 51 | **Create** a new Domain (Collection of Sketches): 52 | ```{r, engine='bash', count_lines} 53 | #CREATE DOM $name $estCardinality $topk 54 | CREATE DOM demostream 10000000 100 55 | ``` 56 | 57 | **Add** values to the domain: 58 | ```{r, engine='bash', count_lines} 59 | #ADD DOM $name $value1, $value2 .... 60 | ADD DOM demostream zod joker grod zod zod grod 61 | ``` 62 | 63 | **Get** the *cardinality* of the domain: 64 | ```{r, engine='bash', count_lines} 65 | # GET CARD $name 66 | GET CARD demostream 67 | 68 | # returns: 69 | # Cardinality: 9 70 | ``` 71 | 72 | **Get** the *rankings* of the domain: 73 | ```{r, engine='bash', count_lines} 74 | # GET RANK $name 75 | GET RANK demostream 76 | 77 | # returns: 78 | # Rank: 1 Value: zod Hits: 3 79 | # Rank: 2 Value: grod Hits: 2 80 | # Rank: 3 Value: joker Hits: 1 81 | ``` 82 | 83 | **Get** the *frequencies* of values in the domain: 84 | ```{r, engine='bash', count_lines} 85 | # GET FREQ $name $value1 $value2 ... 86 | GET FREQ demostream zod joker batman grod 87 | 88 | # returns 89 | # Value: zod Hits: 3 90 | # Value: joker Hits: 1 91 | # Value: batman Hits: 0 92 | # Value: grod Hits: 2 93 | ``` 94 | 95 | **Get** the *membership* of values in the domain: 96 | ```{r, engine='bash', count_lines} 97 | # GET MEMB $name $value1 $value2 ... 98 | GET MEMB demostream zod joker batman grod 99 | 100 | # returns 101 | # Value: zod Member: true 102 | # Value: joker Member: true 103 | # Value: batman Member: false 104 | # Value: grod Member: true 105 | ``` 106 | 107 | **List** all available sketches (created by domains): 108 | ```{r, engine='bash', count_lines} 109 | LIST 110 | 111 | # returns 112 | # Name: demostream Type: CARD 113 | # Name: demostream Type: FREQ 114 | # Name: demostream Type: MEMB 115 | # Name: demostream Type: RANK 116 | ``` 117 | 118 | **Create** a new sketch of type $type (CARD, MEMB, FREQ or RANK): 119 | ```{r, engine='bash', count_lines} 120 | # CREATE CARD $name 121 | CREATE CARD demosketch 122 | ``` 123 | 124 | **Add** values to the sketch of type $type (CARD, MEMB, FREQ or RANK): 125 | ```{r, engine='bash', count_lines} 126 | #ADD $type $name $value1, $value2 .... 127 | ADD CARD demostream zod joker grod zod zod grod 128 | ``` 129 | 130 | ### License 131 | Skizze is available under the Apache License, Version 2.0. 132 | 133 | 134 | ### Authors 135 | - [Seif Lotfy](https://twitter.com/seiflotfy) 136 | - [Neil Jagdish Patel](https://twitter.com/njpatel) 137 | -------------------------------------------------------------------------------- /src/config/bake_default_config.sh: -------------------------------------------------------------------------------- 1 | sed -n '1,/const defaultTomlConfig = `/p' config.go 2 | cat default.toml 3 | echo "" # make sure there's a \n before the ` 4 | sed -n '/^`$/,$p' config.go -------------------------------------------------------------------------------- /src/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/BurntSushi/toml" 7 | "github.com/njpatel/loggo" 8 | 9 | "utils" 10 | ) 11 | 12 | //go:generate bash bake_default_config.sh 13 | const defaultTomlConfig = ` 14 | # This is where top level info is stored for the counter manager we 15 | # could also use a boltDB in the DataDir but this would make it harder 16 | # to sync over replicas since not all replicas will hold the all the 17 | # counters. 18 | info_dir = "~/.skizze" 19 | 20 | # This is where the data is stored either as json or .count (pure bytes) 21 | data_dir = "~/.skizze/data" 22 | 23 | # The host interface for the server 24 | host = "localhost" 25 | 26 | # The port number for the server 27 | port = 3596 28 | 29 | # Treshold for saving a sketch to disk 30 | save_threshold_seconds = 1 31 | ` 32 | 33 | var logger = loggo.GetLogger("config") 34 | 35 | // Config stores all configuration parameters for Go 36 | type Config struct { 37 | InfoDir string `toml:"info_dir"` 38 | DataDir string `toml:"data_dir"` 39 | Host string `toml:"host"` 40 | Port int `toml:"port"` 41 | SaveThresholdSeconds uint `toml:"save_threshold_seconds"` 42 | } 43 | 44 | var config *Config 45 | // InfoDir initialized from config file 46 | var InfoDir string 47 | // DataDir initialized from config file 48 | var DataDir string 49 | // Host initialized from config file 50 | var Host string 51 | // Port initialized from config file 52 | var Port int 53 | // SaveThresholdSeconds initialized from config file 54 | var SaveThresholdSeconds uint 55 | 56 | // MaxKeySize for BoltDB keys in bytes 57 | const MaxKeySize int = 32768 58 | 59 | func parseConfigTOML() *Config { 60 | cfg := &Config{} 61 | if _, err := toml.Decode(defaultTomlConfig, &cfg); err != nil { 62 | utils.PanicOnError(err) 63 | } 64 | 65 | configPath := os.Getenv("SKIZZE_CONFIG") 66 | if configPath != "" { 67 | _, err := os.Open(configPath) 68 | if err != nil { 69 | logger.Warningf("Unable to find config file, using defaults") 70 | return cfg 71 | } 72 | if _, err := toml.DecodeFile(configPath, &cfg); err != nil { 73 | logger.Warningf("Error parsing config file, using defaults") 74 | } 75 | } 76 | // make paths absolute 77 | infodir, err := utils.FullPath(cfg.InfoDir) 78 | if err != nil {panic(err)} 79 | datadir, err := utils.FullPath(cfg.DataDir) 80 | if err != nil {panic(err)} 81 | cfg.InfoDir = infodir 82 | cfg.DataDir = datadir 83 | return cfg 84 | } 85 | 86 | // GetConfig returns a singleton Configuration 87 | func GetConfig() *Config { 88 | if config == nil { 89 | config = parseConfigTOML() 90 | 91 | InfoDir = config.InfoDir 92 | DataDir = config.DataDir 93 | Host = config.Host 94 | Port = config.Port 95 | SaveThresholdSeconds = config.SaveThresholdSeconds 96 | 97 | if err := os.MkdirAll(InfoDir, os.ModePerm); err != nil { 98 | panic(err) 99 | } 100 | if err := os.MkdirAll(DataDir, os.ModePerm); err != nil { 101 | panic(err) 102 | } 103 | } 104 | return config 105 | } 106 | 107 | // init initializes a singleton Configuration 108 | func init() { 109 | GetConfig() 110 | } 111 | 112 | // Reset ... 113 | func Reset() { 114 | GetConfig() 115 | } 116 | -------------------------------------------------------------------------------- /src/config/default.toml: -------------------------------------------------------------------------------- 1 | # This is where top level info is stored for the counter manager we 2 | # could also use a boltDB in the DataDir but this would make it harder 3 | # to sync over replicas since not all replicas will hold the all the 4 | # counters. 5 | info_dir = "~/.skizze" 6 | 7 | # This is where the data is stored either as json or .count (pure bytes) 8 | data_dir = "~/.skizze/data" 9 | 10 | # The host interface for the server 11 | host = "localhost" 12 | 13 | # The port number for the server 14 | port = 3596 15 | 16 | # Treshold for saving a sketch to disk 17 | save_threshold_seconds = 1 -------------------------------------------------------------------------------- /src/datamodel/constants.go: -------------------------------------------------------------------------------- 1 | package datamodel 2 | 3 | import pb "datamodel/protobuf" 4 | 5 | /* 6 | HLLPP => HyperLogLogPlusPlus 7 | CML => Count-min-log sketch 8 | TopK => Top-K 9 | Bloom => Bloom Filter 10 | */ 11 | const ( 12 | DOM = "dom" 13 | HLLPP = "card" 14 | CML = "freq" 15 | TopK = "rank" 16 | Bloom = "memb" 17 | ) 18 | 19 | /* 20 | MEMB = 1; 21 | FREQ = 2; 22 | RANK = 3; 23 | CARD = 4; 24 | */ 25 | var typeMap = map[pb.SketchType]string{ 26 | pb.SketchType_MEMB: Bloom, 27 | pb.SketchType_FREQ: CML, 28 | pb.SketchType_RANK: TopK, 29 | pb.SketchType_CARD: HLLPP, 30 | } 31 | 32 | // GetTypes ... 33 | func GetTypes() []string { 34 | return []string{HLLPP, CML, TopK, Bloom} 35 | } 36 | 37 | // GetTypeString ... 38 | func GetTypeString(typ pb.SketchType) string { 39 | return typeMap[typ] 40 | } 41 | 42 | // GetTypesPb ... 43 | func GetTypesPb() []pb.SketchType { 44 | return []pb.SketchType{ 45 | pb.SketchType_MEMB, 46 | pb.SketchType_FREQ, 47 | pb.SketchType_RANK, 48 | pb.SketchType_CARD, 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/datamodel/info.go: -------------------------------------------------------------------------------- 1 | package datamodel 2 | 3 | import ( 4 | pb "datamodel/protobuf" 5 | "fmt" 6 | "utils" 7 | ) 8 | 9 | // Info represents a info string describing the sketch 10 | type Info struct { 11 | *pb.Sketch 12 | locked bool 13 | id string 14 | } 15 | 16 | // ID return a unique ID based on the name and type 17 | func (info *Info) ID() string { 18 | if len(info.id) == 0 { 19 | info.id = fmt.Sprintf("%s.%s", info.GetName(), info.GetType()) 20 | } 21 | return info.id 22 | } 23 | 24 | // Locked returns the lock state of the sketch 25 | func (info *Info) Locked() bool { 26 | return info.locked 27 | } 28 | 29 | // Lock the Sketch 30 | func (info *Info) Lock() { 31 | info.locked = true 32 | } 33 | 34 | // Unlock the Sketch 35 | func (info *Info) Unlock() { 36 | info.locked = false 37 | } 38 | 39 | // Copy sketch 40 | func (info *Info) Copy() *Info { 41 | typ := info.GetType() 42 | return &Info{ 43 | Sketch: &pb.Sketch{ 44 | Properties: &pb.SketchProperties{ 45 | ErrorRate: utils.Float32p(info.Properties.GetErrorRate()), 46 | MaxUniqueItems: utils.Int64p(info.Properties.GetMaxUniqueItems()), 47 | Size: utils.Int64p(info.Properties.GetSize()), 48 | }, 49 | State: &pb.SketchState{ 50 | FillRate: utils.Float32p(info.State.GetFillRate()), 51 | LastSnapshot: utils.Int64p(info.State.GetLastSnapshot()), 52 | }, 53 | Name: utils.Stringp(info.GetName()), 54 | Type: &typ, 55 | }, 56 | } 57 | } 58 | 59 | // NewEmptyProperties returns an empty property struct 60 | func NewEmptyProperties() *pb.SketchProperties { 61 | return &pb.SketchProperties{ 62 | ErrorRate: utils.Float32p(0), 63 | MaxUniqueItems: utils.Int64p(0), 64 | Size: utils.Int64p(0), 65 | } 66 | } 67 | 68 | // NewEmptyState returns an empty state struct 69 | func NewEmptyState() *pb.SketchState { 70 | return &pb.SketchState{ 71 | FillRate: utils.Float32p(0), 72 | LastSnapshot: utils.Int64p(0), 73 | } 74 | } 75 | 76 | // NewEmptyInfo returns an empty info struct 77 | func NewEmptyInfo() *Info { 78 | sketch := &pb.Sketch{ 79 | Properties: NewEmptyProperties(), 80 | State: NewEmptyState(), 81 | } 82 | return &Info{Sketch: sketch, locked: false} 83 | } 84 | -------------------------------------------------------------------------------- /src/datamodel/protobuf/skizze.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. 2 | // source: src/datamodel/protobuf/skizze.proto 3 | // DO NOT EDIT! 4 | 5 | /* 6 | Package protobuf is a generated protocol buffer package. 7 | 8 | It is generated from these files: 9 | src/datamodel/protobuf/skizze.proto 10 | 11 | It has these top-level messages: 12 | Empty 13 | SketchProperties 14 | SketchState 15 | Domain 16 | Sketch 17 | Membership 18 | Frequency 19 | Rank 20 | CreateSnapshotRequest 21 | CreateSnapshotReply 22 | GetSnapshotRequest 23 | GetSnapshotReply 24 | ListRequest 25 | ListReply 26 | ListDomainsReply 27 | AddRequest 28 | AddReply 29 | GetRequest 30 | MembershipResult 31 | FrequencyResult 32 | CardinalityResult 33 | RankingsResult 34 | GetMembershipReply 35 | GetFrequencyReply 36 | GetCardinalityReply 37 | GetRankingsReply 38 | */ 39 | package protobuf 40 | 41 | import proto "github.com/golang/protobuf/proto" 42 | import fmt "fmt" 43 | import math "math" 44 | 45 | import ( 46 | context "golang.org/x/net/context" 47 | grpc "google.golang.org/grpc" 48 | ) 49 | 50 | // Reference imports to suppress errors if they are not otherwise used. 51 | var _ = proto.Marshal 52 | var _ = fmt.Errorf 53 | var _ = math.Inf 54 | 55 | // This is a compile-time assertion to ensure that this generated file 56 | // is compatible with the proto package it is being compiled against. 57 | const _ = proto.ProtoPackageIsVersion1 58 | 59 | // 60 | // Enums 61 | // 62 | type SketchType int32 63 | 64 | const ( 65 | SketchType_MEMB SketchType = 1 66 | SketchType_FREQ SketchType = 2 67 | SketchType_RANK SketchType = 3 68 | SketchType_CARD SketchType = 4 69 | ) 70 | 71 | var SketchType_name = map[int32]string{ 72 | 1: "MEMB", 73 | 2: "FREQ", 74 | 3: "RANK", 75 | 4: "CARD", 76 | } 77 | var SketchType_value = map[string]int32{ 78 | "MEMB": 1, 79 | "FREQ": 2, 80 | "RANK": 3, 81 | "CARD": 4, 82 | } 83 | 84 | func (x SketchType) Enum() *SketchType { 85 | p := new(SketchType) 86 | *p = x 87 | return p 88 | } 89 | func (x SketchType) String() string { 90 | return proto.EnumName(SketchType_name, int32(x)) 91 | } 92 | func (x *SketchType) UnmarshalJSON(data []byte) error { 93 | value, err := proto.UnmarshalJSONEnum(SketchType_value, data, "SketchType") 94 | if err != nil { 95 | return err 96 | } 97 | *x = SketchType(value) 98 | return nil 99 | } 100 | func (SketchType) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } 101 | 102 | type SnapshotStatus int32 103 | 104 | const ( 105 | SnapshotStatus_PENDING SnapshotStatus = 1 106 | SnapshotStatus_IN_PROGRESS SnapshotStatus = 2 107 | SnapshotStatus_SUCCESSFUL SnapshotStatus = 3 108 | SnapshotStatus_FAILED SnapshotStatus = 4 109 | ) 110 | 111 | var SnapshotStatus_name = map[int32]string{ 112 | 1: "PENDING", 113 | 2: "IN_PROGRESS", 114 | 3: "SUCCESSFUL", 115 | 4: "FAILED", 116 | } 117 | var SnapshotStatus_value = map[string]int32{ 118 | "PENDING": 1, 119 | "IN_PROGRESS": 2, 120 | "SUCCESSFUL": 3, 121 | "FAILED": 4, 122 | } 123 | 124 | func (x SnapshotStatus) Enum() *SnapshotStatus { 125 | p := new(SnapshotStatus) 126 | *p = x 127 | return p 128 | } 129 | func (x SnapshotStatus) String() string { 130 | return proto.EnumName(SnapshotStatus_name, int32(x)) 131 | } 132 | func (x *SnapshotStatus) UnmarshalJSON(data []byte) error { 133 | value, err := proto.UnmarshalJSONEnum(SnapshotStatus_value, data, "SnapshotStatus") 134 | if err != nil { 135 | return err 136 | } 137 | *x = SnapshotStatus(value) 138 | return nil 139 | } 140 | func (SnapshotStatus) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } 141 | 142 | // 143 | // Generic Structures 144 | // 145 | type Empty struct { 146 | XXX_unrecognized []byte `json:"-"` 147 | } 148 | 149 | func (m *Empty) Reset() { *m = Empty{} } 150 | func (m *Empty) String() string { return proto.CompactTextString(m) } 151 | func (*Empty) ProtoMessage() {} 152 | func (*Empty) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } 153 | 154 | type SketchProperties struct { 155 | MaxUniqueItems *int64 `protobuf:"varint,1,opt,name=maxUniqueItems" json:"maxUniqueItems,omitempty"` 156 | ErrorRate *float32 `protobuf:"fixed32,2,opt,name=errorRate" json:"errorRate,omitempty"` 157 | Size *int64 `protobuf:"varint,3,opt,name=size" json:"size,omitempty"` 158 | XXX_unrecognized []byte `json:"-"` 159 | } 160 | 161 | func (m *SketchProperties) Reset() { *m = SketchProperties{} } 162 | func (m *SketchProperties) String() string { return proto.CompactTextString(m) } 163 | func (*SketchProperties) ProtoMessage() {} 164 | func (*SketchProperties) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } 165 | 166 | func (m *SketchProperties) GetMaxUniqueItems() int64 { 167 | if m != nil && m.MaxUniqueItems != nil { 168 | return *m.MaxUniqueItems 169 | } 170 | return 0 171 | } 172 | 173 | func (m *SketchProperties) GetErrorRate() float32 { 174 | if m != nil && m.ErrorRate != nil { 175 | return *m.ErrorRate 176 | } 177 | return 0 178 | } 179 | 180 | func (m *SketchProperties) GetSize() int64 { 181 | if m != nil && m.Size != nil { 182 | return *m.Size 183 | } 184 | return 0 185 | } 186 | 187 | type SketchState struct { 188 | FillRate *float32 `protobuf:"fixed32,1,opt,name=fillRate" json:"fillRate,omitempty"` 189 | LastSnapshot *int64 `protobuf:"varint,2,opt,name=lastSnapshot" json:"lastSnapshot,omitempty"` 190 | XXX_unrecognized []byte `json:"-"` 191 | } 192 | 193 | func (m *SketchState) Reset() { *m = SketchState{} } 194 | func (m *SketchState) String() string { return proto.CompactTextString(m) } 195 | func (*SketchState) ProtoMessage() {} 196 | func (*SketchState) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } 197 | 198 | func (m *SketchState) GetFillRate() float32 { 199 | if m != nil && m.FillRate != nil { 200 | return *m.FillRate 201 | } 202 | return 0 203 | } 204 | 205 | func (m *SketchState) GetLastSnapshot() int64 { 206 | if m != nil && m.LastSnapshot != nil { 207 | return *m.LastSnapshot 208 | } 209 | return 0 210 | } 211 | 212 | // CreateDomain: name:required, propertiess:optional (array = nSketchTypes, order of types above) 213 | // DeleteDomain: name:required 214 | // GetDomain : name:required 215 | type Domain struct { 216 | Name *string `protobuf:"bytes,1,req,name=name" json:"name,omitempty"` 217 | Sketches []*Sketch `protobuf:"bytes,2,rep,name=sketches" json:"sketches,omitempty"` 218 | XXX_unrecognized []byte `json:"-"` 219 | } 220 | 221 | func (m *Domain) Reset() { *m = Domain{} } 222 | func (m *Domain) String() string { return proto.CompactTextString(m) } 223 | func (*Domain) ProtoMessage() {} 224 | func (*Domain) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} } 225 | 226 | func (m *Domain) GetName() string { 227 | if m != nil && m.Name != nil { 228 | return *m.Name 229 | } 230 | return "" 231 | } 232 | 233 | func (m *Domain) GetSketches() []*Sketch { 234 | if m != nil { 235 | return m.Sketches 236 | } 237 | return nil 238 | } 239 | 240 | // CreateSketch: name:required, type:required, properties:optional 241 | // DeleteSketch: name:required, type:required 242 | // GetSketch : name:required, type:required 243 | type Sketch struct { 244 | Name *string `protobuf:"bytes,1,req,name=name" json:"name,omitempty"` 245 | Type *SketchType `protobuf:"varint,2,req,name=type,enum=protobuf.SketchType" json:"type,omitempty"` 246 | Properties *SketchProperties `protobuf:"bytes,3,opt,name=properties" json:"properties,omitempty"` 247 | State *SketchState `protobuf:"bytes,4,opt,name=state" json:"state,omitempty"` 248 | XXX_unrecognized []byte `json:"-"` 249 | } 250 | 251 | func (m *Sketch) Reset() { *m = Sketch{} } 252 | func (m *Sketch) String() string { return proto.CompactTextString(m) } 253 | func (*Sketch) ProtoMessage() {} 254 | func (*Sketch) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} } 255 | 256 | func (m *Sketch) GetName() string { 257 | if m != nil && m.Name != nil { 258 | return *m.Name 259 | } 260 | return "" 261 | } 262 | 263 | func (m *Sketch) GetType() SketchType { 264 | if m != nil && m.Type != nil { 265 | return *m.Type 266 | } 267 | return SketchType_MEMB 268 | } 269 | 270 | func (m *Sketch) GetProperties() *SketchProperties { 271 | if m != nil { 272 | return m.Properties 273 | } 274 | return nil 275 | } 276 | 277 | func (m *Sketch) GetState() *SketchState { 278 | if m != nil { 279 | return m.State 280 | } 281 | return nil 282 | } 283 | 284 | type Membership struct { 285 | Value *string `protobuf:"bytes,1,req,name=value" json:"value,omitempty"` 286 | IsMember *bool `protobuf:"varint,2,req,name=isMember" json:"isMember,omitempty"` 287 | XXX_unrecognized []byte `json:"-"` 288 | } 289 | 290 | func (m *Membership) Reset() { *m = Membership{} } 291 | func (m *Membership) String() string { return proto.CompactTextString(m) } 292 | func (*Membership) ProtoMessage() {} 293 | func (*Membership) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} } 294 | 295 | func (m *Membership) GetValue() string { 296 | if m != nil && m.Value != nil { 297 | return *m.Value 298 | } 299 | return "" 300 | } 301 | 302 | func (m *Membership) GetIsMember() bool { 303 | if m != nil && m.IsMember != nil { 304 | return *m.IsMember 305 | } 306 | return false 307 | } 308 | 309 | type Frequency struct { 310 | Value *string `protobuf:"bytes,1,req,name=value" json:"value,omitempty"` 311 | Count *int64 `protobuf:"varint,2,req,name=count" json:"count,omitempty"` 312 | XXX_unrecognized []byte `json:"-"` 313 | } 314 | 315 | func (m *Frequency) Reset() { *m = Frequency{} } 316 | func (m *Frequency) String() string { return proto.CompactTextString(m) } 317 | func (*Frequency) ProtoMessage() {} 318 | func (*Frequency) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} } 319 | 320 | func (m *Frequency) GetValue() string { 321 | if m != nil && m.Value != nil { 322 | return *m.Value 323 | } 324 | return "" 325 | } 326 | 327 | func (m *Frequency) GetCount() int64 { 328 | if m != nil && m.Count != nil { 329 | return *m.Count 330 | } 331 | return 0 332 | } 333 | 334 | type Rank struct { 335 | Value *string `protobuf:"bytes,1,req,name=value" json:"value,omitempty"` 336 | Count *int64 `protobuf:"varint,2,req,name=count" json:"count,omitempty"` 337 | XXX_unrecognized []byte `json:"-"` 338 | } 339 | 340 | func (m *Rank) Reset() { *m = Rank{} } 341 | func (m *Rank) String() string { return proto.CompactTextString(m) } 342 | func (*Rank) ProtoMessage() {} 343 | func (*Rank) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} } 344 | 345 | func (m *Rank) GetValue() string { 346 | if m != nil && m.Value != nil { 347 | return *m.Value 348 | } 349 | return "" 350 | } 351 | 352 | func (m *Rank) GetCount() int64 { 353 | if m != nil && m.Count != nil { 354 | return *m.Count 355 | } 356 | return 0 357 | } 358 | 359 | // Right now empty but in the future can request specific snapshot location 360 | // (e.g. S3 or disk) and snapshot options 361 | type CreateSnapshotRequest struct { 362 | XXX_unrecognized []byte `json:"-"` 363 | } 364 | 365 | func (m *CreateSnapshotRequest) Reset() { *m = CreateSnapshotRequest{} } 366 | func (m *CreateSnapshotRequest) String() string { return proto.CompactTextString(m) } 367 | func (*CreateSnapshotRequest) ProtoMessage() {} 368 | func (*CreateSnapshotRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} } 369 | 370 | type CreateSnapshotReply struct { 371 | Status *SnapshotStatus `protobuf:"varint,1,req,name=status,enum=protobuf.SnapshotStatus" json:"status,omitempty"` 372 | StatusMessage *string `protobuf:"bytes,2,opt,name=statusMessage" json:"statusMessage,omitempty"` 373 | XXX_unrecognized []byte `json:"-"` 374 | } 375 | 376 | func (m *CreateSnapshotReply) Reset() { *m = CreateSnapshotReply{} } 377 | func (m *CreateSnapshotReply) String() string { return proto.CompactTextString(m) } 378 | func (*CreateSnapshotReply) ProtoMessage() {} 379 | func (*CreateSnapshotReply) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} } 380 | 381 | func (m *CreateSnapshotReply) GetStatus() SnapshotStatus { 382 | if m != nil && m.Status != nil { 383 | return *m.Status 384 | } 385 | return SnapshotStatus_PENDING 386 | } 387 | 388 | func (m *CreateSnapshotReply) GetStatusMessage() string { 389 | if m != nil && m.StatusMessage != nil { 390 | return *m.StatusMessage 391 | } 392 | return "" 393 | } 394 | 395 | // Empty for now, future can send in id of specific snapshot 396 | type GetSnapshotRequest struct { 397 | XXX_unrecognized []byte `json:"-"` 398 | } 399 | 400 | func (m *GetSnapshotRequest) Reset() { *m = GetSnapshotRequest{} } 401 | func (m *GetSnapshotRequest) String() string { return proto.CompactTextString(m) } 402 | func (*GetSnapshotRequest) ProtoMessage() {} 403 | func (*GetSnapshotRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} } 404 | 405 | type GetSnapshotReply struct { 406 | Status *SnapshotStatus `protobuf:"varint,1,req,name=status,enum=protobuf.SnapshotStatus" json:"status,omitempty"` 407 | StatusMessage *string `protobuf:"bytes,2,opt,name=statusMessage" json:"statusMessage,omitempty"` 408 | Timestamp *int64 `protobuf:"varint,3,opt,name=timestamp" json:"timestamp,omitempty"` 409 | XXX_unrecognized []byte `json:"-"` 410 | } 411 | 412 | func (m *GetSnapshotReply) Reset() { *m = GetSnapshotReply{} } 413 | func (m *GetSnapshotReply) String() string { return proto.CompactTextString(m) } 414 | func (*GetSnapshotReply) ProtoMessage() {} 415 | func (*GetSnapshotReply) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} } 416 | 417 | func (m *GetSnapshotReply) GetStatus() SnapshotStatus { 418 | if m != nil && m.Status != nil { 419 | return *m.Status 420 | } 421 | return SnapshotStatus_PENDING 422 | } 423 | 424 | func (m *GetSnapshotReply) GetStatusMessage() string { 425 | if m != nil && m.StatusMessage != nil { 426 | return *m.StatusMessage 427 | } 428 | return "" 429 | } 430 | 431 | func (m *GetSnapshotReply) GetTimestamp() int64 { 432 | if m != nil && m.Timestamp != nil { 433 | return *m.Timestamp 434 | } 435 | return 0 436 | } 437 | 438 | type ListRequest struct { 439 | Type *SketchType `protobuf:"varint,1,req,name=type,enum=protobuf.SketchType" json:"type,omitempty"` 440 | XXX_unrecognized []byte `json:"-"` 441 | } 442 | 443 | func (m *ListRequest) Reset() { *m = ListRequest{} } 444 | func (m *ListRequest) String() string { return proto.CompactTextString(m) } 445 | func (*ListRequest) ProtoMessage() {} 446 | func (*ListRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} } 447 | 448 | func (m *ListRequest) GetType() SketchType { 449 | if m != nil && m.Type != nil { 450 | return *m.Type 451 | } 452 | return SketchType_MEMB 453 | } 454 | 455 | type ListReply struct { 456 | Sketches []*Sketch `protobuf:"bytes,1,rep,name=sketches" json:"sketches,omitempty"` 457 | XXX_unrecognized []byte `json:"-"` 458 | } 459 | 460 | func (m *ListReply) Reset() { *m = ListReply{} } 461 | func (m *ListReply) String() string { return proto.CompactTextString(m) } 462 | func (*ListReply) ProtoMessage() {} 463 | func (*ListReply) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} } 464 | 465 | func (m *ListReply) GetSketches() []*Sketch { 466 | if m != nil { 467 | return m.Sketches 468 | } 469 | return nil 470 | } 471 | 472 | type ListDomainsReply struct { 473 | Names []string `protobuf:"bytes,1,rep,name=names" json:"names,omitempty"` 474 | XXX_unrecognized []byte `json:"-"` 475 | } 476 | 477 | func (m *ListDomainsReply) Reset() { *m = ListDomainsReply{} } 478 | func (m *ListDomainsReply) String() string { return proto.CompactTextString(m) } 479 | func (*ListDomainsReply) ProtoMessage() {} 480 | func (*ListDomainsReply) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} } 481 | 482 | func (m *ListDomainsReply) GetNames() []string { 483 | if m != nil { 484 | return m.Names 485 | } 486 | return nil 487 | } 488 | 489 | type AddRequest struct { 490 | Domain *Domain `protobuf:"bytes,1,opt,name=domain" json:"domain,omitempty"` 491 | Sketch *Sketch `protobuf:"bytes,2,opt,name=sketch" json:"sketch,omitempty"` 492 | Values []string `protobuf:"bytes,3,rep,name=values" json:"values,omitempty"` 493 | XXX_unrecognized []byte `json:"-"` 494 | } 495 | 496 | func (m *AddRequest) Reset() { *m = AddRequest{} } 497 | func (m *AddRequest) String() string { return proto.CompactTextString(m) } 498 | func (*AddRequest) ProtoMessage() {} 499 | func (*AddRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} } 500 | 501 | func (m *AddRequest) GetDomain() *Domain { 502 | if m != nil { 503 | return m.Domain 504 | } 505 | return nil 506 | } 507 | 508 | func (m *AddRequest) GetSketch() *Sketch { 509 | if m != nil { 510 | return m.Sketch 511 | } 512 | return nil 513 | } 514 | 515 | func (m *AddRequest) GetValues() []string { 516 | if m != nil { 517 | return m.Values 518 | } 519 | return nil 520 | } 521 | 522 | type AddReply struct { 523 | XXX_unrecognized []byte `json:"-"` 524 | } 525 | 526 | func (m *AddReply) Reset() { *m = AddReply{} } 527 | func (m *AddReply) String() string { return proto.CompactTextString(m) } 528 | func (*AddReply) ProtoMessage() {} 529 | func (*AddReply) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{16} } 530 | 531 | // All Sketches will be of one kind 532 | // All values will apply to all sketches (if card or ranking, values will be ignored) 533 | type GetRequest struct { 534 | Sketches []*Sketch `protobuf:"bytes,1,rep,name=sketches" json:"sketches,omitempty"` 535 | Values []string `protobuf:"bytes,2,rep,name=values" json:"values,omitempty"` 536 | XXX_unrecognized []byte `json:"-"` 537 | } 538 | 539 | func (m *GetRequest) Reset() { *m = GetRequest{} } 540 | func (m *GetRequest) String() string { return proto.CompactTextString(m) } 541 | func (*GetRequest) ProtoMessage() {} 542 | func (*GetRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} } 543 | 544 | func (m *GetRequest) GetSketches() []*Sketch { 545 | if m != nil { 546 | return m.Sketches 547 | } 548 | return nil 549 | } 550 | 551 | func (m *GetRequest) GetValues() []string { 552 | if m != nil { 553 | return m.Values 554 | } 555 | return nil 556 | } 557 | 558 | type MembershipResult struct { 559 | Memberships []*Membership `protobuf:"bytes,1,rep,name=memberships" json:"memberships,omitempty"` 560 | XXX_unrecognized []byte `json:"-"` 561 | } 562 | 563 | func (m *MembershipResult) Reset() { *m = MembershipResult{} } 564 | func (m *MembershipResult) String() string { return proto.CompactTextString(m) } 565 | func (*MembershipResult) ProtoMessage() {} 566 | func (*MembershipResult) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} } 567 | 568 | func (m *MembershipResult) GetMemberships() []*Membership { 569 | if m != nil { 570 | return m.Memberships 571 | } 572 | return nil 573 | } 574 | 575 | type FrequencyResult struct { 576 | Frequencies []*Frequency `protobuf:"bytes,2,rep,name=frequencies" json:"frequencies,omitempty"` 577 | XXX_unrecognized []byte `json:"-"` 578 | } 579 | 580 | func (m *FrequencyResult) Reset() { *m = FrequencyResult{} } 581 | func (m *FrequencyResult) String() string { return proto.CompactTextString(m) } 582 | func (*FrequencyResult) ProtoMessage() {} 583 | func (*FrequencyResult) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} } 584 | 585 | func (m *FrequencyResult) GetFrequencies() []*Frequency { 586 | if m != nil { 587 | return m.Frequencies 588 | } 589 | return nil 590 | } 591 | 592 | type CardinalityResult struct { 593 | Cardinality *int64 `protobuf:"varint,1,req,name=cardinality" json:"cardinality,omitempty"` 594 | XXX_unrecognized []byte `json:"-"` 595 | } 596 | 597 | func (m *CardinalityResult) Reset() { *m = CardinalityResult{} } 598 | func (m *CardinalityResult) String() string { return proto.CompactTextString(m) } 599 | func (*CardinalityResult) ProtoMessage() {} 600 | func (*CardinalityResult) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} } 601 | 602 | func (m *CardinalityResult) GetCardinality() int64 { 603 | if m != nil && m.Cardinality != nil { 604 | return *m.Cardinality 605 | } 606 | return 0 607 | } 608 | 609 | type RankingsResult struct { 610 | Rankings []*Rank `protobuf:"bytes,1,rep,name=rankings" json:"rankings,omitempty"` 611 | XXX_unrecognized []byte `json:"-"` 612 | } 613 | 614 | func (m *RankingsResult) Reset() { *m = RankingsResult{} } 615 | func (m *RankingsResult) String() string { return proto.CompactTextString(m) } 616 | func (*RankingsResult) ProtoMessage() {} 617 | func (*RankingsResult) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} } 618 | 619 | func (m *RankingsResult) GetRankings() []*Rank { 620 | if m != nil { 621 | return m.Rankings 622 | } 623 | return nil 624 | } 625 | 626 | type GetMembershipReply struct { 627 | Results []*MembershipResult `protobuf:"bytes,1,rep,name=results" json:"results,omitempty"` 628 | XXX_unrecognized []byte `json:"-"` 629 | } 630 | 631 | func (m *GetMembershipReply) Reset() { *m = GetMembershipReply{} } 632 | func (m *GetMembershipReply) String() string { return proto.CompactTextString(m) } 633 | func (*GetMembershipReply) ProtoMessage() {} 634 | func (*GetMembershipReply) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} } 635 | 636 | func (m *GetMembershipReply) GetResults() []*MembershipResult { 637 | if m != nil { 638 | return m.Results 639 | } 640 | return nil 641 | } 642 | 643 | type GetFrequencyReply struct { 644 | Results []*FrequencyResult `protobuf:"bytes,1,rep,name=results" json:"results,omitempty"` 645 | XXX_unrecognized []byte `json:"-"` 646 | } 647 | 648 | func (m *GetFrequencyReply) Reset() { *m = GetFrequencyReply{} } 649 | func (m *GetFrequencyReply) String() string { return proto.CompactTextString(m) } 650 | func (*GetFrequencyReply) ProtoMessage() {} 651 | func (*GetFrequencyReply) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23} } 652 | 653 | func (m *GetFrequencyReply) GetResults() []*FrequencyResult { 654 | if m != nil { 655 | return m.Results 656 | } 657 | return nil 658 | } 659 | 660 | type GetCardinalityReply struct { 661 | Results []*CardinalityResult `protobuf:"bytes,1,rep,name=results" json:"results,omitempty"` 662 | XXX_unrecognized []byte `json:"-"` 663 | } 664 | 665 | func (m *GetCardinalityReply) Reset() { *m = GetCardinalityReply{} } 666 | func (m *GetCardinalityReply) String() string { return proto.CompactTextString(m) } 667 | func (*GetCardinalityReply) ProtoMessage() {} 668 | func (*GetCardinalityReply) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{24} } 669 | 670 | func (m *GetCardinalityReply) GetResults() []*CardinalityResult { 671 | if m != nil { 672 | return m.Results 673 | } 674 | return nil 675 | } 676 | 677 | type GetRankingsReply struct { 678 | Results []*RankingsResult `protobuf:"bytes,1,rep,name=results" json:"results,omitempty"` 679 | XXX_unrecognized []byte `json:"-"` 680 | } 681 | 682 | func (m *GetRankingsReply) Reset() { *m = GetRankingsReply{} } 683 | func (m *GetRankingsReply) String() string { return proto.CompactTextString(m) } 684 | func (*GetRankingsReply) ProtoMessage() {} 685 | func (*GetRankingsReply) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{25} } 686 | 687 | func (m *GetRankingsReply) GetResults() []*RankingsResult { 688 | if m != nil { 689 | return m.Results 690 | } 691 | return nil 692 | } 693 | 694 | func init() { 695 | proto.RegisterType((*Empty)(nil), "protobuf.Empty") 696 | proto.RegisterType((*SketchProperties)(nil), "protobuf.SketchProperties") 697 | proto.RegisterType((*SketchState)(nil), "protobuf.SketchState") 698 | proto.RegisterType((*Domain)(nil), "protobuf.Domain") 699 | proto.RegisterType((*Sketch)(nil), "protobuf.Sketch") 700 | proto.RegisterType((*Membership)(nil), "protobuf.Membership") 701 | proto.RegisterType((*Frequency)(nil), "protobuf.Frequency") 702 | proto.RegisterType((*Rank)(nil), "protobuf.Rank") 703 | proto.RegisterType((*CreateSnapshotRequest)(nil), "protobuf.CreateSnapshotRequest") 704 | proto.RegisterType((*CreateSnapshotReply)(nil), "protobuf.CreateSnapshotReply") 705 | proto.RegisterType((*GetSnapshotRequest)(nil), "protobuf.GetSnapshotRequest") 706 | proto.RegisterType((*GetSnapshotReply)(nil), "protobuf.GetSnapshotReply") 707 | proto.RegisterType((*ListRequest)(nil), "protobuf.ListRequest") 708 | proto.RegisterType((*ListReply)(nil), "protobuf.ListReply") 709 | proto.RegisterType((*ListDomainsReply)(nil), "protobuf.ListDomainsReply") 710 | proto.RegisterType((*AddRequest)(nil), "protobuf.AddRequest") 711 | proto.RegisterType((*AddReply)(nil), "protobuf.AddReply") 712 | proto.RegisterType((*GetRequest)(nil), "protobuf.GetRequest") 713 | proto.RegisterType((*MembershipResult)(nil), "protobuf.MembershipResult") 714 | proto.RegisterType((*FrequencyResult)(nil), "protobuf.FrequencyResult") 715 | proto.RegisterType((*CardinalityResult)(nil), "protobuf.CardinalityResult") 716 | proto.RegisterType((*RankingsResult)(nil), "protobuf.RankingsResult") 717 | proto.RegisterType((*GetMembershipReply)(nil), "protobuf.GetMembershipReply") 718 | proto.RegisterType((*GetFrequencyReply)(nil), "protobuf.GetFrequencyReply") 719 | proto.RegisterType((*GetCardinalityReply)(nil), "protobuf.GetCardinalityReply") 720 | proto.RegisterType((*GetRankingsReply)(nil), "protobuf.GetRankingsReply") 721 | proto.RegisterEnum("protobuf.SketchType", SketchType_name, SketchType_value) 722 | proto.RegisterEnum("protobuf.SnapshotStatus", SnapshotStatus_name, SnapshotStatus_value) 723 | } 724 | 725 | // Reference imports to suppress errors if they are not otherwise used. 726 | var _ context.Context 727 | var _ grpc.ClientConn 728 | 729 | // Client API for Skizze service 730 | 731 | type SkizzeClient interface { 732 | CreateSnapshot(ctx context.Context, in *CreateSnapshotRequest, opts ...grpc.CallOption) (*CreateSnapshotReply, error) 733 | GetSnapshot(ctx context.Context, in *GetSnapshotRequest, opts ...grpc.CallOption) (*GetSnapshotReply, error) 734 | List(ctx context.Context, in *ListRequest, opts ...grpc.CallOption) (*ListReply, error) 735 | ListAll(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*ListReply, error) 736 | ListDomains(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*ListDomainsReply, error) 737 | CreateDomain(ctx context.Context, in *Domain, opts ...grpc.CallOption) (*Domain, error) 738 | DeleteDomain(ctx context.Context, in *Domain, opts ...grpc.CallOption) (*Empty, error) 739 | GetDomain(ctx context.Context, in *Domain, opts ...grpc.CallOption) (*Domain, error) 740 | CreateSketch(ctx context.Context, in *Sketch, opts ...grpc.CallOption) (*Sketch, error) 741 | DeleteSketch(ctx context.Context, in *Sketch, opts ...grpc.CallOption) (*Empty, error) 742 | GetSketch(ctx context.Context, in *Sketch, opts ...grpc.CallOption) (*Sketch, error) 743 | Add(ctx context.Context, in *AddRequest, opts ...grpc.CallOption) (*AddReply, error) 744 | GetMembership(ctx context.Context, in *GetRequest, opts ...grpc.CallOption) (*GetMembershipReply, error) 745 | GetFrequency(ctx context.Context, in *GetRequest, opts ...grpc.CallOption) (*GetFrequencyReply, error) 746 | GetCardinality(ctx context.Context, in *GetRequest, opts ...grpc.CallOption) (*GetCardinalityReply, error) 747 | GetRankings(ctx context.Context, in *GetRequest, opts ...grpc.CallOption) (*GetRankingsReply, error) 748 | } 749 | 750 | type skizzeClient struct { 751 | cc *grpc.ClientConn 752 | } 753 | 754 | func NewSkizzeClient(cc *grpc.ClientConn) SkizzeClient { 755 | return &skizzeClient{cc} 756 | } 757 | 758 | func (c *skizzeClient) CreateSnapshot(ctx context.Context, in *CreateSnapshotRequest, opts ...grpc.CallOption) (*CreateSnapshotReply, error) { 759 | out := new(CreateSnapshotReply) 760 | err := grpc.Invoke(ctx, "/protobuf.Skizze/CreateSnapshot", in, out, c.cc, opts...) 761 | if err != nil { 762 | return nil, err 763 | } 764 | return out, nil 765 | } 766 | 767 | func (c *skizzeClient) GetSnapshot(ctx context.Context, in *GetSnapshotRequest, opts ...grpc.CallOption) (*GetSnapshotReply, error) { 768 | out := new(GetSnapshotReply) 769 | err := grpc.Invoke(ctx, "/protobuf.Skizze/GetSnapshot", in, out, c.cc, opts...) 770 | if err != nil { 771 | return nil, err 772 | } 773 | return out, nil 774 | } 775 | 776 | func (c *skizzeClient) List(ctx context.Context, in *ListRequest, opts ...grpc.CallOption) (*ListReply, error) { 777 | out := new(ListReply) 778 | err := grpc.Invoke(ctx, "/protobuf.Skizze/List", in, out, c.cc, opts...) 779 | if err != nil { 780 | return nil, err 781 | } 782 | return out, nil 783 | } 784 | 785 | func (c *skizzeClient) ListAll(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*ListReply, error) { 786 | out := new(ListReply) 787 | err := grpc.Invoke(ctx, "/protobuf.Skizze/ListAll", in, out, c.cc, opts...) 788 | if err != nil { 789 | return nil, err 790 | } 791 | return out, nil 792 | } 793 | 794 | func (c *skizzeClient) ListDomains(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*ListDomainsReply, error) { 795 | out := new(ListDomainsReply) 796 | err := grpc.Invoke(ctx, "/protobuf.Skizze/ListDomains", in, out, c.cc, opts...) 797 | if err != nil { 798 | return nil, err 799 | } 800 | return out, nil 801 | } 802 | 803 | func (c *skizzeClient) CreateDomain(ctx context.Context, in *Domain, opts ...grpc.CallOption) (*Domain, error) { 804 | out := new(Domain) 805 | err := grpc.Invoke(ctx, "/protobuf.Skizze/CreateDomain", in, out, c.cc, opts...) 806 | if err != nil { 807 | return nil, err 808 | } 809 | return out, nil 810 | } 811 | 812 | func (c *skizzeClient) DeleteDomain(ctx context.Context, in *Domain, opts ...grpc.CallOption) (*Empty, error) { 813 | out := new(Empty) 814 | err := grpc.Invoke(ctx, "/protobuf.Skizze/DeleteDomain", in, out, c.cc, opts...) 815 | if err != nil { 816 | return nil, err 817 | } 818 | return out, nil 819 | } 820 | 821 | func (c *skizzeClient) GetDomain(ctx context.Context, in *Domain, opts ...grpc.CallOption) (*Domain, error) { 822 | out := new(Domain) 823 | err := grpc.Invoke(ctx, "/protobuf.Skizze/GetDomain", in, out, c.cc, opts...) 824 | if err != nil { 825 | return nil, err 826 | } 827 | return out, nil 828 | } 829 | 830 | func (c *skizzeClient) CreateSketch(ctx context.Context, in *Sketch, opts ...grpc.CallOption) (*Sketch, error) { 831 | out := new(Sketch) 832 | err := grpc.Invoke(ctx, "/protobuf.Skizze/CreateSketch", in, out, c.cc, opts...) 833 | if err != nil { 834 | return nil, err 835 | } 836 | return out, nil 837 | } 838 | 839 | func (c *skizzeClient) DeleteSketch(ctx context.Context, in *Sketch, opts ...grpc.CallOption) (*Empty, error) { 840 | out := new(Empty) 841 | err := grpc.Invoke(ctx, "/protobuf.Skizze/DeleteSketch", in, out, c.cc, opts...) 842 | if err != nil { 843 | return nil, err 844 | } 845 | return out, nil 846 | } 847 | 848 | func (c *skizzeClient) GetSketch(ctx context.Context, in *Sketch, opts ...grpc.CallOption) (*Sketch, error) { 849 | out := new(Sketch) 850 | err := grpc.Invoke(ctx, "/protobuf.Skizze/GetSketch", in, out, c.cc, opts...) 851 | if err != nil { 852 | return nil, err 853 | } 854 | return out, nil 855 | } 856 | 857 | func (c *skizzeClient) Add(ctx context.Context, in *AddRequest, opts ...grpc.CallOption) (*AddReply, error) { 858 | out := new(AddReply) 859 | err := grpc.Invoke(ctx, "/protobuf.Skizze/Add", in, out, c.cc, opts...) 860 | if err != nil { 861 | return nil, err 862 | } 863 | return out, nil 864 | } 865 | 866 | func (c *skizzeClient) GetMembership(ctx context.Context, in *GetRequest, opts ...grpc.CallOption) (*GetMembershipReply, error) { 867 | out := new(GetMembershipReply) 868 | err := grpc.Invoke(ctx, "/protobuf.Skizze/GetMembership", in, out, c.cc, opts...) 869 | if err != nil { 870 | return nil, err 871 | } 872 | return out, nil 873 | } 874 | 875 | func (c *skizzeClient) GetFrequency(ctx context.Context, in *GetRequest, opts ...grpc.CallOption) (*GetFrequencyReply, error) { 876 | out := new(GetFrequencyReply) 877 | err := grpc.Invoke(ctx, "/protobuf.Skizze/GetFrequency", in, out, c.cc, opts...) 878 | if err != nil { 879 | return nil, err 880 | } 881 | return out, nil 882 | } 883 | 884 | func (c *skizzeClient) GetCardinality(ctx context.Context, in *GetRequest, opts ...grpc.CallOption) (*GetCardinalityReply, error) { 885 | out := new(GetCardinalityReply) 886 | err := grpc.Invoke(ctx, "/protobuf.Skizze/GetCardinality", in, out, c.cc, opts...) 887 | if err != nil { 888 | return nil, err 889 | } 890 | return out, nil 891 | } 892 | 893 | func (c *skizzeClient) GetRankings(ctx context.Context, in *GetRequest, opts ...grpc.CallOption) (*GetRankingsReply, error) { 894 | out := new(GetRankingsReply) 895 | err := grpc.Invoke(ctx, "/protobuf.Skizze/GetRankings", in, out, c.cc, opts...) 896 | if err != nil { 897 | return nil, err 898 | } 899 | return out, nil 900 | } 901 | 902 | // Server API for Skizze service 903 | 904 | type SkizzeServer interface { 905 | CreateSnapshot(context.Context, *CreateSnapshotRequest) (*CreateSnapshotReply, error) 906 | GetSnapshot(context.Context, *GetSnapshotRequest) (*GetSnapshotReply, error) 907 | List(context.Context, *ListRequest) (*ListReply, error) 908 | ListAll(context.Context, *Empty) (*ListReply, error) 909 | ListDomains(context.Context, *Empty) (*ListDomainsReply, error) 910 | CreateDomain(context.Context, *Domain) (*Domain, error) 911 | DeleteDomain(context.Context, *Domain) (*Empty, error) 912 | GetDomain(context.Context, *Domain) (*Domain, error) 913 | CreateSketch(context.Context, *Sketch) (*Sketch, error) 914 | DeleteSketch(context.Context, *Sketch) (*Empty, error) 915 | GetSketch(context.Context, *Sketch) (*Sketch, error) 916 | Add(context.Context, *AddRequest) (*AddReply, error) 917 | GetMembership(context.Context, *GetRequest) (*GetMembershipReply, error) 918 | GetFrequency(context.Context, *GetRequest) (*GetFrequencyReply, error) 919 | GetCardinality(context.Context, *GetRequest) (*GetCardinalityReply, error) 920 | GetRankings(context.Context, *GetRequest) (*GetRankingsReply, error) 921 | } 922 | 923 | func RegisterSkizzeServer(s *grpc.Server, srv SkizzeServer) { 924 | s.RegisterService(&_Skizze_serviceDesc, srv) 925 | } 926 | 927 | func _Skizze_CreateSnapshot_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { 928 | in := new(CreateSnapshotRequest) 929 | if err := dec(in); err != nil { 930 | return nil, err 931 | } 932 | out, err := srv.(SkizzeServer).CreateSnapshot(ctx, in) 933 | if err != nil { 934 | return nil, err 935 | } 936 | return out, nil 937 | } 938 | 939 | func _Skizze_GetSnapshot_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { 940 | in := new(GetSnapshotRequest) 941 | if err := dec(in); err != nil { 942 | return nil, err 943 | } 944 | out, err := srv.(SkizzeServer).GetSnapshot(ctx, in) 945 | if err != nil { 946 | return nil, err 947 | } 948 | return out, nil 949 | } 950 | 951 | func _Skizze_List_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { 952 | in := new(ListRequest) 953 | if err := dec(in); err != nil { 954 | return nil, err 955 | } 956 | out, err := srv.(SkizzeServer).List(ctx, in) 957 | if err != nil { 958 | return nil, err 959 | } 960 | return out, nil 961 | } 962 | 963 | func _Skizze_ListAll_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { 964 | in := new(Empty) 965 | if err := dec(in); err != nil { 966 | return nil, err 967 | } 968 | out, err := srv.(SkizzeServer).ListAll(ctx, in) 969 | if err != nil { 970 | return nil, err 971 | } 972 | return out, nil 973 | } 974 | 975 | func _Skizze_ListDomains_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { 976 | in := new(Empty) 977 | if err := dec(in); err != nil { 978 | return nil, err 979 | } 980 | out, err := srv.(SkizzeServer).ListDomains(ctx, in) 981 | if err != nil { 982 | return nil, err 983 | } 984 | return out, nil 985 | } 986 | 987 | func _Skizze_CreateDomain_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { 988 | in := new(Domain) 989 | if err := dec(in); err != nil { 990 | return nil, err 991 | } 992 | out, err := srv.(SkizzeServer).CreateDomain(ctx, in) 993 | if err != nil { 994 | return nil, err 995 | } 996 | return out, nil 997 | } 998 | 999 | func _Skizze_DeleteDomain_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { 1000 | in := new(Domain) 1001 | if err := dec(in); err != nil { 1002 | return nil, err 1003 | } 1004 | out, err := srv.(SkizzeServer).DeleteDomain(ctx, in) 1005 | if err != nil { 1006 | return nil, err 1007 | } 1008 | return out, nil 1009 | } 1010 | 1011 | func _Skizze_GetDomain_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { 1012 | in := new(Domain) 1013 | if err := dec(in); err != nil { 1014 | return nil, err 1015 | } 1016 | out, err := srv.(SkizzeServer).GetDomain(ctx, in) 1017 | if err != nil { 1018 | return nil, err 1019 | } 1020 | return out, nil 1021 | } 1022 | 1023 | func _Skizze_CreateSketch_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { 1024 | in := new(Sketch) 1025 | if err := dec(in); err != nil { 1026 | return nil, err 1027 | } 1028 | out, err := srv.(SkizzeServer).CreateSketch(ctx, in) 1029 | if err != nil { 1030 | return nil, err 1031 | } 1032 | return out, nil 1033 | } 1034 | 1035 | func _Skizze_DeleteSketch_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { 1036 | in := new(Sketch) 1037 | if err := dec(in); err != nil { 1038 | return nil, err 1039 | } 1040 | out, err := srv.(SkizzeServer).DeleteSketch(ctx, in) 1041 | if err != nil { 1042 | return nil, err 1043 | } 1044 | return out, nil 1045 | } 1046 | 1047 | func _Skizze_GetSketch_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { 1048 | in := new(Sketch) 1049 | if err := dec(in); err != nil { 1050 | return nil, err 1051 | } 1052 | out, err := srv.(SkizzeServer).GetSketch(ctx, in) 1053 | if err != nil { 1054 | return nil, err 1055 | } 1056 | return out, nil 1057 | } 1058 | 1059 | func _Skizze_Add_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { 1060 | in := new(AddRequest) 1061 | if err := dec(in); err != nil { 1062 | return nil, err 1063 | } 1064 | out, err := srv.(SkizzeServer).Add(ctx, in) 1065 | if err != nil { 1066 | return nil, err 1067 | } 1068 | return out, nil 1069 | } 1070 | 1071 | func _Skizze_GetMembership_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { 1072 | in := new(GetRequest) 1073 | if err := dec(in); err != nil { 1074 | return nil, err 1075 | } 1076 | out, err := srv.(SkizzeServer).GetMembership(ctx, in) 1077 | if err != nil { 1078 | return nil, err 1079 | } 1080 | return out, nil 1081 | } 1082 | 1083 | func _Skizze_GetFrequency_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { 1084 | in := new(GetRequest) 1085 | if err := dec(in); err != nil { 1086 | return nil, err 1087 | } 1088 | out, err := srv.(SkizzeServer).GetFrequency(ctx, in) 1089 | if err != nil { 1090 | return nil, err 1091 | } 1092 | return out, nil 1093 | } 1094 | 1095 | func _Skizze_GetCardinality_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { 1096 | in := new(GetRequest) 1097 | if err := dec(in); err != nil { 1098 | return nil, err 1099 | } 1100 | out, err := srv.(SkizzeServer).GetCardinality(ctx, in) 1101 | if err != nil { 1102 | return nil, err 1103 | } 1104 | return out, nil 1105 | } 1106 | 1107 | func _Skizze_GetRankings_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { 1108 | in := new(GetRequest) 1109 | if err := dec(in); err != nil { 1110 | return nil, err 1111 | } 1112 | out, err := srv.(SkizzeServer).GetRankings(ctx, in) 1113 | if err != nil { 1114 | return nil, err 1115 | } 1116 | return out, nil 1117 | } 1118 | 1119 | var _Skizze_serviceDesc = grpc.ServiceDesc{ 1120 | ServiceName: "protobuf.Skizze", 1121 | HandlerType: (*SkizzeServer)(nil), 1122 | Methods: []grpc.MethodDesc{ 1123 | { 1124 | MethodName: "CreateSnapshot", 1125 | Handler: _Skizze_CreateSnapshot_Handler, 1126 | }, 1127 | { 1128 | MethodName: "GetSnapshot", 1129 | Handler: _Skizze_GetSnapshot_Handler, 1130 | }, 1131 | { 1132 | MethodName: "List", 1133 | Handler: _Skizze_List_Handler, 1134 | }, 1135 | { 1136 | MethodName: "ListAll", 1137 | Handler: _Skizze_ListAll_Handler, 1138 | }, 1139 | { 1140 | MethodName: "ListDomains", 1141 | Handler: _Skizze_ListDomains_Handler, 1142 | }, 1143 | { 1144 | MethodName: "CreateDomain", 1145 | Handler: _Skizze_CreateDomain_Handler, 1146 | }, 1147 | { 1148 | MethodName: "DeleteDomain", 1149 | Handler: _Skizze_DeleteDomain_Handler, 1150 | }, 1151 | { 1152 | MethodName: "GetDomain", 1153 | Handler: _Skizze_GetDomain_Handler, 1154 | }, 1155 | { 1156 | MethodName: "CreateSketch", 1157 | Handler: _Skizze_CreateSketch_Handler, 1158 | }, 1159 | { 1160 | MethodName: "DeleteSketch", 1161 | Handler: _Skizze_DeleteSketch_Handler, 1162 | }, 1163 | { 1164 | MethodName: "GetSketch", 1165 | Handler: _Skizze_GetSketch_Handler, 1166 | }, 1167 | { 1168 | MethodName: "Add", 1169 | Handler: _Skizze_Add_Handler, 1170 | }, 1171 | { 1172 | MethodName: "GetMembership", 1173 | Handler: _Skizze_GetMembership_Handler, 1174 | }, 1175 | { 1176 | MethodName: "GetFrequency", 1177 | Handler: _Skizze_GetFrequency_Handler, 1178 | }, 1179 | { 1180 | MethodName: "GetCardinality", 1181 | Handler: _Skizze_GetCardinality_Handler, 1182 | }, 1183 | { 1184 | MethodName: "GetRankings", 1185 | Handler: _Skizze_GetRankings_Handler, 1186 | }, 1187 | }, 1188 | Streams: []grpc.StreamDesc{}, 1189 | } 1190 | 1191 | var fileDescriptor0 = []byte{ 1192 | // 1037 bytes of a gzipped FileDescriptorProto 1193 | 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x56, 0xed, 0x72, 0xdb, 0x44, 1194 | 0x14, 0xad, 0x3f, 0xe2, 0x8f, 0xab, 0xd4, 0x71, 0x37, 0x29, 0x18, 0xb5, 0x1d, 0x3a, 0x0b, 0xc3, 1195 | 0x78, 0x02, 0x93, 0x50, 0x37, 0x81, 0x81, 0xe9, 0x30, 0x63, 0x1c, 0x3b, 0x38, 0xc4, 0x21, 0xac, 1196 | 0xc9, 0x6f, 0x46, 0xb5, 0x37, 0x8d, 0x26, 0x92, 0x6d, 0xb4, 0x32, 0xd3, 0xe4, 0x09, 0x78, 0x1b, 1197 | 0x9e, 0x86, 0xf7, 0x61, 0x3f, 0x24, 0xed, 0xae, 0x6c, 0xd3, 0xc9, 0x0f, 0xfe, 0x69, 0xef, 0xde, 1198 | 0x73, 0xee, 0xd5, 0xd9, 0xbd, 0x47, 0x82, 0xcf, 0x58, 0x34, 0x39, 0x9c, 0x7a, 0xb1, 0x17, 0xce, 1199 | 0xa7, 0x34, 0x38, 0x5c, 0x44, 0xf3, 0x78, 0xfe, 0x76, 0x79, 0x7d, 0xc8, 0x6e, 0xfd, 0xfb, 0x7b, 1200 | 0x7a, 0x20, 0xd7, 0xa8, 0x96, 0x86, 0x71, 0x15, 0xb6, 0xfa, 0xe1, 0x22, 0xbe, 0xc3, 0x01, 0x34, 1201 | 0xc7, 0xb7, 0x34, 0x9e, 0xdc, 0x5c, 0x46, 0xf3, 0x05, 0x8d, 0x62, 0x9f, 0x32, 0xf4, 0x05, 0x34, 1202 | 0x42, 0xef, 0xfd, 0xd5, 0xcc, 0xff, 0x63, 0x49, 0x87, 0x31, 0x0d, 0x59, 0xab, 0xf0, 0xb2, 0xd0, 1203 | 0x2e, 0x91, 0x5c, 0x14, 0x3d, 0x87, 0x3a, 0x8d, 0xa2, 0x79, 0x44, 0xbc, 0x98, 0xb6, 0x8a, 0x3c, 1204 | 0xa5, 0x48, 0x74, 0x00, 0x21, 0x28, 0x33, 0xff, 0x9e, 0xb6, 0x4a, 0x12, 0x2b, 0x9f, 0xf1, 0x08, 1205 | 0x1c, 0x55, 0x6d, 0x1c, 0x8b, 0x14, 0x17, 0x6a, 0xd7, 0x7e, 0x10, 0x48, 0x7c, 0x41, 0xe2, 0xb3, 1206 | 0x35, 0xc2, 0xb0, 0x1d, 0x78, 0x2c, 0x1e, 0xcf, 0xbc, 0x05, 0xbb, 0x99, 0xc7, 0x92, 0xbf, 0x44, 1207 | 0xac, 0x18, 0x3e, 0x83, 0xca, 0xc9, 0x3c, 0xf4, 0xfc, 0x99, 0x28, 0x36, 0xf3, 0x42, 0xc1, 0x52, 1208 | 0x6c, 0xd7, 0x89, 0x7c, 0x46, 0x5f, 0x41, 0x8d, 0xc9, 0x62, 0x94, 0x71, 0x74, 0xa9, 0xed, 0x74, 1209 | 0x9a, 0x07, 0xa9, 0x00, 0x07, 0xaa, 0x0d, 0x92, 0x65, 0xe0, 0xbf, 0x0b, 0x50, 0x51, 0xc1, 0xb5, 1210 | 0x64, 0x6d, 0x28, 0xc7, 0x77, 0x0b, 0xf1, 0x9a, 0xc5, 0x76, 0xa3, 0xb3, 0x97, 0x27, 0xfa, 0x8d, 1211 | 0xef, 0x11, 0x99, 0x81, 0xbe, 0x07, 0x58, 0x64, 0x5a, 0xca, 0xb7, 0x77, 0x3a, 0x6e, 0x3e, 0x5f, 1212 | 0xab, 0x4d, 0x8c, 0x6c, 0xf4, 0x25, 0x6c, 0x31, 0xa1, 0x4c, 0xab, 0x2c, 0x61, 0x4f, 0xf3, 0x30, 1213 | 0x29, 0x1b, 0x51, 0x39, 0xf8, 0x07, 0x80, 0x11, 0x0d, 0xdf, 0xd2, 0x88, 0xdd, 0xf8, 0x0b, 0xb4, 1214 | 0x07, 0x5b, 0x7f, 0x7a, 0xc1, 0x32, 0xed, 0x5a, 0x2d, 0x84, 0xc2, 0x3e, 0x53, 0x59, 0xb2, 0xf5, 1215 | 0x1a, 0xc9, 0xd6, 0xf8, 0x5b, 0xa8, 0x0f, 0x22, 0xca, 0x4f, 0x73, 0x36, 0xb9, 0xdb, 0x00, 0xe7, 1216 | 0xd1, 0xc9, 0x7c, 0x39, 0x8b, 0x25, 0xb6, 0x44, 0xd4, 0x02, 0x77, 0xa0, 0x4c, 0xbc, 0xd9, 0xed, 1217 | 0x83, 0x30, 0x1f, 0xc3, 0xd3, 0x5e, 0x44, 0x79, 0xdb, 0xe9, 0xe1, 0x11, 0x51, 0x99, 0xc5, 0x38, 1218 | 0x84, 0xdd, 0xfc, 0xc6, 0x22, 0xb8, 0x43, 0x5f, 0x43, 0x45, 0xbc, 0xe5, 0x92, 0x49, 0xf2, 0x46, 1219 | 0xa7, 0x65, 0x48, 0x91, 0x24, 0x8e, 0xe5, 0x3e, 0x49, 0xf2, 0xd0, 0xe7, 0xf0, 0x58, 0x3d, 0x8d, 1220 | 0x28, 0x63, 0xde, 0x3b, 0x75, 0x23, 0xeb, 0xc4, 0x0e, 0xe2, 0x3d, 0x40, 0xa7, 0x34, 0xce, 0x37, 1221 | 0xf1, 0x57, 0x01, 0x9a, 0x56, 0xf8, 0x7f, 0x6c, 0x41, 0x8c, 0x4d, 0xec, 0x87, 0xbc, 0xac, 0x17, 1222 | 0x2e, 0x92, 0xe9, 0xd0, 0x01, 0x7e, 0x2a, 0xce, 0xb9, 0xcf, 0xd2, 0xce, 0xb2, 0x7b, 0x57, 0xf8, 1223 | 0xd0, 0xbd, 0xc3, 0xdf, 0x41, 0x5d, 0x01, 0x45, 0xef, 0xe6, 0xdd, 0x2f, 0x7c, 0xf0, 0xee, 0xb7, 1224 | 0xa1, 0x29, 0xa0, 0x6a, 0x96, 0x98, 0x62, 0xe0, 0xc7, 0x28, 0x2e, 0xbe, 0x82, 0xf3, 0xc3, 0x95, 1225 | 0x0b, 0xfc, 0x1e, 0xa0, 0x3b, 0x9d, 0xea, 0xe6, 0x2a, 0x53, 0x89, 0x91, 0xd3, 0x6b, 0xd5, 0x50, 1226 | 0x5c, 0x24, 0xd9, 0x17, 0x99, 0xaa, 0x9a, 0x94, 0x64, 0x5d, 0x37, 0xc9, 0x3e, 0xfa, 0x08, 0x2a, 1227 | 0xf2, 0x1e, 0x89, 0xd1, 0x11, 0x85, 0x93, 0x15, 0x06, 0xa8, 0xc9, 0xca, 0xbc, 0x37, 0x4c, 0x00, 1228 | 0xf8, 0x69, 0xa5, 0x5d, 0x3c, 0xe8, 0x5d, 0x0d, 0xfe, 0xa2, 0xc5, 0x7f, 0x06, 0x4d, 0x3d, 0x4d, 1229 | 0x84, 0xb2, 0x65, 0x10, 0xa3, 0x6f, 0xc0, 0x09, 0xb3, 0x58, 0x4a, 0x6e, 0x9c, 0x81, 0x01, 0x30, 1230 | 0x13, 0xf1, 0x4f, 0xb0, 0x93, 0x4d, 0x56, 0x42, 0x75, 0x0c, 0xce, 0x75, 0x12, 0xf2, 0x33, 0x3f, 1231 | 0xda, 0xd5, 0x54, 0x3a, 0xdf, 0xcc, 0xc3, 0xc7, 0xf0, 0xa4, 0xe7, 0x45, 0x53, 0x7f, 0xe6, 0x05, 1232 | 0x7e, 0x9c, 0x72, 0xbd, 0x04, 0x67, 0xa2, 0x83, 0xf2, 0x6a, 0x94, 0x88, 0x19, 0xc2, 0x6f, 0xa0, 1233 | 0x21, 0x26, 0xd4, 0x9f, 0xbd, 0x63, 0x09, 0x66, 0x1f, 0x6a, 0x51, 0x12, 0x49, 0xde, 0xa3, 0xa1, 1234 | 0x8b, 0x8b, 0x5c, 0x92, 0xed, 0x73, 0x29, 0xc4, 0x8c, 0x98, 0x6a, 0x88, 0x0b, 0x71, 0x04, 0xd5, 1235 | 0x48, 0x72, 0xa5, 0x04, 0xee, 0x5a, 0x21, 0x64, 0x0a, 0x49, 0x53, 0xb9, 0x14, 0x4f, 0x38, 0x97, 1236 | 0xa1, 0x86, 0xa0, 0x7a, 0x9d, 0xa7, 0xfa, 0x64, 0x9d, 0x10, 0x39, 0xa6, 0x73, 0xd8, 0xe5, 0x4c, 1237 | 0x96, 0x1a, 0x82, 0xeb, 0x38, 0xcf, 0xf5, 0x4c, 0x73, 0xad, 0x48, 0xa7, 0xd9, 0x06, 0x72, 0xe0, 1238 | 0xb5, 0x48, 0x82, 0xaa, 0x93, 0xa7, 0x6a, 0xd9, 0x12, 0x69, 0x39, 0x33, 0x9e, 0xfd, 0x23, 0x00, 1239 | 0x3d, 0x89, 0xa8, 0x06, 0xe5, 0x51, 0x7f, 0xf4, 0x63, 0xb3, 0x20, 0x9e, 0x06, 0xa4, 0xff, 0x6b, 1240 | 0xb3, 0x28, 0x9e, 0x48, 0xf7, 0xe2, 0xe7, 0x66, 0x49, 0x3c, 0xf5, 0xba, 0xe4, 0xa4, 0x59, 0xde, 1241 | 0x3f, 0x83, 0x86, 0x6d, 0x21, 0xc8, 0x81, 0xea, 0x65, 0xff, 0xe2, 0x64, 0x78, 0x71, 0xca, 0xc1, 1242 | 0x3b, 0xe0, 0x0c, 0x2f, 0x7e, 0xbf, 0x24, 0xbf, 0x9c, 0x92, 0xfe, 0x78, 0xcc, 0x39, 0x1a, 0xbc, 1243 | 0xca, 0x55, 0xaf, 0xc7, 0x17, 0x83, 0xab, 0x73, 0xce, 0x04, 0x50, 0x19, 0x74, 0x87, 0xe7, 0x7d, 1244 | 0xce, 0xd5, 0xf9, 0xa7, 0x2a, 0x3e, 0x5c, 0xe2, 0x2b, 0x8f, 0x08, 0x34, 0x6c, 0x2f, 0x45, 0x9f, 1245 | 0x1a, 0x62, 0xac, 0xb3, 0x5f, 0xf7, 0xc5, 0xe6, 0x04, 0x31, 0x69, 0x8f, 0xd0, 0x10, 0x1c, 0xc3, 1246 | 0x19, 0xd1, 0x73, 0x9d, 0xbf, 0xea, 0xa3, 0xae, 0xbb, 0x61, 0x57, 0x51, 0x1d, 0x41, 0x59, 0xd8, 1247 | 0x0c, 0x32, 0x3e, 0x6b, 0x86, 0xd5, 0xb9, 0xbb, 0xf9, 0xb0, 0x42, 0xbd, 0x82, 0xaa, 0x58, 0x76, 1248 | 0x83, 0x00, 0xed, 0xe8, 0x0c, 0xf9, 0xf7, 0xb2, 0x09, 0xf2, 0x46, 0x79, 0x68, 0xe2, 0x67, 0xab, 1249 | 0x30, 0xd7, 0x86, 0x99, 0xbe, 0x27, 0xdb, 0xdc, 0x56, 0x52, 0x24, 0xff, 0x16, 0x2b, 0xae, 0xe6, 1250 | 0xae, 0x44, 0x38, 0xea, 0x35, 0x6c, 0x9f, 0xd0, 0x80, 0xfe, 0x07, 0x2a, 0xdf, 0x86, 0x7c, 0xb7, 1251 | 0x3a, 0xd7, 0xe9, 0x41, 0x75, 0xb2, 0xee, 0x92, 0x9f, 0x95, 0x15, 0xaf, 0x73, 0x57, 0x22, 0x66, 1252 | 0x77, 0x1b, 0x51, 0x1b, 0xbb, 0x7b, 0x50, 0x9d, 0x57, 0x50, 0xe2, 0x2e, 0x8d, 0x0c, 0x8f, 0xd4, 1253 | 0x9f, 0x0b, 0x17, 0xe5, 0xa2, 0x4a, 0xee, 0x3e, 0x3c, 0xb6, 0xdc, 0xc6, 0x04, 0x6b, 0x97, 0x77, 1254 | 0xed, 0x8b, 0x97, 0x33, 0x27, 0x4e, 0xd3, 0x83, 0x6d, 0xd3, 0x68, 0x36, 0xb0, 0x3c, 0xb3, 0xa2, 1255 | 0xb6, 0x2d, 0x71, 0x92, 0x53, 0x68, 0xd8, 0x1e, 0xb3, 0x81, 0xe6, 0x85, 0x15, 0xcd, 0x7b, 0x12, 1256 | 0x27, 0xea, 0xca, 0xa9, 0x49, 0x4d, 0x63, 0x03, 0x8b, 0x3d, 0x2d, 0x96, 0x17, 0xe1, 0x47, 0xff, 1257 | 0x06, 0x00, 0x00, 0xff, 0xff, 0xc7, 0x64, 0x21, 0x77, 0xd2, 0x0b, 0x00, 0x00, 1258 | } 1259 | -------------------------------------------------------------------------------- /src/datamodel/protobuf/skizze.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package protobuf; 4 | 5 | service Skizze { 6 | rpc CreateSnapshot (CreateSnapshotRequest) returns (CreateSnapshotReply) {} 7 | rpc GetSnapshot (GetSnapshotRequest) returns (GetSnapshotReply) {} 8 | 9 | rpc List (ListRequest) returns (ListReply) {} 10 | rpc ListAll (Empty) returns (ListReply) {} 11 | rpc ListDomains (Empty) returns (ListDomainsReply) {} 12 | 13 | rpc CreateDomain (Domain) returns (Domain) {} 14 | rpc DeleteDomain (Domain) returns (Empty) {} 15 | rpc GetDomain (Domain) returns (Domain) {} 16 | 17 | rpc CreateSketch(Sketch) returns (Sketch) {} 18 | rpc DeleteSketch(Sketch) returns (Empty) {} 19 | rpc GetSketch(Sketch) returns (Sketch) {} 20 | 21 | rpc Add (AddRequest) returns (AddReply) {} 22 | 23 | rpc GetMembership (GetRequest) returns (GetMembershipReply) {} 24 | rpc GetFrequency (GetRequest) returns (GetFrequencyReply) {} 25 | rpc GetCardinality (GetRequest) returns (GetCardinalityReply) {} 26 | rpc GetRankings (GetRequest) returns (GetRankingsReply) {} 27 | } 28 | 29 | 30 | // 31 | // Enums 32 | // 33 | enum SketchType { 34 | MEMB = 1; 35 | FREQ = 2; 36 | RANK = 3; 37 | CARD = 4; 38 | } 39 | 40 | enum SnapshotStatus { 41 | PENDING = 1; 42 | IN_PROGRESS = 2; 43 | SUCCESSFUL = 3; 44 | FAILED = 4; 45 | } 46 | 47 | 48 | // 49 | // Generic Structures 50 | // 51 | message Empty { 52 | } 53 | 54 | message SketchProperties { 55 | optional int64 maxUniqueItems = 1; // MEMB, FREQ 56 | optional float errorRate = 2; // MEMB, FREQ 57 | optional int64 size = 3; // RANK 58 | } 59 | 60 | message SketchState { 61 | optional float fillRate = 1; // 0.0 -> 1.0 62 | optional int64 lastSnapshot = 2; // Age of last snapshot in seconds since epoch 63 | } 64 | 65 | // CreateDomain: name:required, propertiess:optional (array = nSketchTypes, order of types above) 66 | // DeleteDomain: name:required 67 | // GetDomain : name:required 68 | message Domain { 69 | required string name = 1; 70 | repeated Sketch sketches = 2; 71 | } 72 | 73 | // CreateSketch: name:required, type:required, properties:optional 74 | // DeleteSketch: name:required, type:required 75 | // GetSketch : name:required, type:required 76 | message Sketch { 77 | required string name = 1; 78 | required SketchType type = 2; 79 | optional SketchProperties properties = 3; 80 | optional SketchState state = 4; 81 | } 82 | 83 | message Membership { 84 | required string value = 1; 85 | required bool isMember = 2; 86 | } 87 | 88 | message Frequency { 89 | required string value = 1; 90 | required int64 count = 2; 91 | } 92 | 93 | message Rank { 94 | required string value = 1; 95 | required int64 count = 2; 96 | } 97 | 98 | 99 | // 100 | // Request/Reply Envelopes 101 | // 102 | 103 | // Right now empty but in the future can request specific snapshot location 104 | // (e.g. S3 or disk) and snapshot options 105 | message CreateSnapshotRequest { 106 | } 107 | 108 | message CreateSnapshotReply { 109 | required SnapshotStatus status = 1; 110 | optional string statusMessage = 2; 111 | } 112 | 113 | // Empty for now, future can send in id of specific snapshot 114 | message GetSnapshotRequest { 115 | } 116 | 117 | message GetSnapshotReply { 118 | required SnapshotStatus status = 1; 119 | optional string statusMessage = 2; 120 | optional int64 timestamp = 3; 121 | } 122 | 123 | message ListRequest { 124 | required SketchType type = 1; 125 | } 126 | 127 | message ListReply { 128 | repeated Sketch sketches = 1; 129 | } 130 | 131 | message ListDomainsReply { 132 | repeated string names = 1; 133 | } 134 | 135 | message AddRequest { 136 | optional Domain domain = 1; 137 | optional Sketch sketch = 2; 138 | repeated string values = 3; 139 | } 140 | 141 | message AddReply { 142 | } 143 | 144 | // All Sketches will be of one kind 145 | // All values will apply to all sketches (if card or ranking, values will be ignored) 146 | message GetRequest { 147 | repeated Sketch sketches = 1; // MEMB:users-20151214,MEMB:users-20151214 148 | repeated string values = 2; // "gary","michelle","ray","harpindar" // Apply to all sketches above 149 | } 150 | 151 | message MembershipResult { 152 | repeated Membership memberships = 1; 153 | } 154 | 155 | message FrequencyResult { 156 | repeated Frequency frequencies = 2; 157 | } 158 | 159 | message CardinalityResult { 160 | required int64 cardinality = 1; 161 | } 162 | 163 | message RankingsResult { 164 | repeated Rank rankings = 1; 165 | } 166 | 167 | message GetMembershipReply { 168 | repeated MembershipResult results = 1; 169 | } 170 | 171 | message GetFrequencyReply { 172 | repeated FrequencyResult results = 1; 173 | } 174 | 175 | message GetCardinalityReply { 176 | repeated CardinalityResult results = 1; 177 | } 178 | 179 | message GetRankingsReply { 180 | repeated RankingsResult results = 1; 181 | } 182 | -------------------------------------------------------------------------------- /src/datamodel/sketcher.go: -------------------------------------------------------------------------------- 1 | package datamodel 2 | 3 | // Sketcher ... 4 | type Sketcher interface { 5 | Add([][]byte) (bool, error) 6 | Get(interface{}) (interface{}, error) 7 | } 8 | -------------------------------------------------------------------------------- /src/manager/domain.go: -------------------------------------------------------------------------------- 1 | package manager 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | 7 | "github.com/gogo/protobuf/proto" 8 | 9 | "datamodel" 10 | pb "datamodel/protobuf" 11 | ) 12 | 13 | type domainManager struct { 14 | domains map[string][]string 15 | sketches *sketchManager 16 | info *infoManager 17 | } 18 | 19 | func newDomainManager(info *infoManager, sketches *sketchManager) *domainManager { 20 | return &domainManager{ 21 | domains: make(map[string][]string), 22 | info: info, 23 | sketches: sketches, 24 | } 25 | } 26 | 27 | func (m *domainManager) create(id string, infos map[string]*datamodel.Info) error { 28 | if _, ok := m.domains[id]; ok { 29 | return fmt.Errorf(`Domain with name "%s" already exists`, id) 30 | } 31 | 32 | var err error 33 | var ids []string 34 | tmpInfos := make(map[string]*datamodel.Info) 35 | tmpSketches := make(map[string]*datamodel.Info) 36 | for id, info := range infos { 37 | if err = m.info.create(info); err != nil { 38 | break 39 | } 40 | tmpInfos[id] = info 41 | if err = m.sketches.create(info); err != nil { 42 | break 43 | } 44 | tmpSketches[id] = info 45 | ids = append(ids, info.ID()) 46 | } 47 | 48 | if len(tmpInfos) != len(infos) { 49 | for _, v := range tmpInfos { 50 | if err := m.info.delete(v.ID()); err != nil { 51 | // TODO: print out something 52 | } 53 | } 54 | } 55 | if len(tmpSketches) != len(infos) { 56 | for _, v := range tmpSketches { 57 | if err := m.sketches.delete(v.ID()); err != nil { 58 | // TODO: print out something 59 | } 60 | } 61 | } 62 | 63 | if len(ids) < len(datamodel.GetTypes()) { 64 | return fmt.Errorf("Not enough sketches") 65 | } 66 | m.domains[id] = ids 67 | return nil 68 | } 69 | 70 | // FIXME: maybe return a list of errors? 71 | func (m *domainManager) delete(id string) error { 72 | var lastErr error 73 | if ids, ok := m.domains[id]; ok { 74 | for _, id := range ids { 75 | if info := m.info.get(id); info != nil { 76 | if err := m.sketches.delete(info.ID()); err != nil { 77 | // TODO: print something ? 78 | lastErr = err 79 | } 80 | if err := m.info.delete(info.ID()); err != nil { 81 | // TODO: print something ? 82 | lastErr = err 83 | } 84 | } 85 | } 86 | } 87 | delete(m.domains, id) 88 | // FIXME: return error if not exist ? 89 | return lastErr 90 | } 91 | 92 | func (m *domainManager) add(id string, values []string) error { 93 | sketches, ok := m.domains[id] 94 | 95 | if !ok { 96 | return fmt.Errorf(`Domain "%s" does not exists`, id) 97 | } 98 | 99 | var wg sync.WaitGroup 100 | wg.Add(len(sketches)) 101 | 102 | for _, sketch := range sketches { 103 | go func(sk string) { 104 | if err := m.sketches.add(sk, values); err != nil { 105 | logger.Errorf("%q\n", err) 106 | } 107 | wg.Done() 108 | }(sketch) 109 | } 110 | 111 | wg.Wait() 112 | return nil 113 | } 114 | 115 | // FIXME: return all sketches with domain 116 | func (m *domainManager) get(id string) (*pb.Domain, error) { 117 | sketchIds, ok := m.domains[id] 118 | if !ok { 119 | return nil, fmt.Errorf("Could not find domain %s", id) 120 | } 121 | var sketches []*pb.Sketch 122 | for _, id := range sketchIds { 123 | s := m.info.get(id) 124 | if s != nil { 125 | sketches = append(sketches, s.Sketch) 126 | } 127 | } 128 | domain := &pb.Domain{ 129 | Name: proto.String(id), 130 | Sketches: sketches, 131 | } 132 | return domain, nil 133 | } 134 | -------------------------------------------------------------------------------- /src/manager/info.go: -------------------------------------------------------------------------------- 1 | package manager 2 | 3 | import ( 4 | "fmt" 5 | 6 | "datamodel" 7 | ) 8 | 9 | type infoManager struct { 10 | info map[string]*datamodel.Info 11 | } 12 | 13 | func newInfoManager() *infoManager { 14 | return &infoManager{ 15 | info: make(map[string]*datamodel.Info), 16 | } 17 | } 18 | 19 | func (m *infoManager) get(id string) *datamodel.Info { 20 | info, _ := m.info[id] 21 | return info 22 | } 23 | 24 | func (m *infoManager) create(info *datamodel.Info) error { 25 | if _, ok := m.info[info.ID()]; ok { 26 | return fmt.Errorf(`Sketch of type "%s" with name "%s" already exists`, 27 | info.GetType(), info.GetName()) 28 | } 29 | m.info[info.ID()] = info 30 | return nil 31 | } 32 | 33 | // FIXME: should take array or map instead? 34 | func (m *infoManager) delete(id string) error { 35 | if _, ok := m.info[id]; !ok { 36 | return fmt.Errorf(`Sketch "%s" already exists`, id) 37 | } 38 | // FIXME: return error if not exist 39 | delete(m.info, id) 40 | return nil 41 | } 42 | -------------------------------------------------------------------------------- /src/manager/manager.go: -------------------------------------------------------------------------------- 1 | package manager 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | "strconv" 7 | 8 | "datamodel" 9 | pb "datamodel/protobuf" 10 | 11 | "github.com/njpatel/loggo" 12 | ) 13 | 14 | var logger = loggo.GetLogger("manager") 15 | 16 | func isValidType(info *datamodel.Info) bool { 17 | if info.Type == nil { 18 | return false 19 | } 20 | return len(datamodel.GetTypeString(info.GetType())) != 0 21 | } 22 | 23 | // Manager is responsible for manipulating the sketches and syncing to disk 24 | type Manager struct { 25 | infos *infoManager 26 | sketches *sketchManager 27 | domains *domainManager 28 | } 29 | 30 | // NewManager ... 31 | func NewManager() *Manager { 32 | sketches := newSketchManager() 33 | infos := newInfoManager() 34 | domains := newDomainManager(infos, sketches) 35 | 36 | m := &Manager{ 37 | sketches: sketches, 38 | infos: infos, 39 | domains: domains, 40 | } 41 | 42 | return m 43 | } 44 | 45 | // CreateSketch ... 46 | func (m *Manager) CreateSketch(info *datamodel.Info) error { 47 | if !isValidType(info) { 48 | return fmt.Errorf("Can not create sketch of type %s, invalid type.", info.Type) 49 | } 50 | if err := m.infos.create(info); err != nil { 51 | return err 52 | } 53 | if err := m.sketches.create(info); err != nil { 54 | // If error occurred during creation of sketch, delete info 55 | if err2 := m.infos.delete(info.ID()); err2 != nil { 56 | return fmt.Errorf("%q\n%q ", err, err2) 57 | } 58 | return err 59 | } 60 | return nil 61 | } 62 | 63 | // CreateDomain ... 64 | func (m *Manager) CreateDomain(info *datamodel.Info) error { 65 | infos := make(map[string]*datamodel.Info) 66 | for _, typ := range datamodel.GetTypesPb() { 67 | styp := typ 68 | tmpInfo := info.Copy() 69 | tmpInfo.Type = &styp 70 | infos[tmpInfo.ID()] = tmpInfo 71 | } 72 | return m.domains.create(info.GetName(), infos) 73 | } 74 | 75 | // AddToSketch ... 76 | func (m *Manager) AddToSketch(id string, values []string) error { 77 | return m.sketches.add(id, values) 78 | } 79 | 80 | // AddToDomain ... 81 | func (m *Manager) AddToDomain(id string, values []string) error { 82 | return m.domains.add(id, values) 83 | } 84 | 85 | // DeleteSketch ... 86 | func (m *Manager) DeleteSketch(id string) error { 87 | if err := m.infos.delete(id); err != nil { 88 | return err 89 | } 90 | return m.sketches.delete(id) 91 | } 92 | 93 | // DeleteDomain ... 94 | func (m *Manager) DeleteDomain(id string) error { 95 | return m.domains.delete(id) 96 | } 97 | 98 | type tupleResult [][2]string 99 | 100 | func (slice tupleResult) Len() int { 101 | return len(slice) 102 | } 103 | 104 | func (slice tupleResult) Less(i, j int) bool { 105 | if slice[i][0] == slice[j][0] { 106 | return slice[i][1] < slice[j][1] 107 | } 108 | return slice[i][0] < slice[j][0] 109 | } 110 | 111 | func (slice tupleResult) Swap(i, j int) { 112 | slice[i], slice[j] = slice[j], slice[i] 113 | } 114 | 115 | // GetSketches return a list of sketch tuples [name, type] 116 | func (m *Manager) GetSketches() [][2]string { 117 | sketches := tupleResult{} 118 | for _, v := range m.infos.info { 119 | sketches = append(sketches, 120 | [2]string{v.GetName(), 121 | datamodel.GetTypeString(v.GetType())}) 122 | } 123 | sort.Sort(sketches) 124 | return sketches 125 | } 126 | 127 | // GetDomains return a list of sketch tuples [name, type] 128 | func (m *Manager) GetDomains() [][2]string { 129 | domains := tupleResult{} 130 | for k, v := range m.domains.domains { 131 | domains = append(domains, [2]string{k, strconv.Itoa(len(v))}) 132 | } 133 | sort.Sort(domains) 134 | return domains 135 | } 136 | 137 | // GetSketch ... 138 | func (m *Manager) GetSketch(id string) (*datamodel.Info, error) { 139 | info := m.infos.get(id) 140 | if info == nil { 141 | return nil, fmt.Errorf("No such sketch %s", id) 142 | } 143 | return info, nil 144 | } 145 | 146 | // GetDomain ... 147 | func (m *Manager) GetDomain(id string) (*pb.Domain, error) { 148 | return m.domains.get(id) 149 | } 150 | 151 | // GetFromSketch ... 152 | func (m *Manager) GetFromSketch(id string, data interface{}) (interface{}, error) { 153 | return m.sketches.get(id, data) 154 | } 155 | 156 | // Destroy ... 157 | func (m *Manager) Destroy() { 158 | } 159 | -------------------------------------------------------------------------------- /src/manager/manager_test.go: -------------------------------------------------------------------------------- 1 | package manager 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "config" 8 | "datamodel" 9 | pb "datamodel/protobuf" 10 | "utils" 11 | "testutils" 12 | ) 13 | 14 | func TestNoSketches(t *testing.T) { 15 | config.Reset() 16 | testutils.SetupTests() 17 | defer testutils.TearDownTests() 18 | m := NewManager() 19 | if sketches := m.GetSketches(); len(sketches) != 0 { 20 | t.Error("Expected 0 sketches, got", len(sketches)) 21 | } 22 | } 23 | 24 | func TestCreateSketch(t *testing.T) { 25 | config.Reset() 26 | testutils.SetupTests() 27 | defer testutils.TearDownTests() 28 | 29 | m := NewManager() 30 | info := datamodel.NewEmptyInfo() 31 | typ := pb.SketchType_CARD 32 | info.Properties.MaxUniqueItems = utils.Int64p(10000) 33 | info.Name = utils.Stringp(fmt.Sprintf("marvel")) 34 | info.Type = &typ 35 | 36 | if err := m.CreateSketch(info); err != nil { 37 | t.Error("Expected no errors, got", err) 38 | } 39 | if sketches := m.GetSketches(); len(sketches) != 1 { 40 | t.Error("Expected 1 sketches, got", len(sketches)) 41 | } else if sketches[0][0] != "marvel" || sketches[0][1] != "card" { 42 | t.Error("Expected [[marvel card]], got", sketches) 43 | } 44 | 45 | // Create a second Sketch 46 | info2 := datamodel.NewEmptyInfo() 47 | typ2 := pb.SketchType_RANK 48 | info2.Properties.Size = utils.Int64p(10) 49 | info2.Name = utils.Stringp(fmt.Sprintf("marvel")) 50 | info2.Type = &typ2 51 | 52 | if err := m.CreateSketch(info2); err != nil { 53 | t.Error("Expected no errors, got", err) 54 | } 55 | if sketches := m.GetSketches(); len(sketches) != 2 { 56 | t.Error("Expected 2 sketches, got", len(sketches)) 57 | } else if sketches[0][0] != "marvel" || sketches[0][1] != "card" { 58 | t.Error("Expected [[marvel card]], got", sketches[0][0], sketches[0][1]) 59 | } else if sketches[1][0] != "marvel" || sketches[1][1] != "rank" { 60 | t.Error("Expected [[marvel rank]], got", sketches[1][0], sketches[1][1]) 61 | } 62 | } 63 | 64 | func TestCreateAndSaveSketch(t *testing.T) { 65 | config.Reset() 66 | testutils.SetupTests() 67 | defer testutils.TearDownTests() 68 | 69 | m := NewManager() 70 | info := datamodel.NewEmptyInfo() 71 | typ := pb.SketchType_CARD 72 | info.Properties.MaxUniqueItems = utils.Int64p(10000) 73 | info.Name = utils.Stringp(fmt.Sprintf("marvel")) 74 | info.Type = &typ 75 | 76 | if err := m.CreateSketch(info); err != nil { 77 | t.Error("Expected no errors, got", err) 78 | } 79 | if sketches := m.GetSketches(); len(sketches) != 1 { 80 | t.Error("Expected 1 sketches, got", len(sketches)) 81 | } else if sketches[0][0] != "marvel" || sketches[0][1] != "card" { 82 | t.Error("Expected [[marvel card]], got", sketches) 83 | } 84 | 85 | // Create a second Sketch 86 | info2 := datamodel.NewEmptyInfo() 87 | typ2 := pb.SketchType_RANK 88 | info2.Properties.MaxUniqueItems = utils.Int64p(10000) 89 | info2.Name = utils.Stringp(fmt.Sprintf("marvel")) 90 | info2.Type = &typ2 91 | 92 | if err := m.CreateSketch(info2); err != nil { 93 | t.Error("Expected no errors, got", err) 94 | } 95 | if sketches := m.GetSketches(); len(sketches) != 2 { 96 | t.Error("Expected 2 sketches, got", len(sketches)) 97 | } else if sketches[0][0] != "marvel" || sketches[0][1] != "card" { 98 | t.Error("Expected [[marvel card]], got", sketches[0][0], sketches[0][1]) 99 | } else if sketches[1][0] != "marvel" || sketches[1][1] != "rank" { 100 | t.Error("Expected [[marvel rank]], got", sketches[1][0], sketches[1][1]) 101 | } 102 | 103 | } 104 | 105 | func TestCreateDuplicateSketch(t *testing.T) { 106 | config.Reset() 107 | testutils.SetupTests() 108 | defer testutils.TearDownTests() 109 | 110 | m := NewManager() 111 | info := datamodel.NewEmptyInfo() 112 | typ := pb.SketchType_CARD 113 | info.Properties.MaxUniqueItems = utils.Int64p(10000) 114 | info.Name = utils.Stringp(fmt.Sprintf("marvel")) 115 | info.Type = &typ 116 | 117 | if err := m.CreateSketch(info); err != nil { 118 | t.Error("Expected no errors, got", err) 119 | } 120 | if err := m.CreateSketch(info); err == nil { 121 | t.Error("Expected error (duplicate sketch), got", err) 122 | } 123 | if sketches := m.GetSketches(); len(sketches) != 1 { 124 | t.Error("Expected 1 sketches, got", len(sketches)) 125 | } else if sketches[0][0] != "marvel" || sketches[0][1] != "card" { 126 | t.Error("Expected [[marvel card]], got", sketches) 127 | } 128 | } 129 | 130 | func TestCreateInvalidSketch(t *testing.T) { 131 | config.Reset() 132 | testutils.SetupTests() 133 | defer testutils.TearDownTests() 134 | 135 | m := NewManager() 136 | info := datamodel.NewEmptyInfo() 137 | info.Properties.MaxUniqueItems = utils.Int64p(10000) 138 | info.Name = utils.Stringp("avengers") 139 | info.Type = nil 140 | if err := m.CreateSketch(info); err == nil { 141 | t.Error("Expected error invalid sketch, got", err) 142 | } 143 | if sketches := m.GetSketches(); len(sketches) != 0 { 144 | t.Error("Expected 0 sketches, got", len(sketches)) 145 | } 146 | } 147 | 148 | func TestDeleteNonExistingSketch(t *testing.T) { 149 | config.Reset() 150 | testutils.SetupTests() 151 | defer testutils.TearDownTests() 152 | 153 | m := NewManager() 154 | info := datamodel.NewEmptyInfo() 155 | typ := pb.SketchType_CARD 156 | info.Properties.MaxUniqueItems = utils.Int64p(10000) 157 | info.Name = utils.Stringp(fmt.Sprintf("marvel")) 158 | info.Type = &typ 159 | 160 | if err := m.DeleteSketch(info.ID()); err == nil { 161 | t.Error("Expected errors deleting non-existing sketch, got", err) 162 | } 163 | if sketches := m.GetSketches(); len(sketches) != 0 { 164 | t.Error("Expected 0 sketches, got", len(sketches)) 165 | } 166 | } 167 | 168 | func TestDeleteSketch(t *testing.T) { 169 | config.Reset() 170 | testutils.SetupTests() 171 | defer testutils.TearDownTests() 172 | 173 | m := NewManager() 174 | info := datamodel.NewEmptyInfo() 175 | typ := pb.SketchType_CARD 176 | info.Properties.MaxUniqueItems = utils.Int64p(10000) 177 | info.Name = utils.Stringp(fmt.Sprintf("marvel")) 178 | info.Type = &typ 179 | 180 | if err := m.CreateSketch(info); err != nil { 181 | t.Error("Expected no errors, got", err) 182 | } 183 | if sketches := m.GetSketches(); len(sketches) != 1 { 184 | t.Error("Expected 1 sketches, got", len(sketches)) 185 | } else if sketches[0][0] != "marvel" || sketches[0][1] != "card" { 186 | t.Error("Expected [[marvel card]], got", sketches) 187 | } 188 | if err := m.DeleteSketch(info.ID()); err != nil { 189 | t.Error("Expected no errors, got", err) 190 | } 191 | if sketches := m.GetSketches(); len(sketches) != 0 { 192 | t.Error("Expected 0 sketches, got", len(sketches)) 193 | } 194 | } 195 | 196 | func TestCardSaveLoad(t *testing.T) { 197 | config.Reset() 198 | testutils.SetupTests() 199 | defer testutils.TearDownTests() 200 | 201 | m := NewManager() 202 | info := datamodel.NewEmptyInfo() 203 | typ := pb.SketchType_CARD 204 | info.Properties.MaxUniqueItems = utils.Int64p(10000) 205 | info.Name = utils.Stringp(fmt.Sprintf("marvel")) 206 | info.Type = &typ 207 | 208 | if err := m.CreateSketch(info); err != nil { 209 | t.Error("Expected no errors, got", err) 210 | } 211 | 212 | if sketches := m.GetSketches(); len(sketches) != 1 { 213 | t.Error("Expected 1 sketch, got", len(sketches)) 214 | } else if sketches[0][0] != "marvel" || sketches[0][1] != "card" { 215 | t.Error("Expected [[marvel card]], got", sketches) 216 | } 217 | 218 | if err := m.AddToSketch(info.ID(), []string{"hulk", "thor", "iron man", "hawk-eye"}); err != nil { 219 | t.Error("Expected no errors, got", err) 220 | } 221 | 222 | if err := m.AddToSketch(info.ID(), []string{"hulk", "black widow"}); err != nil { 223 | t.Error("Expected no errors, got", err) 224 | } 225 | 226 | if res, err := m.GetFromSketch(info.ID(), nil); err != nil { 227 | t.Error("Expected no errors, got", err) 228 | } else if res.(*pb.CardinalityResult).GetCardinality() != 5 { 229 | t.Error("Expected res = 5, got", res.(*pb.CardinalityResult).GetCardinality()) 230 | } 231 | } 232 | 233 | func TestFreqSaveLoad(t *testing.T) { 234 | config.Reset() 235 | testutils.SetupTests() 236 | defer testutils.TearDownTests() 237 | 238 | m := NewManager() 239 | info := datamodel.NewEmptyInfo() 240 | typ := pb.SketchType_FREQ 241 | info.Properties.MaxUniqueItems = utils.Int64p(10000) 242 | info.Name = utils.Stringp(fmt.Sprintf("marvel")) 243 | info.Type = &typ 244 | 245 | if err := m.CreateSketch(info); err != nil { 246 | t.Error("Expected no errors, got", err) 247 | } 248 | 249 | if sketches := m.GetSketches(); len(sketches) != 1 { 250 | t.Error("Expected 1 sketch, got", len(sketches)) 251 | } else if sketches[0][0] != "marvel" || sketches[0][1] != "freq" { 252 | t.Error("Expected [[marvel freq]], got", sketches) 253 | } 254 | 255 | if err := m.AddToSketch(info.ID(), []string{"hulk", "thor", "iron man", "hawk-eye"}); err != nil { 256 | t.Error("Expected no errors, got", err) 257 | } 258 | 259 | if err := m.AddToSketch(info.ID(), []string{"hulk", "black widow"}); err != nil { 260 | t.Error("Expected no errors, got", err) 261 | } 262 | 263 | if res, err := m.GetFromSketch(info.ID(), []string{"hulk", "thor", "iron man", "hawk-eye"}); err != nil { 264 | t.Error("Expected no errors, got", err) 265 | } else if res.(*pb.FrequencyResult).GetFrequencies()[0].GetCount() != 2 { 266 | t.Error("Expected res = 2, got", res.(*pb.FrequencyResult).GetFrequencies()[0].GetCount()) 267 | } 268 | } 269 | 270 | func TestRankSaveLoad(t *testing.T) { 271 | config.Reset() 272 | testutils.SetupTests() 273 | defer testutils.TearDownTests() 274 | 275 | m := NewManager() 276 | info := datamodel.NewEmptyInfo() 277 | typ := pb.SketchType_RANK 278 | info.Properties.Size = utils.Int64p(10) 279 | info.Name = utils.Stringp(fmt.Sprintf("marvel")) 280 | info.Type = &typ 281 | 282 | if err := m.CreateSketch(info); err != nil { 283 | t.Error("Expected no errors, got", err) 284 | } 285 | 286 | if sketches := m.GetSketches(); len(sketches) != 1 { 287 | t.Error("Expected 1 sketch, got", len(sketches)) 288 | } else if sketches[0][0] != "marvel" || sketches[0][1] != "rank" { 289 | t.Error("Expected [[marvel rank]], got", sketches) 290 | } 291 | 292 | if err := m.AddToSketch(info.ID(), []string{"hulk", "hulk", "thor", "iron man", "hawk-eye"}); err != nil { 293 | t.Error("Expected no errors, got", err) 294 | } 295 | 296 | if err := m.AddToSketch(info.ID(), []string{"hulk", "black widow", "black widow", "black widow", "black widow"}); err != nil { 297 | t.Error("Expected no errors, got", err) 298 | } 299 | 300 | if res, err := m.GetFromSketch(info.ID(), nil); err != nil { 301 | t.Error("Expected no errors, got", err) 302 | } else if len(res.(*pb.RankingsResult).GetRankings()) != 5 { 303 | t.Error("Expected len(res) = 5, got", len(res.(*pb.RankingsResult).GetRankings())) 304 | } else if res.(*pb.RankingsResult).GetRankings()[0].GetValue() != "black widow" { 305 | t.Error("Expected 'black widow', got", res.(*pb.RankingsResult).GetRankings()[0].GetValue()) 306 | } 307 | } 308 | 309 | func TestMembershipSaveLoad(t *testing.T) { 310 | config.Reset() 311 | testutils.SetupTests() 312 | defer testutils.TearDownTests() 313 | 314 | m := NewManager() 315 | info := datamodel.NewEmptyInfo() 316 | typ := pb.SketchType_MEMB 317 | info.Properties.MaxUniqueItems = utils.Int64p(1000) 318 | info.Name = utils.Stringp(fmt.Sprintf("marvel")) 319 | info.Type = &typ 320 | 321 | if err := m.CreateSketch(info); err != nil { 322 | t.Error("Expected no errors, got", err) 323 | } 324 | 325 | if sketches := m.GetSketches(); len(sketches) != 1 { 326 | t.Error("Expected 1 sketch, got", len(sketches)) 327 | } else if sketches[0][0] != "marvel" || sketches[0][1] != "memb" { 328 | t.Error("Expected [[marvel memb]], got", sketches) 329 | } 330 | 331 | if err := m.AddToSketch(info.ID(), []string{"hulk", "hulk", "thor", "iron man", "hawk-eye"}); err != nil { 332 | t.Error("Expected no errors, got", err) 333 | } 334 | 335 | if err := m.AddToSketch(info.ID(), []string{"hulk", "black widow", "black widow", "black widow", "black widow"}); err != nil { 336 | t.Error("Expected no errors, got", err) 337 | } 338 | 339 | if res, err := m.GetFromSketch(info.ID(), []string{"hulk", "captian america", "black widow"}); err != nil { 340 | t.Error("Expected no errors, got", err) 341 | } else if len(res.(*pb.MembershipResult).GetMemberships()) != 3 { 342 | t.Error("Expected len(res) = 3, got", len(res.(*pb.MembershipResult).GetMemberships())) 343 | } else if v := res.(*pb.MembershipResult).GetMemberships()[0].GetIsMember(); !v { 344 | t.Error("Expected 'hulk' == true , got", v) 345 | } else if v := res.(*pb.MembershipResult).GetMemberships()[1].GetIsMember(); v { 346 | t.Error("Expected 'captian america' == false , got", v) 347 | } else if v := res.(*pb.MembershipResult).GetMemberships()[2].GetIsMember(); !v { 348 | t.Error("Expected 'captian america' == true , got", v) 349 | } 350 | } 351 | 352 | func TestCreateDeleteDomain(t *testing.T) { 353 | config.Reset() 354 | testutils.SetupTests() 355 | defer testutils.TearDownTests() 356 | 357 | m := NewManager() 358 | info := datamodel.NewEmptyInfo() 359 | info.Properties.MaxUniqueItems = utils.Int64p(10000) 360 | info.Properties.Size = utils.Int64p(10000) 361 | info.Name = utils.Stringp(fmt.Sprintf("marvel")) 362 | 363 | if err := m.CreateDomain(info); err != nil { 364 | t.Error("Expected no errors, got", err) 365 | } 366 | if sketches := m.GetSketches(); len(sketches) != 4 { 367 | t.Error("Expected 1 sketches, got", len(sketches)) 368 | } else if sketches[0][0] != "marvel" || sketches[0][1] != "card" { 369 | t.Error("Expected [[marvel card]], got", sketches) 370 | } 371 | 372 | // Create a second Sketch 373 | info2 := datamodel.NewEmptyInfo() 374 | info2.Properties.MaxUniqueItems = utils.Int64p(10000) 375 | info2.Name = utils.Stringp("dc") 376 | if err := m.CreateDomain(info2); err != nil { 377 | t.Error("Expected no errors, got", err) 378 | } 379 | if sketches := m.GetSketches(); len(sketches) != 8 { 380 | t.Error("Expected 8 sketches, got", len(sketches)) 381 | } else if sketches[0][0] != "dc" || sketches[0][1] != "card" { 382 | t.Error("Expected [[dc card]], got", sketches[0][0], sketches[0][1]) 383 | } else if sketches[1][0] != "dc" || sketches[1][1] != "freq" { 384 | t.Error("Expected [[dc freq]], got", sketches[1][0], sketches[1][1]) 385 | } 386 | } 387 | -------------------------------------------------------------------------------- /src/manager/sketch.go: -------------------------------------------------------------------------------- 1 | package manager 2 | 3 | import ( 4 | "fmt" 5 | 6 | "datamodel" 7 | "sketches" 8 | ) 9 | 10 | type sketchManager struct { 11 | sketches map[string]*sketches.SketchProxy 12 | } 13 | 14 | func newSketchManager() *sketchManager { 15 | return &sketchManager{ 16 | sketches: make(map[string]*sketches.SketchProxy), 17 | } 18 | } 19 | 20 | // CreateSketch ... 21 | func (m *sketchManager) create(info *datamodel.Info) error { 22 | sketch, err := sketches.CreateSketch(info) 23 | if err != nil { 24 | return err 25 | } 26 | m.sketches[info.ID()] = sketch 27 | return nil 28 | } 29 | 30 | func (m *sketchManager) add(id string, values []string) error { 31 | sketch, ok := m.sketches[id] 32 | if !ok { 33 | return fmt.Errorf(`Sketch "%s" does not exists`, id) 34 | } 35 | if sketch.Locked() { 36 | // Append to File here 37 | return nil //&lockedError{} 38 | } 39 | 40 | byts := make([][]byte, len(values), len(values)) 41 | for i, v := range values { 42 | byts[i] = []byte(v) 43 | } 44 | // FIXME: return if adding was successful or not 45 | _, err := sketch.Add(byts) 46 | return err 47 | } 48 | 49 | func (m *sketchManager) delete(id string) error { 50 | if _, ok := m.sketches[id]; !ok { 51 | return fmt.Errorf(`Sketch "%s" does not exists`, id) 52 | } 53 | delete(m.sketches, id) 54 | return nil 55 | } 56 | 57 | func (m *sketchManager) get(id string, data interface{}) (interface{}, error) { 58 | var values []string 59 | if data != nil { 60 | values = data.([]string) 61 | } 62 | byts := make([][]byte, len(values), len(values)) 63 | for i, v := range values { 64 | byts[i] = []byte(v) 65 | } 66 | v, ok := m.sketches[id] 67 | if !ok { 68 | return nil, fmt.Errorf("No such key %s", id) 69 | } 70 | return v.Get(byts) 71 | } 72 | -------------------------------------------------------------------------------- /src/server/domain.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "datamodel" 5 | pb "datamodel/protobuf" 6 | 7 | "storage" 8 | 9 | "golang.org/x/net/context" 10 | ) 11 | 12 | func (s *serverStruct) createDomain(ctx context.Context, in *pb.Domain) (*pb.Domain, error) { 13 | info := datamodel.NewEmptyInfo() 14 | info.Name = in.Name 15 | // FIXME: A Domain's info should have an array of properties for each Sketch (or just an array 16 | // of Sketches, like what the proto has). This is just a hack to choose the first Sketch and 17 | // use it's info for now 18 | info.Properties.MaxUniqueItems = in.GetSketches()[0].GetProperties().MaxUniqueItems 19 | info.Properties.Size = in.GetSketches()[0].GetProperties().Size 20 | if info.Properties.Size == nil || *info.Properties.Size == 0 { 21 | var defaultSize int64 = 100 22 | info.Properties.Size = &defaultSize 23 | } 24 | // FIXME: We should be passing a pb.Domain and not a datamodel.Info to manager.CreateDomain 25 | err := s.manager.CreateDomain(info) 26 | if err != nil { 27 | return nil, err 28 | } 29 | return in, nil 30 | } 31 | 32 | func (s *serverStruct) CreateDomain(ctx context.Context, in *pb.Domain) (*pb.Domain, error) { 33 | if err := s.storage.Append(storage.CreateDom, in); err != nil { 34 | return nil, err 35 | } 36 | return s.createDomain(ctx, in) 37 | } 38 | 39 | func (s *serverStruct) ListDomains(ctx context.Context, in *pb.Empty) (*pb.ListDomainsReply, error) { 40 | res := s.manager.GetDomains() 41 | names := make([]string, len(res), len(res)) 42 | for i, n := range res { 43 | names[i] = n[0] 44 | } 45 | doms := &pb.ListDomainsReply{ 46 | Names: names, 47 | } 48 | return doms, nil 49 | } 50 | 51 | func (s *serverStruct) deleteDomain(ctx context.Context, in *pb.Domain) (*pb.Empty, error) { 52 | return &pb.Empty{}, s.manager.DeleteDomain(in.GetName()) 53 | } 54 | 55 | func (s *serverStruct) DeleteDomain(ctx context.Context, in *pb.Domain) (*pb.Empty, error) { 56 | if err := s.storage.Append(storage.DeleteDom, in); err != nil { 57 | return nil, err 58 | } 59 | return s.deleteDomain(ctx, in) 60 | } 61 | 62 | func (s *serverStruct) GetDomain(ctx context.Context, in *pb.Domain) (*pb.Domain, error) { 63 | return s.manager.GetDomain(in.GetName()) 64 | } 65 | -------------------------------------------------------------------------------- /src/server/global.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | pb "datamodel/protobuf" 5 | "errors" 6 | 7 | "golang.org/x/net/context" 8 | ) 9 | 10 | func (s *serverStruct) CreateSnapshot(ctx context.Context, in *pb.CreateSnapshotRequest) (*pb.CreateSnapshotReply, error) { 11 | //err := s.manager.Save() 12 | // FIXME: return a snapshot ID 13 | status := pb.SnapshotStatus_FAILED 14 | return &pb.CreateSnapshotReply{Status: &status}, errors.New("Snapshots not supported (yet)!") 15 | } 16 | -------------------------------------------------------------------------------- /src/server/server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "path/filepath" 7 | "runtime" 8 | 9 | "github.com/golang/protobuf/proto" 10 | "google.golang.org/grpc" 11 | 12 | pb "datamodel/protobuf" 13 | "manager" 14 | "storage" 15 | "utils" 16 | 17 | "golang.org/x/net/context" 18 | ) 19 | 20 | type serverStruct struct { 21 | manager *manager.Manager 22 | g *grpc.Server 23 | storage *storage.AOF 24 | } 25 | 26 | var server *serverStruct 27 | 28 | // Run ... 29 | func Run(manager *manager.Manager, host string, port int, datadir string) { 30 | nCPU := runtime.NumCPU() 31 | runtime.GOMAXPROCS(nCPU) 32 | path := filepath.Join(datadir, "skizze.aof") 33 | aof := storage.NewAOF(path) 34 | 35 | lis, err := net.Listen("tcp", fmt.Sprintf("%s:%d", host, port)) // RPC port 36 | if err != nil { 37 | logger.Criticalf("failed to listen: %v", err) 38 | } 39 | g := grpc.NewServer() 40 | 41 | server = &serverStruct{manager, g, aof} 42 | pb.RegisterSkizzeServer(g, server) 43 | server.replay() 44 | aof.Run() 45 | _ = g.Serve(lis) 46 | } 47 | 48 | func unmarshalSketch(e *storage.Entry) *pb.Sketch { 49 | sketch := &pb.Sketch{} 50 | err := proto.Unmarshal(e.RawMsg(), sketch) 51 | utils.PanicOnError(err) 52 | return sketch 53 | } 54 | 55 | func unmarshalDom(e *storage.Entry) *pb.Domain { 56 | dom := &pb.Domain{} 57 | err := proto.Unmarshal(e.RawMsg(), dom) 58 | utils.PanicOnError(err) 59 | return dom 60 | } 61 | 62 | func (server *serverStruct) replay() { 63 | logger.Infof("Replaying ...") 64 | for { 65 | e, err := server.storage.Read() 66 | if err != nil && err.Error() == "EOF" { 67 | break 68 | } else { 69 | utils.PanicOnError(err) 70 | } 71 | 72 | switch e.OpType() { 73 | case storage.Add: 74 | req := &pb.AddRequest{} 75 | err = proto.Unmarshal(e.RawMsg(), req) 76 | utils.PanicOnError(err) 77 | if _, err := server.add(context.Background(), req); err != nil { 78 | logger.Errorf("an error has occurred while replaying: %s", err.Error()) 79 | } 80 | case storage.CreateSketch: 81 | sketch := unmarshalSketch(e) 82 | if _, err := server.createSketch(context.Background(), sketch); err != nil { 83 | logger.Errorf("an error has occurred while replaying: %s", err.Error()) 84 | } 85 | case storage.DeleteSketch: 86 | sketch := unmarshalSketch(e) 87 | if _, err := server.deleteSketch(context.Background(), sketch); err != nil { 88 | logger.Errorf("an error has occurred while replaying: %s", err.Error()) 89 | } 90 | case storage.CreateDom: 91 | dom := unmarshalDom(e) 92 | if _, err := server.createDomain(context.Background(), dom); err != nil { 93 | logger.Errorf("an error has occurred while replaying: %s", err.Error()) 94 | } 95 | case storage.DeleteDom: 96 | dom := unmarshalDom(e) 97 | if _, err := server.deleteDomain(context.Background(), dom); err != nil { 98 | logger.Errorf("an error has occurred while replaying: %s", err.Error()) 99 | } 100 | default: 101 | continue 102 | } 103 | } 104 | } 105 | 106 | // Stop ... 107 | func Stop() { 108 | server.g.Stop() 109 | } 110 | -------------------------------------------------------------------------------- /src/server/server_test.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "time" 5 | "config" 6 | 7 | "google.golang.org/grpc" 8 | 9 | pb "datamodel/protobuf" 10 | "manager" 11 | "testutils" 12 | ) 13 | 14 | 15 | func setupClient() (pb.SkizzeClient, *grpc.ClientConn) { 16 | testutils.SetupTests() 17 | m := manager.NewManager() 18 | datadir := config.DataDir 19 | go Run(m, "127.0.0.1", 7777, datadir) 20 | time.Sleep(time.Millisecond * 50) 21 | 22 | // Connect to the server. 23 | conn, err := grpc.Dial("127.0.0.1:7777", grpc.WithInsecure()) 24 | if err != nil { 25 | logger.Criticalf("fail to dial: %v", err) 26 | } 27 | return pb.NewSkizzeClient(conn), conn 28 | } 29 | 30 | func tearDownClient(conn *grpc.ClientConn) { 31 | _ = conn.Close() 32 | Stop() 33 | testutils.TearDownTests() 34 | } 35 | -------------------------------------------------------------------------------- /src/server/sketch.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "datamodel" 5 | pb "datamodel/protobuf" 6 | "storage" 7 | 8 | "github.com/gogo/protobuf/proto" 9 | "github.com/njpatel/loggo" 10 | "golang.org/x/net/context" 11 | ) 12 | 13 | var logger = loggo.GetLogger("server") 14 | 15 | func (s *serverStruct) createSketch(ctx context.Context, in *pb.Sketch) (*pb.Sketch, error) { 16 | info := &datamodel.Info{Sketch: in} 17 | if err := s.manager.CreateSketch(info); err != nil { 18 | return nil, err 19 | } 20 | return in, nil 21 | } 22 | 23 | func (s *serverStruct) CreateSketch(ctx context.Context, in *pb.Sketch) (*pb.Sketch, error) { 24 | if err := s.storage.Append(storage.CreateSketch, in); err != nil { 25 | return nil, err 26 | } 27 | return s.createSketch(ctx, in) 28 | } 29 | 30 | func (s *serverStruct) add(ctx context.Context, in *pb.AddRequest) (*pb.AddReply, error) { 31 | info := datamodel.NewEmptyInfo() 32 | // FIXME: use domain or sketch directly and stop casting to Info 33 | if dom := in.GetDomain(); dom != nil { 34 | info.Name = dom.Name 35 | err := s.manager.AddToDomain(info.GetName(), in.GetValues()) 36 | if err != nil { 37 | return nil, err 38 | } 39 | } else if sketch := in.GetSketch(); sketch != nil { 40 | info := &datamodel.Info{Sketch: sketch} 41 | err := s.manager.AddToSketch(info.ID(), in.GetValues()) 42 | if err != nil { 43 | return nil, err 44 | } 45 | } 46 | return &pb.AddReply{}, nil 47 | } 48 | 49 | func (s *serverStruct) Add(ctx context.Context, in *pb.AddRequest) (*pb.AddReply, error) { 50 | if err := s.storage.Append(storage.Add, in); err != nil { 51 | return nil, err 52 | } 53 | return s.add(ctx, in) 54 | } 55 | 56 | func (s *serverStruct) GetMembership(ctx context.Context, in *pb.GetRequest) (*pb.GetMembershipReply, error) { 57 | reply := &pb.GetMembershipReply{} 58 | 59 | for _, sketch := range in.GetSketches() { 60 | info := &datamodel.Info{Sketch: sketch} 61 | res, err := s.manager.GetFromSketch(info.ID(), in.GetValues()) 62 | if err != nil { 63 | return nil, err 64 | } 65 | reply.Results = append(reply.Results, res.(*pb.MembershipResult)) 66 | } 67 | return reply, nil 68 | } 69 | 70 | func (s *serverStruct) GetFrequency(ctx context.Context, in *pb.GetRequest) (*pb.GetFrequencyReply, error) { 71 | reply := &pb.GetFrequencyReply{} 72 | 73 | for _, sketch := range in.GetSketches() { 74 | info := &datamodel.Info{Sketch: sketch} 75 | res, err := s.manager.GetFromSketch(info.ID(), in.GetValues()) 76 | if err != nil { 77 | return nil, err 78 | } 79 | reply.Results = append(reply.Results, res.(*pb.FrequencyResult)) 80 | } 81 | return reply, nil 82 | } 83 | 84 | func (s *serverStruct) GetCardinality(ctx context.Context, in *pb.GetRequest) (*pb.GetCardinalityReply, error) { 85 | reply := &pb.GetCardinalityReply{} 86 | 87 | for _, sketch := range in.GetSketches() { 88 | info := &datamodel.Info{Sketch: sketch} 89 | res, err := s.manager.GetFromSketch(info.ID(), in.GetValues()) 90 | if err != nil { 91 | return nil, err 92 | } 93 | reply.Results = append(reply.Results, res.(*pb.CardinalityResult)) 94 | } 95 | return reply, nil 96 | } 97 | 98 | func (s *serverStruct) GetRankings(ctx context.Context, in *pb.GetRequest) (*pb.GetRankingsReply, error) { 99 | reply := &pb.GetRankingsReply{} 100 | 101 | for _, sketch := range in.GetSketches() { 102 | info := &datamodel.Info{Sketch: sketch} 103 | 104 | res, err := s.manager.GetFromSketch(info.ID(), in.GetValues()) 105 | if err != nil { 106 | return nil, err 107 | } 108 | reply.Results = append(reply.Results, res.(*pb.RankingsResult)) 109 | } 110 | return reply, nil 111 | } 112 | 113 | func (s *serverStruct) deleteSketch(ctx context.Context, in *pb.Sketch) (*pb.Empty, error) { 114 | info := &datamodel.Info{Sketch: in} 115 | return &pb.Empty{}, s.manager.DeleteSketch(info.ID()) 116 | } 117 | 118 | func (s *serverStruct) DeleteSketch(ctx context.Context, in *pb.Sketch) (*pb.Empty, error) { 119 | if err := s.storage.Append(storage.DeleteSketch, in); err != nil { 120 | logger.Errorf("an error has occurred while deleting a sketch: %s", err.Error()) 121 | } 122 | return s.deleteSketch(ctx, in) 123 | } 124 | 125 | func (s *serverStruct) ListAll(ctx context.Context, in *pb.Empty) (*pb.ListReply, error) { 126 | sketches := s.manager.GetSketches() 127 | filtered := &pb.ListReply{} 128 | for _, v := range sketches { 129 | var typ pb.SketchType 130 | switch v[1] { 131 | case datamodel.CML: 132 | typ = pb.SketchType_FREQ 133 | case datamodel.TopK: 134 | typ = pb.SketchType_RANK 135 | case datamodel.HLLPP: 136 | typ = pb.SketchType_CARD 137 | case datamodel.Bloom: 138 | typ = pb.SketchType_MEMB 139 | default: 140 | continue 141 | } 142 | filtered.Sketches = append(filtered.Sketches, &pb.Sketch{Name: proto.String(v[0]), Type: &typ}) 143 | } 144 | return filtered, nil 145 | } 146 | 147 | func (s *serverStruct) GetSketch(ctx context.Context, in *pb.Sketch) (*pb.Sketch, error) { 148 | var err error 149 | info := &datamodel.Info{Sketch: in} 150 | info, err = s.manager.GetSketch(info.ID()) 151 | return in, err 152 | } 153 | 154 | func (s *serverStruct) List(ctx context.Context, in *pb.ListRequest) (*pb.ListReply, error) { 155 | sketches := s.manager.GetSketches() 156 | filtered := &pb.ListReply{} 157 | for _, v := range sketches { 158 | var typ pb.SketchType 159 | switch v[1] { 160 | case datamodel.CML: 161 | typ = pb.SketchType_FREQ 162 | case datamodel.TopK: 163 | typ = pb.SketchType_RANK 164 | case datamodel.HLLPP: 165 | typ = pb.SketchType_CARD 166 | case datamodel.Bloom: 167 | typ = pb.SketchType_MEMB 168 | default: 169 | continue 170 | } 171 | if in.GetType() == typ { 172 | filtered.Sketches = append(filtered.Sketches, &pb.Sketch{Name: proto.String(v[0]), Type: &typ}) 173 | } 174 | } 175 | return filtered, nil 176 | } 177 | -------------------------------------------------------------------------------- /src/server/sketch_test.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/gogo/protobuf/proto" 7 | "golang.org/x/net/context" 8 | 9 | "config" 10 | pb "datamodel/protobuf" 11 | "testutils" 12 | ) 13 | 14 | func TestCreateSketch(t *testing.T) { 15 | config.Reset() 16 | testutils.SetupTests() 17 | defer testutils.TearDownTests() 18 | 19 | client, conn := setupClient() 20 | defer tearDownClient(conn) 21 | 22 | typ := pb.SketchType_CARD 23 | name := "yoyo" 24 | 25 | in := &pb.Sketch{ 26 | Name: proto.String(name), 27 | Type: &typ, 28 | } 29 | 30 | if res, err := client.CreateSketch(context.Background(), in); err != nil { 31 | t.Error("Did not expect error, got", err) 32 | } else if res.GetName() != in.GetName() { 33 | t.Errorf("Expected name == %s, got %s", in.GetName(), res.GetName()) 34 | } else if res.GetType() != in.GetType() { 35 | t.Errorf("Expected name == %q, got %q", in.GetType(), res.GetType()) 36 | } 37 | } 38 | 39 | func TestCreateAddInvalidSketch(t *testing.T) { 40 | config.Reset() 41 | testutils.SetupTests() 42 | defer testutils.TearDownTests() 43 | 44 | client, conn := setupClient() 45 | defer tearDownClient(conn) 46 | 47 | typ := pb.SketchType_CARD 48 | name := "yoyo" 49 | 50 | in := &pb.Sketch{ 51 | Name: proto.String(name), 52 | Type: &typ, 53 | Properties: &pb.SketchProperties{ 54 | MaxUniqueItems: proto.Int64(1337), // FIXME: Allow default as -1 55 | Size: proto.Int64(7), 56 | }, 57 | } 58 | 59 | if res, err := client.CreateSketch(context.Background(), in); err != nil { 60 | t.Error("Did not expect error, got", err) 61 | } else if res.GetName() != in.GetName() { 62 | t.Errorf("Expected name == %s, got %s", in.GetName(), res.GetName()) 63 | } else if res.GetType() != in.GetType() { 64 | t.Errorf("Expected name == %q, got %q", in.GetType(), res.GetType()) 65 | } 66 | 67 | typ = pb.SketchType_FREQ 68 | in.Type = &typ 69 | addReq := &pb.AddRequest{ 70 | Sketch: in, 71 | Values: []string{"a", "b", "c", "d", "a", "b"}, 72 | } 73 | 74 | if _, err := client.Add(context.Background(), addReq); err == nil { 75 | t.Error("Expect error, got", err) 76 | } 77 | 78 | } 79 | 80 | func TestCreateAddDeleteAddSketch(t *testing.T) { 81 | config.Reset() 82 | testutils.SetupTests() 83 | defer testutils.TearDownTests() 84 | 85 | client, conn := setupClient() 86 | defer tearDownClient(conn) 87 | 88 | typ := pb.SketchType_CARD 89 | name := "yoyo" 90 | 91 | in := &pb.Sketch{ 92 | Name: proto.String(name), 93 | Type: &typ, 94 | Properties: &pb.SketchProperties{ 95 | MaxUniqueItems: proto.Int64(1337), // FIXME: Allow default as -1 96 | Size: proto.Int64(7), 97 | }, 98 | } 99 | 100 | addReq := &pb.AddRequest{ 101 | Sketch: in, 102 | Values: []string{"a", "b", "c", "d", "a", "b"}, 103 | } 104 | 105 | if _, err := client.CreateSketch(context.Background(), in); err != nil { 106 | t.Error("Did not expect error, got", err) 107 | } 108 | 109 | typ = pb.SketchType_RANK 110 | if _, err := client.CreateSketch(context.Background(), in); err != nil { 111 | t.Error("Did not expect error, got", err) 112 | } 113 | 114 | if _, err := client.Add(context.Background(), addReq); err != nil { 115 | t.Error("Did not expect error, got", err) 116 | } 117 | 118 | if res, err := client.ListAll(context.Background(), &pb.Empty{}); err != nil { 119 | t.Error("Did not expect error, got", err) 120 | } else if len(res.GetSketches()) != 2 { 121 | t.Error("Expected len(res) == 2, got ", len(res.GetSketches())) 122 | } 123 | 124 | if _, err := client.DeleteSketch(context.Background(), in); err != nil { 125 | t.Error("Did not expect error, got", err) 126 | } 127 | 128 | if _, err := client.Add(context.Background(), addReq); err == nil { 129 | t.Error("Expected error, got", err) 130 | } 131 | 132 | if res, err := client.ListAll(context.Background(), &pb.Empty{}); err != nil { 133 | t.Error("Did not expect error, got", err) 134 | } else if len(res.GetSketches()) != 1 { 135 | t.Error("Expected len(res) == 1, got ", len(res.GetSketches())) 136 | } 137 | 138 | } 139 | 140 | func TestAddGetCardSketch(t *testing.T) { 141 | config.Reset() 142 | testutils.SetupTests() 143 | defer testutils.TearDownTests() 144 | 145 | client, conn := setupClient() 146 | defer tearDownClient(conn) 147 | 148 | typ := pb.SketchType_CARD 149 | name := "yoyo" 150 | 151 | in := &pb.Sketch{ 152 | Name: proto.String(name), 153 | Type: &typ, 154 | Properties: &pb.SketchProperties{ 155 | MaxUniqueItems: proto.Int64(1337), // FIXME: Allow default as -1 156 | Size: proto.Int64(7), 157 | }, 158 | } 159 | 160 | if res, err := client.CreateSketch(context.Background(), in); err != nil { 161 | t.Error("Did not expect error, got", err) 162 | } else if res.GetName() != in.GetName() { 163 | t.Errorf("Expected name == %s, got %s", in.GetName(), res.GetName()) 164 | } else if res.GetType() != in.GetType() { 165 | t.Errorf("Expected name == %q, got %q", in.GetType(), res.GetType()) 166 | } 167 | 168 | addReq := &pb.AddRequest{ 169 | Sketch: in, 170 | Values: []string{"a", "b", "c", "d", "a", "b"}, 171 | } 172 | 173 | if _, err := client.Add(context.Background(), addReq); err != nil { 174 | t.Error("Did not expect error, got", err) 175 | } 176 | 177 | getReq := &pb.GetRequest{ 178 | Sketches: []*pb.Sketch{in}, 179 | Values: []string{}, 180 | } 181 | 182 | if res, err := client.GetCardinality(context.Background(), getReq); err != nil { 183 | t.Error("Did not expect error, got", err) 184 | 185 | } else if res.GetResults()[0].GetCardinality() != 4 { 186 | t.Error("Expected cardinality 4, got", res.GetResults()[0].GetCardinality()) 187 | } 188 | } 189 | 190 | func TestAddGetMembSketch(t *testing.T) { 191 | config.Reset() 192 | testutils.SetupTests() 193 | defer testutils.TearDownTests() 194 | 195 | client, conn := setupClient() 196 | defer tearDownClient(conn) 197 | 198 | typ := pb.SketchType_MEMB 199 | name := "yoyo" 200 | 201 | in := &pb.Sketch{ 202 | Name: proto.String(name), 203 | Type: &typ, 204 | Properties: &pb.SketchProperties{ 205 | MaxUniqueItems: proto.Int64(1337), // FIXME: Allow default as -1 206 | Size: proto.Int64(7), 207 | }, 208 | } 209 | if res, err := client.CreateSketch(context.Background(), in); err != nil { 210 | t.Error("Did not expect error, got", err) 211 | } else if res.GetName() != in.GetName() { 212 | t.Errorf("Expected name == %s, got %s", in.GetName(), res.GetName()) 213 | } else if res.GetType() != in.GetType() { 214 | t.Errorf("Expected name == %q, got %q", in.GetType(), res.GetType()) 215 | } 216 | 217 | addReq := &pb.AddRequest{ 218 | Sketch: in, 219 | Values: []string{"a", "a", "b", "c", "d"}, 220 | } 221 | 222 | if _, err := client.Add(context.Background(), addReq); err != nil { 223 | t.Error("Did not expect error, got", err) 224 | } 225 | 226 | getReq := &pb.GetRequest{ 227 | Sketches: []*pb.Sketch{in}, 228 | Values: []string{"a", "b", "c", "d", "e", "b"}, 229 | } 230 | 231 | expected := map[string]bool{ 232 | "a": true, "b": true, "c": true, "d": true, "e": false, 233 | } 234 | 235 | if res, err := client.GetMembership(context.Background(), getReq); err != nil { 236 | t.Error("Did not expect error, got", err) 237 | } else { 238 | for _, v := range res.GetResults()[0].Memberships { 239 | if expected[v.GetValue()] != v.GetIsMember() { 240 | t.Errorf("Expected %s == %t, got", v.GetValue(), v.GetIsMember()) 241 | } 242 | } 243 | } 244 | } 245 | 246 | func TestAddGetFreqSketch(t *testing.T) { 247 | config.Reset() 248 | testutils.SetupTests() 249 | defer testutils.TearDownTests() 250 | 251 | client, conn := setupClient() 252 | defer tearDownClient(conn) 253 | 254 | typ := pb.SketchType_FREQ 255 | name := "yoyo" 256 | 257 | in := &pb.Sketch{ 258 | Name: proto.String(name), 259 | Type: &typ, 260 | Properties: &pb.SketchProperties{ 261 | MaxUniqueItems: proto.Int64(1337), // FIXME: Allow default as -1 262 | Size: proto.Int64(7), 263 | }, 264 | } 265 | 266 | if res, err := client.CreateSketch(context.Background(), in); err != nil { 267 | t.Error("Did not expect error, got", err) 268 | } else if res.GetName() != in.GetName() { 269 | t.Errorf("Expected name == %s, got %s", in.GetName(), res.GetName()) 270 | } else if res.GetType() != in.GetType() { 271 | t.Errorf("Expected name == %q, got %q", in.GetType(), res.GetType()) 272 | } 273 | 274 | addReq := &pb.AddRequest{ 275 | Sketch: in, 276 | Values: []string{"a", "a", "b", "c", "d"}, 277 | } 278 | 279 | expected := map[string]int64{ 280 | "a": 2, "b": 1, "c": 1, "d": 1, "e": 0, 281 | } 282 | 283 | if _, err := client.Add(context.Background(), addReq); err != nil { 284 | t.Error("Did not expect error, got", err) 285 | } 286 | 287 | getReq := &pb.GetRequest{ 288 | Sketches: []*pb.Sketch{in}, 289 | Values: []string{"a", "b", "c", "d", "e", "b"}, 290 | } 291 | 292 | if res, err := client.GetFrequency(context.Background(), getReq); err != nil { 293 | t.Error("Did not expect error, got", err) 294 | } else { 295 | for _, v := range res.GetResults()[0].GetFrequencies() { 296 | if expected[v.GetValue()] != v.GetCount() { 297 | t.Errorf("Expected %s == %d, got", v.GetValue(), v.GetCount()) 298 | } 299 | } 300 | } 301 | } 302 | 303 | func TestAddGetRankSketch(t *testing.T) { 304 | config.Reset() 305 | testutils.SetupTests() 306 | defer testutils.TearDownTests() 307 | 308 | client, conn := setupClient() 309 | defer tearDownClient(conn) 310 | 311 | typ := pb.SketchType_RANK 312 | name := "yoyo" 313 | 314 | in := &pb.Sketch{ 315 | Name: proto.String(name), 316 | Type: &typ, 317 | Properties: &pb.SketchProperties{ 318 | MaxUniqueItems: proto.Int64(1337), // FIXME: Allow default as -1 319 | Size: proto.Int64(7), 320 | }, 321 | } 322 | 323 | if res, err := client.CreateSketch(context.Background(), in); err != nil { 324 | t.Error("Did not expect error, got", err) 325 | } else if res.GetName() != in.GetName() { 326 | t.Errorf("Expected name == %s, got %s", in.GetName(), res.GetName()) 327 | } else if res.GetType() != in.GetType() { 328 | t.Errorf("Expected name == %q, got %q", in.GetType(), res.GetType()) 329 | } 330 | 331 | addReq := &pb.AddRequest{ 332 | Sketch: in, 333 | Values: []string{"a", "a", "b", "c", "d", "a", "b", "a", "b", "c"}, 334 | } 335 | 336 | expected := map[string]int64{ 337 | "a": 4, "b": 3, "c": 2, "d": 1, "e": 0, 338 | } 339 | 340 | if _, err := client.Add(context.Background(), addReq); err != nil { 341 | t.Error("Did not expect error, got", err) 342 | } 343 | 344 | getReq := &pb.GetRequest{ 345 | Sketches: []*pb.Sketch{in}, 346 | } 347 | 348 | if res, err := client.GetRankings(context.Background(), getReq); err != nil { 349 | t.Error("Did not expect error, got", err) 350 | } else { 351 | for _, v := range res.GetResults()[0].GetRankings() { 352 | if expected[v.GetValue()] != v.GetCount() { 353 | t.Errorf("Expected %s == %d, got", v.GetValue(), v.GetCount()) 354 | } 355 | } 356 | } 357 | } 358 | -------------------------------------------------------------------------------- /src/server/todo.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "golang.org/x/net/context" 5 | 6 | pb "datamodel/protobuf" 7 | ) 8 | 9 | func (s *serverStruct) GetSnapshot(ctx context.Context, in *pb.GetSnapshotRequest) (*pb.GetSnapshotReply, error) { 10 | return nil, nil 11 | } 12 | -------------------------------------------------------------------------------- /src/sketches/bloom.go: -------------------------------------------------------------------------------- 1 | package sketches 2 | 3 | import ( 4 | bloom "github.com/AndreasBriese/bbloom" 5 | 6 | "datamodel" 7 | pb "datamodel/protobuf" 8 | "utils" 9 | ) 10 | 11 | // BloomSketch is the toplevel Sketch to control the count-min-log implementation 12 | type BloomSketch struct { 13 | *datamodel.Info 14 | impl *bloom.Bloom 15 | } 16 | 17 | // NewBloomSketch ... 18 | func NewBloomSketch(info *datamodel.Info) (*BloomSketch, error) { 19 | // FIXME: We are converting from int64 to uint 20 | sketch := bloom.New(float64(info.Properties.GetMaxUniqueItems()), 4.0) 21 | d := BloomSketch{info, &sketch} 22 | return &d, nil 23 | } 24 | 25 | // Add ... 26 | func (d *BloomSketch) Add(values [][]byte) (bool, error) { 27 | success := true 28 | dict := make(map[string]uint) 29 | for _, v := range values { 30 | dict[string(v)]++ 31 | } 32 | for v := range dict { 33 | d.impl.Add([]byte(v)) 34 | } 35 | // Fixme: return what was added and what not 36 | return success, nil 37 | } 38 | 39 | // Get ... 40 | func (d *BloomSketch) Get(data interface{}) (interface{}, error) { 41 | values := data.([][]byte) 42 | tmpRes := make(map[string]*pb.Membership) 43 | res := &pb.MembershipResult{ 44 | Memberships: make([]*pb.Membership, len(values), len(values)), 45 | } 46 | for i, v := range values { 47 | if r, ok := tmpRes[string(v)]; ok { 48 | res.Memberships[i] = r 49 | continue 50 | } 51 | res.Memberships[i] = &pb.Membership{ 52 | Value: utils.Stringp(string(v)), 53 | IsMember: utils.Boolp(d.impl.Has(v)), 54 | } 55 | tmpRes[string(v)] = res.Memberships[i] 56 | } 57 | return res, nil 58 | } 59 | -------------------------------------------------------------------------------- /src/sketches/bloom_test.go: -------------------------------------------------------------------------------- 1 | package sketches 2 | 3 | import ( 4 | "strconv" 5 | "testing" 6 | 7 | "datamodel" 8 | pb "datamodel/protobuf" 9 | "utils" 10 | "testutils" 11 | ) 12 | 13 | func TestAddBloom(t *testing.T) { 14 | testutils.SetupTests() 15 | defer testutils.TearDownTests() 16 | 17 | info := datamodel.NewEmptyInfo() 18 | info.Properties.MaxUniqueItems = utils.Int64p(1024) 19 | info.Name = utils.Stringp("marvel") 20 | sketch, err := NewBloomSketch(info) 21 | 22 | if err != nil { 23 | t.Error("expected avengers to have no error, got", err) 24 | } 25 | 26 | values := [][]byte{ 27 | []byte("sabertooth"), 28 | []byte("thunderbolt"), 29 | []byte("havoc"), 30 | []byte("cyclops"), 31 | []byte("cyclops"), 32 | []byte("cyclops"), 33 | []byte("havoc")} 34 | 35 | if _, err := sketch.Add(values); err != nil { 36 | t.Error("expected no errors, got", err) 37 | } 38 | 39 | check := map[string]bool{ 40 | "sabertooth": true, 41 | "thunderbolt": true, 42 | "havoc": true, 43 | "cyclops": true, 44 | "wolverine": false, 45 | "iceman": false, 46 | "rogue": false, 47 | "storm": false} 48 | 49 | if res, err := sketch.Get(values); err != nil { 50 | t.Error("expected no errors, got", err) 51 | } else { 52 | tmp := res.(*pb.MembershipResult) 53 | mres := tmp.GetMemberships() 54 | for key := range check { 55 | for i := 0; i < len(mres); i++ { 56 | if mres[i].GetValue() == key && 57 | mres[i].GetIsMember() != check[key] { 58 | t.Error("expected member == "+strconv.FormatBool(check[key])+", got", mres[i].GetIsMember()) 59 | } 60 | } 61 | } 62 | } 63 | } 64 | 65 | func TestStressBloom(t *testing.T) { 66 | testutils.SetupTests() 67 | defer testutils.TearDownTests() 68 | 69 | values := make([][]byte, 10) 70 | for i := 0; i < 1024; i++ { 71 | avenger := "avenger" + strconv.Itoa(i) 72 | values = append(values, []byte(avenger)) 73 | } 74 | 75 | info := datamodel.NewEmptyInfo() 76 | info.Properties.MaxUniqueItems = utils.Int64p(1024) 77 | info.Name = utils.Stringp("marvel") 78 | sketch, err := NewBloomSketch(info) 79 | 80 | for i := 0; i < 1024; i++ { 81 | if err != nil { 82 | t.Error("expected avengers to have no error, got", err) 83 | } 84 | 85 | if _, err := sketch.Add(values); err != nil { 86 | t.Error("expected no errors, got", err) 87 | } 88 | } 89 | } 90 | 91 | func BenchmarkBloom(b *testing.B) { 92 | testutils.SetupTests() 93 | defer testutils.TearDownTests() 94 | values := make([][]byte, 10) 95 | for i := 0; i < 1024; i++ { 96 | avenger := "avenger" + strconv.Itoa(i) 97 | values = append(values, []byte(avenger)) 98 | } 99 | for n := 0; n < b.N; n++ { 100 | info := datamodel.NewEmptyInfo() 101 | info.Properties.MaxUniqueItems = utils.Int64p(1000) 102 | info.Name = utils.Stringp("marvel") 103 | sketch, err := NewBloomSketch(info) 104 | if err != nil { 105 | b.Error("expected no errors, got", err) 106 | } 107 | for i := 0; i < 1000; i++ { 108 | if _, err := sketch.Add(values); err != nil { 109 | b.Error("expected no errors, got", err) 110 | } 111 | } 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /src/sketches/cml.go: -------------------------------------------------------------------------------- 1 | package sketches 2 | 3 | import ( 4 | "github.com/skizzehq/count-min-log" 5 | 6 | "datamodel" 7 | pb "datamodel/protobuf" 8 | "utils" 9 | ) 10 | 11 | // CMLSketch is the toplevel Sketch to control the count-min-log implementation 12 | type CMLSketch struct { 13 | *datamodel.Info 14 | impl *cml.Sketch 15 | } 16 | 17 | // NewCMLSketch ... 18 | func NewCMLSketch(info *datamodel.Info) (*CMLSketch, error) { 19 | sketch, err := cml.NewForCapacity16(uint64(info.Properties.GetMaxUniqueItems()), 0.01) 20 | d := CMLSketch{info, sketch} 21 | if err != nil { 22 | logger.Errorf("an error has occurred while saving CMLSketch: %s", err.Error()) 23 | } 24 | return &d, nil 25 | } 26 | 27 | // Add ... 28 | func (d *CMLSketch) Add(values [][]byte) (bool, error) { 29 | success := true 30 | 31 | dict := make(map[string]uint) 32 | for _, v := range values { 33 | dict[string(v)]++ 34 | } 35 | for v, count := range dict { 36 | if b := d.impl.BulkUpdate([]byte(v), count); !b { 37 | success = false 38 | } 39 | } 40 | return success, nil 41 | } 42 | 43 | // Get ... 44 | func (d *CMLSketch) Get(data interface{}) (interface{}, error) { 45 | values := data.([][]byte) 46 | res := &pb.FrequencyResult{ 47 | Frequencies: make([]*pb.Frequency, len(values), len(values)), 48 | } 49 | tmpRes := make(map[string]*pb.Frequency) 50 | for i, v := range values { 51 | if r, ok := tmpRes[string(v)]; ok { 52 | res.Frequencies[i] = r 53 | continue 54 | } 55 | res.Frequencies[i] = &pb.Frequency{ 56 | Value: utils.Stringp(string(v)), 57 | Count: utils.Int64p(int64(d.impl.Query(v))), 58 | } 59 | tmpRes[string(v)] = res.Frequencies[i] 60 | } 61 | return res, nil 62 | } 63 | -------------------------------------------------------------------------------- /src/sketches/cml_test.go: -------------------------------------------------------------------------------- 1 | package sketches 2 | 3 | import ( 4 | "strconv" 5 | "testing" 6 | 7 | "datamodel" 8 | pb "datamodel/protobuf" 9 | "utils" 10 | "testutils" 11 | ) 12 | 13 | func TestAdd(t *testing.T) { 14 | testutils.SetupTests() 15 | defer testutils.TearDownTests() 16 | 17 | info := datamodel.NewEmptyInfo() 18 | info.Properties.MaxUniqueItems = utils.Int64p(1000000) 19 | info.Name = utils.Stringp("marvel") 20 | sketch, err := NewCMLSketch(info) 21 | 22 | if err != nil { 23 | t.Error("expected avengers to have no error, got", err) 24 | } 25 | 26 | values := [][]byte{ 27 | []byte("sabertooth"), 28 | []byte("thunderbolt"), 29 | []byte("havoc"), 30 | []byte("cyclops"), 31 | []byte("cyclops"), 32 | []byte("cyclops"), 33 | []byte("havoc")} 34 | 35 | if _, err := sketch.Add(values); err != nil { 36 | t.Error("expected no errors, got", err) 37 | } 38 | 39 | if res, err := sketch.Get([][]byte{[]byte("cyclops")}); err != nil { 40 | t.Error("expected no errors, got", err) 41 | } else if res.(*pb.FrequencyResult).Frequencies[0].GetCount() != 3 { 42 | t.Error("expected 'cyclops' count == 3, got", res.(*pb.FrequencyResult).Frequencies[0].GetCount()) 43 | } 44 | } 45 | 46 | func BenchmarkCML(b *testing.B) { 47 | values := make([][]byte, 10) 48 | for i := 0; i < 1024; i++ { 49 | avenger := "avenger" + strconv.Itoa(i) 50 | values = append(values, []byte(avenger)) 51 | } 52 | 53 | for n := 0; n < b.N; n++ { 54 | info := datamodel.NewEmptyInfo() 55 | info.Properties.MaxUniqueItems = utils.Int64p(1000) 56 | info.Name = utils.Stringp("marvel2") 57 | sketch, err := NewCMLSketch(info) 58 | if err != nil { 59 | b.Error("expected no errors, got", err) 60 | } 61 | for i := 0; i < 1000; i++ { 62 | if _, err := sketch.Add(values); err != nil { 63 | b.Error("expected no errors, got", err) 64 | } 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/sketches/hllpp.go: -------------------------------------------------------------------------------- 1 | package sketches 2 | 3 | import ( 4 | "github.com/retailnext/hllpp" 5 | 6 | "datamodel" 7 | pb "datamodel/protobuf" 8 | "utils" 9 | ) 10 | 11 | // HLLPPSketch is the toplevel sketch to control the HLL implementation 12 | type HLLPPSketch struct { 13 | *datamodel.Info 14 | impl *hllpp.HLLPP 15 | } 16 | 17 | // NewHLLPPSketch ... 18 | func NewHLLPPSketch(info *datamodel.Info) (*HLLPPSketch, error) { 19 | d := HLLPPSketch{info, hllpp.New()} 20 | return &d, nil 21 | } 22 | 23 | // Add ... 24 | func (d *HLLPPSketch) Add(values [][]byte) (bool, error) { 25 | dict := make(map[string]uint) 26 | for _, v := range values { 27 | dict[string(v)]++ 28 | } 29 | for v := range dict { 30 | d.impl.Add([]byte(v)) 31 | } 32 | return true, nil 33 | } 34 | 35 | // Get ... 36 | func (d *HLLPPSketch) Get(interface{}) (interface{}, error) { 37 | return &pb.CardinalityResult{ 38 | Cardinality: utils.Int64p(int64(d.impl.Count())), 39 | }, nil 40 | } 41 | -------------------------------------------------------------------------------- /src/sketches/hllpp_test.go: -------------------------------------------------------------------------------- 1 | package sketches 2 | 3 | import ( 4 | "strconv" 5 | "testing" 6 | 7 | "datamodel" 8 | pb "datamodel/protobuf" 9 | "utils" 10 | "testutils" 11 | ) 12 | 13 | func TestAddHLLPP(t *testing.T) { 14 | testutils.SetupTests() 15 | defer testutils.TearDownTests() 16 | 17 | info := datamodel.NewEmptyInfo() 18 | info.Properties.MaxUniqueItems = utils.Int64p(1024) 19 | info.Name = utils.Stringp("marvel") 20 | sketch, err := NewHLLPPSketch(info) 21 | 22 | if err != nil { 23 | t.Error("expected avengers to have no error, got", err) 24 | } 25 | 26 | values := [][]byte{ 27 | []byte("sabertooth"), 28 | []byte("thunderbolt"), 29 | []byte("havoc"), 30 | []byte("cyclops"), 31 | []byte("cyclops"), 32 | []byte("cyclops"), 33 | []byte("havoc")} 34 | 35 | if _, err := sketch.Add(values); err != nil { 36 | t.Error("expected no errors, got", err) 37 | } 38 | 39 | const expectedCardinality int64 = 4 40 | 41 | if res, err := sketch.Get(values); err != nil { 42 | t.Error("expected no errors, got", err) 43 | } else { 44 | tmp := res.(*pb.CardinalityResult) 45 | mres := tmp.GetCardinality() 46 | if mres != int64(expectedCardinality) { 47 | t.Error("expected cardinality == "+strconv.FormatInt(expectedCardinality, 10)+", got", mres) 48 | } 49 | } 50 | } 51 | 52 | func BenchmarkHLLPP(b *testing.B) { 53 | values := make([][]byte, 10) 54 | for i := 0; i < 1024; i++ { 55 | avenger := "avenger" + strconv.Itoa(i) 56 | values = append(values, []byte(avenger)) 57 | } 58 | 59 | for n := 0; n < b.N; n++ { 60 | info := datamodel.NewEmptyInfo() 61 | info.Properties.Size = utils.Int64p(1000) 62 | info.Name = utils.Stringp("marvel3") 63 | sketch, err := NewHLLPPSketch(info) 64 | if err != nil { 65 | b.Error("expected no errors, got", err) 66 | } 67 | for i := 0; i < 1000; i++ { 68 | if _, err := sketch.Add(values); err != nil { 69 | b.Error("expected no errors, got", err) 70 | } 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/sketches/proxy.go: -------------------------------------------------------------------------------- 1 | package sketches 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | 7 | "github.com/njpatel/loggo" 8 | 9 | "datamodel" 10 | ) 11 | 12 | var logger = loggo.GetLogger("sketches") 13 | 14 | // SketchProxy ... 15 | type SketchProxy struct { 16 | *datamodel.Info 17 | sketch datamodel.Sketcher 18 | lock sync.RWMutex 19 | } 20 | 21 | // Add ... 22 | func (sp *SketchProxy) Add(values [][]byte) (bool, error) { 23 | sp.lock.Lock() 24 | defer sp.lock.Unlock() 25 | return sp.sketch.Add(values) 26 | } 27 | 28 | // Get ... 29 | func (sp *SketchProxy) Get(data interface{}) (interface{}, error) { 30 | switch datamodel.GetTypeString(sp.GetType()) { 31 | case datamodel.HLLPP: 32 | return sp.sketch.Get(nil) 33 | case datamodel.CML: 34 | return sp.sketch.Get(data) 35 | case datamodel.TopK: 36 | return sp.sketch.Get(nil) 37 | case datamodel.Bloom: 38 | return sp.sketch.Get(data) 39 | default: 40 | return nil, fmt.Errorf("Invalid sketch type: %s", sp.GetType()) 41 | } 42 | } 43 | 44 | // CreateSketch ... 45 | func CreateSketch(info *datamodel.Info) (*SketchProxy, error) { 46 | var err error 47 | var sketch datamodel.Sketcher 48 | sp := &SketchProxy{info, sketch, sync.RWMutex{}} 49 | 50 | switch datamodel.GetTypeString(info.GetType()) { 51 | case datamodel.HLLPP: 52 | sp.sketch, err = NewHLLPPSketch(info) 53 | case datamodel.CML: 54 | sp.sketch, err = NewCMLSketch(info) 55 | case datamodel.TopK: 56 | sp.sketch, err = NewTopKSketch(info) 57 | case datamodel.Bloom: 58 | sp.sketch, err = NewBloomSketch(info) 59 | default: 60 | return nil, fmt.Errorf("Invalid sketch type: %s", sp.GetType()) 61 | } 62 | 63 | if err != nil { 64 | return nil, err 65 | } 66 | return sp, nil 67 | } 68 | -------------------------------------------------------------------------------- /src/sketches/topk.go: -------------------------------------------------------------------------------- 1 | package sketches 2 | 3 | import ( 4 | "github.com/dgryski/go-topk" 5 | 6 | "datamodel" 7 | pb "datamodel/protobuf" 8 | 9 | "utils" 10 | ) 11 | 12 | // TopKSketch is the toplevel sketch to control the HLL implementation 13 | type TopKSketch struct { 14 | *datamodel.Info 15 | impl *topk.Stream 16 | } 17 | 18 | // ResultElement ... 19 | type ResultElement topk.Element 20 | 21 | // NewTopKSketch ... 22 | func NewTopKSketch(info *datamodel.Info) (*TopKSketch, error) { 23 | size := int(info.Properties.GetSize()) * 2 // For higher precision 24 | d := TopKSketch{info, topk.New(size)} 25 | return &d, nil 26 | } 27 | 28 | // Add ... 29 | func (d *TopKSketch) Add(values [][]byte) (bool, error) { 30 | dict := make(map[string]int) 31 | for _, v := range values { 32 | dict[string(v)]++ 33 | } 34 | for v, count := range dict { 35 | d.impl.Insert(v, count) 36 | } 37 | return true, nil 38 | } 39 | 40 | // Get ... 41 | func (d *TopKSketch) Get(interface{}) (interface{}, error) { 42 | keys := d.impl.Keys() 43 | size := len(keys) 44 | if size > int(d.Info.Properties.GetSize())/2 { 45 | size = int(d.Info.Properties.GetSize()) / 2 46 | } 47 | result := &pb.RankingsResult{ 48 | Rankings: make([]*pb.Rank, size, size), 49 | } 50 | for i := range result.Rankings { 51 | k := keys[i] 52 | result.Rankings[i] = &pb.Rank{ 53 | Value: utils.Stringp(k.Key), 54 | Count: utils.Int64p(int64(k.Count)), 55 | } 56 | } 57 | return result, nil 58 | } 59 | -------------------------------------------------------------------------------- /src/sketches/topk_test.go: -------------------------------------------------------------------------------- 1 | package sketches 2 | 3 | import ( 4 | "strconv" 5 | "testing" 6 | 7 | "datamodel" 8 | pb "datamodel/protobuf" 9 | "utils" 10 | "testutils" 11 | ) 12 | 13 | func TestAddTopK(t *testing.T) { 14 | testutils.SetupTests() 15 | defer testutils.TearDownTests() 16 | 17 | info := datamodel.NewEmptyInfo() 18 | info.Properties.Size = utils.Int64p(3) 19 | info.Name = utils.Stringp("marvel") 20 | sketch, err := NewTopKSketch(info) 21 | 22 | if err != nil { 23 | t.Error("expected avengers to have no error, got", err) 24 | } 25 | 26 | values := [][]byte{ 27 | []byte("sabertooth"), 28 | []byte("thunderbolt"), 29 | []byte("cyclops"), 30 | []byte("thunderbolt"), 31 | []byte("thunderbolt"), 32 | []byte("havoc"), 33 | []byte("cyclops"), 34 | []byte("cyclops"), 35 | []byte("cyclops"), 36 | []byte("havoc")} 37 | 38 | if _, err := sketch.Add(values); err != nil { 39 | t.Error("expected no errors, got", err) 40 | } 41 | 42 | type RankingsStruct struct { 43 | Value string 44 | Position int64 45 | Count int64 46 | } 47 | expectedRankings := make([]*RankingsStruct, 4, 4) 48 | 49 | expectedRankings[0] = &RankingsStruct{ 50 | Value: "cyclops", 51 | Count: 4, 52 | Position: 1, 53 | } 54 | 55 | expectedRankings[1] = &RankingsStruct{ 56 | Value: "thunderbolt", 57 | Count: 3, 58 | Position: 2, 59 | } 60 | 61 | expectedRankings[2] = &RankingsStruct{ 62 | Value: "havoc", 63 | Count: 2, 64 | Position: 3, 65 | } 66 | 67 | expectedRankings[3] = &RankingsStruct{ 68 | Value: "sabertooth", 69 | Count: 1, 70 | Position: 4, 71 | } 72 | 73 | if res, err := sketch.Get(values); err != nil { 74 | t.Error("expected no errors, got", err) 75 | } else { 76 | tmp := res.(*pb.RankingsResult) 77 | rres := tmp.GetRankings() 78 | for i := 0; i < len(rres); i++ { 79 | if expectedRankings[i].Value != rres[i].GetValue() { 80 | t.Errorf("expected rank of %d := %s, got %s", expectedRankings[i].Position, expectedRankings[i].Value, rres[i].GetValue()) 81 | } 82 | } 83 | } 84 | } 85 | 86 | func BenchmarkTopK(b *testing.B) { 87 | values := make([][]byte, 10) 88 | for i := 0; i < 1024; i++ { 89 | avenger := "avenger" + strconv.Itoa(i) 90 | values = append(values, []byte(avenger)) 91 | } 92 | 93 | for n := 0; n < b.N; n++ { 94 | info := datamodel.NewEmptyInfo() 95 | info.Properties.Size = utils.Int64p(1000) 96 | info.Name = utils.Stringp("marvel3") 97 | sketch, err := NewTopKSketch(info) 98 | if err != nil { 99 | b.Error("expected no errors, got", err) 100 | } 101 | for i := 0; i < 1000; i++ { 102 | if _, err := sketch.Add(values); err != nil { 103 | b.Error("expected no errors, got", err) 104 | } 105 | } 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /src/skizze-cli/bridge/domain.go: -------------------------------------------------------------------------------- 1 | package bridge 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "strings" 7 | 8 | "golang.org/x/net/context" 9 | 10 | pb "datamodel/protobuf" 11 | 12 | "github.com/gogo/protobuf/proto" 13 | ) 14 | 15 | func createDomain(fields []string, in *pb.Domain) error { 16 | if len(fields) != 5 { 17 | return fmt.Errorf("Expected 5 arguments got %d", len(fields)) 18 | } 19 | 20 | // FIXME make last 2 arguments optional 21 | capa, err := strconv.Atoi(fields[3]) 22 | if err != nil { 23 | return fmt.Errorf("Expected 3rd argument to be of type int: %q", err) 24 | } 25 | 26 | size, err := strconv.Atoi(fields[4]) 27 | if err != nil { 28 | return fmt.Errorf("Expected last argument to be of type int: %q", err) 29 | } 30 | 31 | types := []pb.SketchType{pb.SketchType_MEMB, pb.SketchType_FREQ, pb.SketchType_RANK, pb.SketchType_CARD} 32 | for _, ty := range types { 33 | sketch := &pb.Sketch{} 34 | sketch.Name = proto.String("") 35 | sketch.Type = &ty 36 | sketch.Properties = &pb.SketchProperties{ 37 | Size: proto.Int64(int64(size)), 38 | MaxUniqueItems: proto.Int64(int64(capa)), 39 | } 40 | in.Sketches = append(in.Sketches, sketch) 41 | } 42 | 43 | _, err = client.CreateDomain(context.Background(), in) 44 | if err == nil { 45 | fmt.Println("done") 46 | } 47 | return err 48 | } 49 | 50 | func addToDomain(fields []string, in *pb.Domain) error { 51 | if len(fields) < 4 { 52 | return fmt.Errorf("Expected at least 4 values, got %q", len(fields)) 53 | } 54 | addRequest := &pb.AddRequest{ 55 | Domain: in, 56 | Values: fields[3:], 57 | } 58 | _, err := client.Add(context.Background(), addRequest) 59 | if err == nil { 60 | fmt.Println("done") 61 | } 62 | return err 63 | } 64 | 65 | func sendDomainRequest(fields []string) error { 66 | name := fields[2] 67 | in := &pb.Domain{ 68 | Name: proto.String(name), 69 | } 70 | 71 | switch strings.ToLower(fields[0]) { 72 | case "create": 73 | return createDomain(fields, in) 74 | case "add": 75 | return addToDomain(fields, in) 76 | case "destroy": 77 | return deleteDomain(fields, in) 78 | case "info": 79 | return getDomainInfo(fields, in) 80 | default: 81 | return fmt.Errorf("unkown operation: %s", fields[0]) 82 | } 83 | } 84 | 85 | func listDomains() error { 86 | reply, err := client.ListDomains(context.Background(), &pb.Empty{}) 87 | if err == nil { 88 | for _, v := range reply.GetNames() { 89 | _, _ = fmt.Fprintln(w, fmt.Sprintf("Name: %s\t", v)) 90 | } 91 | _ = w.Flush() 92 | } 93 | return err 94 | } 95 | 96 | func deleteDomain(fields []string, in *pb.Domain) error { 97 | _, err := client.DeleteDomain(context.Background(), in) 98 | return err 99 | } 100 | 101 | func getDomainInfo(fields []string, in *pb.Domain) error { 102 | dom, err := client.GetDomain(context.Background(), in) 103 | if err != nil { 104 | return err 105 | } 106 | _, _ = fmt.Fprintln(w, fmt.Sprintf("Name: %s Type: %s\t", dom.GetName(), "")) 107 | _, _ = fmt.Fprintln(w, fmt.Sprintf("%d Sketches attached:", len(dom.GetSketches()))) 108 | for i, v := range dom.GetSketches() { 109 | _, _ = fmt.Fprintln(w, fmt.Sprintf(" %d. Name: %s Type: %s\t", i+1, v.GetName(), v.GetType())) 110 | } 111 | _ = w.Flush() 112 | return err 113 | } 114 | -------------------------------------------------------------------------------- /src/skizze-cli/bridge/loop.go: -------------------------------------------------------------------------------- 1 | package bridge 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io" 7 | "log" 8 | "os" 9 | "path/filepath" 10 | "strings" 11 | "text/tabwriter" 12 | 13 | "github.com/codegangsta/cli" 14 | 15 | "golang.org/x/net/context" 16 | 17 | "google.golang.org/grpc" 18 | 19 | "datamodel" 20 | pb "datamodel/protobuf" 21 | 22 | "github.com/martinpinto/liner" 23 | ) 24 | 25 | const helpString = ` 26 | CREATE DOM Create a new Domain with options 27 | DESTROY DOM Destroy a Domain 28 | 29 | CREATE CARD Create a Cardinality Sketch 30 | CREATE MEMB Create a Membership Sketch 31 | CREATE FREQ Create a Frequency Sketch 32 | CREATE RANK Create a Rankings Sketch 33 | 34 | LIST DOM List existing Domains 35 | LIST List existing Sketches 36 | 37 | INFO DOM Get details of a Domain 38 | INFO Get details of a Sketch 39 | 40 | ADD DOM [value2...] Add values to a Domain 41 | ADD FREQ [value2...] Add values to a frequency Sketch 42 | ADD MEMB [value2...] Add values to a membership Sketch 43 | ADD RANK [value2...] Add values to a rankings Sketch 44 | ADD CARD [value2...] Add values to a cardinality Sketch 45 | 46 | GET FREQ [value2...] Get the frequencies of the values in a FREQ Sketch 47 | GET MEMB [value2...] Get the memberships of the values in a MEMB Sketch 48 | GET RANK Get the top ranking values in a RANK Sketch 49 | GET CARD Get the cardinality of a CARD Sketch 50 | 51 | QUIT Exit skizze-cli 52 | 53 | SHORTCUTS: 54 | Ctrl+d Exit skizze-cli 55 | 56 | EXAMPLES: 57 | CREATE DOM users 100 100000 58 | ADD DOM users neil seif martin conor neil conor seif seif seif 59 | GET FREQ users neil 60 | GET RANK users 61 | GET CARD users 62 | ` 63 | 64 | var ( 65 | address string 66 | client pb.SkizzeClient 67 | completion = []string{ 68 | "create dom", "destroy dom", 69 | "create card", "create memb", "create freq", "create rank", 70 | "list", "list dom", 71 | "info", "info dom", 72 | "add dom", "add freq", "add memb", "add rank", "add card", 73 | "get freq", "get memb", "get rank", "get card", 74 | "help", "exit", 75 | } 76 | conn *grpc.ClientConn 77 | historyFn = filepath.Join(os.TempDir(), ".skizze_history") 78 | w = new(tabwriter.Writer) 79 | typeMap = map[string]pb.SketchType{ 80 | datamodel.HLLPP: pb.SketchType_CARD, 81 | datamodel.CML: pb.SketchType_FREQ, 82 | datamodel.Bloom: pb.SketchType_MEMB, 83 | datamodel.TopK: pb.SketchType_RANK, 84 | } 85 | version string 86 | ) 87 | 88 | func setupClient() (pb.SkizzeClient, *grpc.ClientConn) { 89 | // Connect to the server. 90 | var err error 91 | conn, err = grpc.Dial(address, grpc.WithInsecure()) 92 | if err != nil { 93 | log.Fatalf("fail to dial: %v", err) 94 | } 95 | return pb.NewSkizzeClient(conn), conn 96 | } 97 | 98 | func tearDownClient(conn io.Closer) { 99 | _ = conn.Close() 100 | } 101 | 102 | func getFields(query string) []string { 103 | fields := []string{} 104 | for _, f := range strings.Split(query, " ") { 105 | if len(f) > 0 { 106 | fields = append(fields, f) 107 | } 108 | } 109 | return fields 110 | } 111 | 112 | func evaluateQuery(query string) error { 113 | fields := getFields(query) 114 | if len(fields) != 0 && len(fields) <= 2 { 115 | //TODO: global stuff might be set 116 | switch strings.ToLower(fields[0]) { 117 | case "help": 118 | printHelp() 119 | return nil 120 | case "quit": 121 | tearDownClient(conn) 122 | os.Exit(0) 123 | return nil 124 | case "list": 125 | if len(fields) == 1 { 126 | return listSketches() 127 | } else if len(fields) == 2 && strings.ToLower(fields[1]) == "dom" { 128 | return listDomains() 129 | } else if len(fields) == 2 { 130 | v, ok := typeMap[strings.ToLower(fields[1])] 131 | if !ok { 132 | return fmt.Errorf("Invalid operation: %s", query) 133 | } 134 | return listSketchType(v) 135 | } 136 | case "save": 137 | if len(fields) == 1 { 138 | return save() 139 | } 140 | return fmt.Errorf("Invalid operation: %s", query) 141 | default: 142 | return fmt.Errorf("Invalid operation: %s", query) 143 | } 144 | } 145 | 146 | if len(fields) > 2 { 147 | switch strings.ToLower(fields[1]) { 148 | case datamodel.HLLPP: 149 | return sendSketchRequest(fields, pb.SketchType_CARD) 150 | case datamodel.CML: 151 | return sendSketchRequest(fields, pb.SketchType_FREQ) 152 | case datamodel.TopK: 153 | return sendSketchRequest(fields, pb.SketchType_RANK) 154 | case datamodel.Bloom: 155 | return sendSketchRequest(fields, pb.SketchType_MEMB) 156 | case datamodel.DOM: 157 | return sendDomainRequest(fields) 158 | default: 159 | return fmt.Errorf("unkown field or command %s", fields[1]) 160 | } 161 | } 162 | return errors.New("Invalid operation") 163 | } 164 | 165 | func save() error { 166 | _, err := client.CreateSnapshot(context.Background(), &pb.CreateSnapshotRequest{}) 167 | return err 168 | } 169 | 170 | func printHelp() { 171 | fmt.Printf("USAGE:\n %s", helpString) 172 | } 173 | 174 | // Run ... 175 | func Run() { 176 | app := cli.NewApp() 177 | app.Name = "skizze-cli" 178 | app.Usage = "A Skizze CLI client" 179 | app.Version = version 180 | app.UsageText = helpString 181 | 182 | app.Flags = []cli.Flag{ 183 | cli.StringFlag{ 184 | Name: "address, a", 185 | Value: "localhost:3596", 186 | Usage: "the Skizze server address to bind to", 187 | Destination: &address, 188 | EnvVar: "SKIZZE_ADDRESS", 189 | }, 190 | } 191 | 192 | app.Commands = []cli.Command{ 193 | { 194 | Name: "help, h", 195 | Usage: "print help", 196 | Action: func(*cli.Context) { 197 | printHelp() 198 | os.Exit(0) 199 | }, 200 | }, 201 | } 202 | 203 | app.Action = func(*cli.Context) { 204 | client, conn = setupClient() 205 | line := liner.NewLiner() 206 | w.Init(os.Stdout, 0, 8, 0, '\t', 0) 207 | 208 | defer func() { _ = line.Close() }() 209 | 210 | line.SetCtrlCAborts(true) 211 | 212 | line.SetCompleter(func(line string) (c []string) { 213 | for _, n := range completion { 214 | if strings.HasPrefix(n, strings.ToLower(line)) { 215 | c = append(c, n) 216 | } 217 | } 218 | return 219 | }) 220 | 221 | if f, err := os.Open(historyFn); err == nil { 222 | if _, err := line.ReadHistory(f); err == nil { 223 | _ = f.Close() 224 | } 225 | } 226 | 227 | for { 228 | if query, err := line.Prompt("skizze> "); err == nil { 229 | if err := evaluateQuery(query); err != nil { 230 | log.Printf("Error evaluating query: %s", err.Error()) 231 | } 232 | line.AppendHistory(query) 233 | } else if err == io.EOF { 234 | tearDownClient(conn) 235 | return 236 | } else if err == liner.ErrPromptAborted { 237 | fmt.Println("") 238 | } else { 239 | log.Printf("Error reading line: %s", err.Error()) 240 | } 241 | 242 | if f, err := os.Create(historyFn); err != nil { 243 | log.Fatalf("Error writing history file: %s", err.Error()) 244 | } else { 245 | if _, err := line.WriteHistory(f); err != nil { 246 | _ = f.Close() 247 | } 248 | } 249 | } 250 | } 251 | 252 | app.Run(os.Args) 253 | } 254 | -------------------------------------------------------------------------------- /src/skizze-cli/bridge/sketch.go: -------------------------------------------------------------------------------- 1 | package bridge 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "strconv" 7 | "strings" 8 | 9 | "golang.org/x/net/context" 10 | 11 | pb "datamodel/protobuf" 12 | 13 | "github.com/gogo/protobuf/proto" 14 | ) 15 | 16 | func createSketch(fields []string, in *pb.Sketch) error { 17 | if in.GetType() != pb.SketchType_CARD { 18 | if len(fields) > 4 { 19 | return fmt.Errorf("Too many argumets, expected 4 got %d", len(fields)) 20 | } 21 | num, err := strconv.Atoi(fields[3]) 22 | if err != nil { 23 | return fmt.Errorf("Expected last argument to be of type int: %q", err) 24 | } 25 | 26 | in.Properties = &pb.SketchProperties{ 27 | Size: proto.Int64(int64(num)), 28 | MaxUniqueItems: proto.Int64(int64(num)), 29 | } 30 | } 31 | _, err := client.CreateSketch(context.Background(), in) 32 | return err 33 | } 34 | 35 | func addToSketch(fields []string, in *pb.Sketch) error { 36 | if len(fields) < 4 { 37 | return fmt.Errorf("Expected at least 4 values, got %q", len(fields)) 38 | } 39 | addRequest := &pb.AddRequest{ 40 | Sketch: in, 41 | Values: fields[3:], 42 | } 43 | _, err := client.Add(context.Background(), addRequest) 44 | return err 45 | } 46 | 47 | func sendSketchRequest(fields []string, typ pb.SketchType) error { 48 | name := fields[2] 49 | in := &pb.Sketch{ 50 | Name: proto.String(name), 51 | Type: &typ, 52 | } 53 | 54 | switch strings.ToLower(fields[0]) { 55 | case "create": 56 | return createSketch(fields, in) 57 | case "add": 58 | return addToSketch(fields, in) 59 | case "get": 60 | return getFromSketch(fields, in) 61 | case "destroy": 62 | case "info": 63 | return getSketchInfo(in) 64 | default: 65 | return fmt.Errorf("unkown operation: %s", fields[0]) 66 | } 67 | return nil 68 | } 69 | 70 | func listSketches() error { 71 | reply, err := client.ListAll(context.Background(), &pb.Empty{}) 72 | if err == nil { 73 | for _, v := range reply.GetSketches() { 74 | line := fmt.Sprintf("Name: %s\t Type: %s", v.GetName(), v.GetType().String()) 75 | _, _ = fmt.Fprintln(w, line) 76 | } 77 | _ = w.Flush() 78 | } 79 | return err 80 | } 81 | 82 | func listSketchType(typ pb.SketchType) error { 83 | reply, err := client.List(context.Background(), &pb.ListRequest{Type: &typ}) 84 | if err == nil { 85 | for _, v := range reply.GetSketches() { 86 | line := fmt.Sprintf("Name: %s\t Type: %s", v.GetName(), v.GetType().String()) 87 | _, _ = fmt.Fprintln(w, line) 88 | } 89 | _ = w.Flush() 90 | } 91 | return err 92 | } 93 | 94 | func getSketchInfo(in *pb.Sketch) error { 95 | in.Properties = &pb.SketchProperties{ 96 | MaxUniqueItems: proto.Int64(0), 97 | Size: proto.Int64(0), 98 | } 99 | reply, err := client.GetSketch(context.Background(), in) 100 | log.Printf("%s", reply) 101 | return err 102 | } 103 | 104 | func getFromSketch(fields []string, in *pb.Sketch) error { 105 | if len(fields) < 3 { 106 | return fmt.Errorf("Expected at least 3 values, got %q", len(fields)) 107 | } 108 | getRequest := &pb.GetRequest{ 109 | Sketches: []*pb.Sketch{in}, 110 | Values: fields[3:], 111 | } 112 | 113 | switch in.GetType() { 114 | case pb.SketchType_CARD: 115 | reply, err := client.GetCardinality(context.Background(), getRequest) 116 | if err == nil { 117 | if len(reply.GetResults()) == 0 { 118 | log.Printf("%s does not exist", in.GetName()) 119 | } else { 120 | fmt.Printf("Cardinality: %d", reply.GetResults()[0].GetCardinality()) 121 | fmt.Println("") 122 | } 123 | } 124 | return err 125 | case pb.SketchType_FREQ: 126 | reply, err := client.GetFrequency(context.Background(), getRequest) 127 | if err == nil { 128 | if len(reply.GetResults()) == 0 { 129 | log.Printf("%s does not exist", in.GetName()) 130 | } else { 131 | for _, v := range reply.GetResults()[0].GetFrequencies() { 132 | line := fmt.Sprintf("Value: %s\t Hits: %d", v.GetValue(), v.GetCount()) 133 | _, _ = fmt.Fprintln(w, line) 134 | } 135 | } 136 | _ = w.Flush() 137 | } 138 | return err 139 | case pb.SketchType_MEMB: 140 | reply, err := client.GetMembership(context.Background(), getRequest) 141 | if err == nil { 142 | if len(reply.GetResults()) == 0 { 143 | log.Printf("%s does not exist", in.GetName()) 144 | } else { 145 | for _, v := range reply.GetResults()[0].GetMemberships() { 146 | line := fmt.Sprintf("Value: %s\t Member: %t", v.GetValue(), v.GetIsMember()) 147 | _, _ = fmt.Fprintln(w, line) 148 | } 149 | _ = w.Flush() 150 | } 151 | } 152 | return err 153 | case pb.SketchType_RANK: 154 | reply, err := client.GetRankings(context.Background(), getRequest) 155 | if err == nil { 156 | if len(reply.GetResults()) == 0 { 157 | log.Printf("%s does not exist", in.GetName()) 158 | } else { 159 | for i, v := range reply.GetResults()[0].GetRankings() { 160 | line := fmt.Sprintf("Rank: %d\t Value: %s\t Hits: %d", i+1, v.GetValue(), v.GetCount()) 161 | _, _ = fmt.Fprintln(w, line) 162 | } 163 | _ = w.Flush() 164 | } 165 | } 166 | return err 167 | default: 168 | return fmt.Errorf("Unkown Type %s", in.GetType().String()) 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /src/skizze-cli/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "skizze-cli/bridge" 4 | 5 | func main() { 6 | bridge.Run() 7 | } 8 | -------------------------------------------------------------------------------- /src/skizze/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | _ "net/http/pprof" 7 | 8 | "github.com/codegangsta/cli" 9 | "github.com/njpatel/loggo" 10 | "golang.org/x/crypto/ssh/terminal" 11 | 12 | "utils" 13 | "config" 14 | "manager" 15 | "server" 16 | ) 17 | 18 | var ( 19 | datadir string 20 | host string 21 | port int 22 | logger = loggo.GetLogger("skizze") 23 | version string 24 | ) 25 | 26 | func init() { 27 | setupLoggers() 28 | } 29 | 30 | func main() { 31 | app := cli.NewApp() 32 | app.Name = "Skizze" 33 | app.Usage = "A sketch data store for counting and sketching using probabilistic data-structures" 34 | app.Version = version 35 | 36 | app.Flags = []cli.Flag{ 37 | cli.StringFlag{ 38 | Name: "datadir, d", 39 | Value: config.DataDir, 40 | Usage: "the data directory", 41 | Destination: &datadir, 42 | EnvVar: "SKIZZE_DATA_DIR", 43 | }, 44 | cli.StringFlag{ 45 | Name: "host", 46 | Value: config.Host, 47 | Usage: "the host interface to bind to", 48 | Destination: &host, 49 | EnvVar: "SKIZZE_HOST", 50 | }, 51 | cli.IntFlag{ 52 | Name: "port, p", 53 | Value: config.Port, 54 | Usage: "the port to bind to", 55 | Destination: &port, 56 | EnvVar: "SKIZZE_PORT", 57 | }, 58 | } 59 | 60 | app.Action = func(*cli.Context) { 61 | datadir = setupDirectories(datadir) 62 | 63 | logger.Infof("Starting Skizze...") 64 | logger.Infof("Listening on: %s:%d", host, port) 65 | logger.Infof("Using data dir: %s", datadir) 66 | 67 | mngr := manager.NewManager() 68 | server.Run(mngr, host, port, datadir) 69 | } 70 | 71 | if err := app.Run(os.Args); err != nil { 72 | logger.Criticalf(err.Error()) 73 | } 74 | } 75 | 76 | func setupDirectories(datadir string) string { 77 | // FIXME: Allow specifying infodir 78 | datadir, err := utils.FullPath(datadir) 79 | if err != nil { 80 | panic(err) 81 | } 82 | if err := os.MkdirAll(datadir, os.ModePerm); err != nil { 83 | panic(err) 84 | } 85 | return datadir 86 | } 87 | 88 | func setupLoggers() { 89 | loggerSpec := os.Getenv("SKIZZE_DEBUG") 90 | 91 | // Setup logging and such things if we're running in a term 92 | if terminal.IsTerminal(int(os.Stdout.Fd())) { 93 | if loggerSpec == "" { 94 | loggerSpec = "=DEBUG" 95 | } 96 | // As we're in a terminal, let's make the output a little nicer 97 | _, _ = loggo.ReplaceDefaultWriter(loggo.NewSimpleWriter(os.Stderr, &loggo.ColorFormatter{})) 98 | } else { 99 | if loggerSpec == "" { 100 | loggerSpec = "=WARNING" 101 | } 102 | } 103 | 104 | _ = loggo.ConfigureLoggers(loggerSpec) 105 | } 106 | -------------------------------------------------------------------------------- /src/storage/aof.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "os" 7 | "strconv" 8 | "strings" 9 | "sync" 10 | "time" 11 | 12 | "utils" 13 | 14 | "github.com/golang/protobuf/proto" 15 | "github.com/njpatel/loggo" 16 | ) 17 | 18 | var logger = loggo.GetLogger("storage") 19 | 20 | // AOF ... 21 | type AOF struct { 22 | file *os.File 23 | buffer *bufio.ReadWriter 24 | lock sync.RWMutex 25 | inChan chan *Entry 26 | tickChan <-chan time.Time 27 | } 28 | 29 | // NewAOF ... 30 | func NewAOF(path string) *AOF { 31 | file, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0600) 32 | utils.PanicOnError(err) 33 | rdr := bufio.NewReader(file) 34 | wtr := bufio.NewWriter(file) 35 | inChan := make(chan *Entry, 100) 36 | tickChan := time.NewTicker(time.Second).C 37 | return &AOF{ 38 | file: file, 39 | buffer: bufio.NewReadWriter(rdr, wtr), 40 | lock: sync.RWMutex{}, 41 | inChan: inChan, 42 | tickChan: tickChan, 43 | } 44 | } 45 | 46 | // Run ... 47 | func (aof *AOF) Run() { 48 | go func() { 49 | for { 50 | select { 51 | case e := <-aof.inChan: 52 | aof.write(e) 53 | case <-aof.tickChan: 54 | if err := aof.buffer.Flush(); err != nil { 55 | logger.Errorf("an error has occurred while flushing AOF: %s", err.Error()) 56 | } 57 | } 58 | } 59 | }() 60 | } 61 | 62 | func (aof *AOF) write(e *Entry) { 63 | line := fmt.Sprintf("%d|%s/", e.op, string(e.raw)) 64 | if _, err := aof.buffer.WriteString(line); err != nil { 65 | logger.Errorf("an error has ocurred while writing AOF: %s", err.Error()) 66 | } 67 | } 68 | 69 | // Append ... 70 | func (aof *AOF) Append(op uint8, msg proto.Message) error { 71 | raw, err := proto.Marshal(msg) 72 | if err != nil { 73 | return err 74 | } 75 | e := &Entry{op, msg, raw} 76 | aof.inChan <- e 77 | return nil 78 | } 79 | 80 | // Read ... 81 | func (aof *AOF) Read() (*Entry, error) { 82 | line, err := aof.buffer.ReadBytes('/') 83 | if err != nil { 84 | return nil, err 85 | } 86 | line = line[:len(line)-1] 87 | rs := strings.Split(string(line), "|") 88 | msg := rs[1] 89 | op, err := strconv.Atoi(rs[0]) 90 | if err != nil { 91 | return nil, err 92 | } 93 | e := &Entry{uint8(op), nil, []byte(msg)} 94 | return e, nil 95 | } 96 | -------------------------------------------------------------------------------- /src/storage/aof_test.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "config" 5 | pb "datamodel/protobuf" 6 | "path/filepath" 7 | "testing" 8 | 9 | "github.com/gogo/protobuf/proto" 10 | 11 | "utils" 12 | "testutils" 13 | ) 14 | 15 | func createDom(id string) *pb.Domain { 16 | dom := new(pb.Domain) 17 | dom.Name = utils.Stringp(id) 18 | types := []pb.SketchType{pb.SketchType_MEMB, pb.SketchType_FREQ, pb.SketchType_RANK, pb.SketchType_CARD} 19 | for _, ty := range types { 20 | sketch := &pb.Sketch{} 21 | sketch.Name = dom.Name 22 | sketch.Type = &ty 23 | sketch.Properties = &pb.SketchProperties{ 24 | Size: utils.Int64p(100), 25 | MaxUniqueItems: utils.Int64p(10), 26 | } 27 | dom.Sketches = append(dom.Sketches, sketch) 28 | } 29 | return dom 30 | } 31 | 32 | func createSketch(id string, typ pb.SketchType) *pb.Sketch { 33 | sketch := &pb.Sketch{} 34 | sketch.Name = utils.Stringp(id) 35 | sketch.Type = &typ 36 | sketch.Properties = &pb.SketchProperties{ 37 | Size: utils.Int64p(100), 38 | MaxUniqueItems: utils.Int64p(10), 39 | } 40 | return sketch 41 | } 42 | 43 | func TestCreateDeleteDom(t *testing.T) { 44 | config.Reset() 45 | testutils.SetupTests() 46 | defer testutils.TearDownTests() 47 | 48 | path := filepath.Join(config.DataDir, "skizze.aof") 49 | aof := NewAOF(path) 50 | aof.Run() 51 | 52 | dom := createDom("test1") 53 | err := aof.Append(CreateDom, dom) 54 | if err != nil { 55 | t.Error("Expected no error, got", err) 56 | } 57 | 58 | dom = createDom("test2") 59 | err = aof.Append(CreateDom, dom) 60 | if err != nil { 61 | t.Error("Expected no error, got", err) 62 | } 63 | 64 | // Create new AOF 65 | aof = NewAOF(path) 66 | 67 | for { 68 | e, err2 := aof.Read() 69 | if err2 != nil { 70 | if err2.Error() != "EOF" { 71 | t.Error("Expected no error, got", err2) 72 | } 73 | break 74 | } 75 | dom = &pb.Domain{} 76 | err = proto.Unmarshal(e.raw, dom) 77 | if err != nil { 78 | t.Error("Expected no error, got", err) 79 | break 80 | } 81 | } 82 | aof.Run() 83 | 84 | dom = createDom("test3") 85 | 86 | if err = aof.Append(CreateDom, dom); err != nil { 87 | t.Error("Expected no error, got", err) 88 | } 89 | 90 | dom = new(pb.Domain) 91 | dom.Name = utils.Stringp("test1") 92 | 93 | if err = aof.Append(DeleteDom, dom); err != nil { 94 | t.Error("Expected no error, got", err) 95 | } 96 | 97 | aof = NewAOF(path) 98 | for { 99 | e, err := aof.Read() 100 | if err != nil { 101 | if err.Error() != "EOF" { 102 | t.Error("Expected no error, got", err) 103 | } 104 | break 105 | } 106 | dom := &pb.Domain{} 107 | err = proto.Unmarshal(e.raw, dom) 108 | if err != nil { 109 | t.Error("Expected no error, got", err) 110 | break 111 | } 112 | } 113 | } 114 | 115 | func TestCreateDeleteSketch(t *testing.T) { 116 | config.Reset() 117 | testutils.SetupTests() 118 | defer testutils.TearDownTests() 119 | 120 | path := filepath.Join(config.DataDir, "skizze.aof") 121 | aof := NewAOF(path) 122 | aof.Run() 123 | 124 | sketch := createSketch("skz1", pb.SketchType_CARD) 125 | err := aof.Append(CreateDom, sketch) 126 | if err != nil { 127 | t.Error("Expected no error, got", err) 128 | } 129 | 130 | sketch = createSketch("skz2", pb.SketchType_FREQ) 131 | err = aof.Append(CreateDom, sketch) 132 | if err != nil { 133 | t.Error("Expected no error, got", err) 134 | } 135 | 136 | // Create new AOF 137 | aof = NewAOF(path) 138 | for { 139 | e, err2 := aof.Read() 140 | if err2 != nil { 141 | if err2.Error() != "EOF" { 142 | t.Error("Expected no error, got", err2) 143 | } 144 | break 145 | } 146 | sketch = &pb.Sketch{} 147 | err = proto.Unmarshal(e.raw, sketch) 148 | if err != nil { 149 | t.Error("Expected no error, got", err) 150 | } 151 | } 152 | aof.Run() 153 | 154 | sketch = createSketch("skz3", pb.SketchType_RANK) 155 | err = aof.Append(CreateDom, sketch) 156 | if err != nil { 157 | t.Error("Expected no error, got", err) 158 | } 159 | 160 | sketch = createSketch("skz1", pb.SketchType_RANK) 161 | if err = aof.Append(DeleteDom, sketch); err != nil { 162 | t.Error("Expected no error, got", err) 163 | } 164 | 165 | addReq := &pb.AddRequest{ 166 | Sketch: sketch, 167 | Values: []string{"foo", "bar", "hello", "world"}, 168 | } 169 | if err = aof.Append(4, addReq); err != nil { 170 | t.Error("Expected no error, got", err) 171 | } 172 | 173 | aof = NewAOF(path) 174 | for { 175 | e, err := aof.Read() 176 | if err != nil { 177 | if err.Error() != "EOF" { 178 | t.Error("Expected no error, got", err) 179 | } 180 | break 181 | } 182 | if e.op == Add { 183 | req := &pb.AddRequest{} 184 | err = proto.Unmarshal(e.raw, req) 185 | } else { 186 | sketch := &pb.Sketch{} 187 | err = proto.Unmarshal(e.raw, sketch) 188 | } 189 | if err != nil { 190 | t.Error("Expected no error, got", err) 191 | } 192 | } 193 | } 194 | -------------------------------------------------------------------------------- /src/storage/entry.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import "github.com/golang/protobuf/proto" 4 | 5 | // CreateDom ... 6 | const ( 7 | CreateDom = uint8(0) 8 | DeleteDom = uint8(1) 9 | CreateSketch = uint8(2) 10 | DeleteSketch = uint8(3) 11 | Add = uint8(4) 12 | ) 13 | 14 | // Entry ... 15 | type Entry struct { 16 | op uint8 17 | msg proto.Message 18 | raw []byte 19 | } 20 | 21 | // OpType ... 22 | func (entry *Entry) OpType() uint8 { 23 | return entry.op 24 | } 25 | 26 | // Msg ... 27 | func (entry *Entry) Msg() proto.Message { 28 | return entry.msg 29 | } 30 | 31 | // RawMsg ... 32 | func (entry *Entry) RawMsg() []byte { 33 | return entry.raw 34 | } 35 | -------------------------------------------------------------------------------- /src/testutils/testutils.go: -------------------------------------------------------------------------------- 1 | package testutils 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "time" 8 | 9 | "utils" 10 | "config" 11 | ) 12 | 13 | // SetupTests ... 14 | func SetupTests() { 15 | dataDir := fmt.Sprintf("/tmp/skizze_storage_data_test") 16 | infoDir := fmt.Sprintf("/tmp/skizze_storage_info_test") 17 | 18 | // Cleanup any previously aborted test runs 19 | utils.PanicOnError(os.RemoveAll(dataDir)) 20 | utils.PanicOnError(os.RemoveAll(infoDir)) 21 | 22 | utils.PanicOnError(os.Setenv("SKIZZE_DATA_DIR", dataDir)) 23 | utils.PanicOnError(os.Setenv("SKIZZE_INFO_DIR", infoDir)) 24 | 25 | utils.PanicOnError(os.Mkdir(os.Getenv("SKIZZE_DATA_DIR"), 0777)) 26 | utils.PanicOnError(os.Mkdir(os.Getenv("SKIZZE_INFO_DIR"), 0777)) 27 | 28 | path, err := os.Getwd() 29 | utils.PanicOnError(err) 30 | path = filepath.Dir(path) 31 | configPath := filepath.Join(path, "config/default.toml") 32 | utils.PanicOnError(os.Setenv("SKIZZE_CONFIG", configPath)) 33 | config.DataDir = dataDir 34 | config.InfoDir = infoDir 35 | } 36 | 37 | // TearDownTests ... 38 | func TearDownTests() { 39 | utils.PanicOnError(os.RemoveAll(os.Getenv("SKIZZE_DATA_DIR"))) 40 | utils.PanicOnError(os.RemoveAll(os.Getenv("SKIZZE_INFO_DIR"))) 41 | time.Sleep(50 * time.Millisecond) 42 | config.Reset() 43 | } 44 | -------------------------------------------------------------------------------- /src/utils/test_utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "os" 7 | "path/filepath" 8 | "time" 9 | ) 10 | 11 | // SetupTests ... 12 | func SetupTests() { 13 | rand.Seed(time.Now().UnixNano()) 14 | dataDir := fmt.Sprintf("/tmp/skizze_storage_data_%d_%d", rand.Uint32()%1000, rand.Uint32()%1000) 15 | infoDir := fmt.Sprintf("/tmp/skizze_storage_info_%d_%d", rand.Uint32()%1000, rand.Uint32()%1000) 16 | 17 | PanicOnError(os.Setenv("SKZ_DATA_DIR", dataDir)) 18 | PanicOnError(os.Setenv("SKZ_INFO_DIR", infoDir)) 19 | 20 | PanicOnError(os.Mkdir(os.Getenv("SKZ_DATA_DIR"), 0777)) 21 | PanicOnError(os.Mkdir(os.Getenv("SKZ_INFO_DIR"), 0777)) 22 | 23 | path, err := os.Getwd() 24 | PanicOnError(err) 25 | path = filepath.Dir(path) 26 | configPath := filepath.Join(path, "config/default.toml") 27 | PanicOnError(os.Setenv("SKZ_CONFIG", configPath)) 28 | } 29 | 30 | // TearDownTests ... 31 | func TearDownTests() { 32 | PanicOnError(os.RemoveAll(os.Getenv("SKZ_DATA_DIR"))) 33 | PanicOnError(os.RemoveAll(os.Getenv("SKZ_INFO_DIR"))) 34 | time.Sleep(50 * time.Millisecond) 35 | } 36 | -------------------------------------------------------------------------------- /src/utils/utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "io" 5 | "os" 6 | "os/user" 7 | "strings" 8 | "path" 9 | "path/filepath" 10 | 11 | "github.com/njpatel/loggo" 12 | ) 13 | 14 | var logger = loggo.GetLogger("util") 15 | 16 | // PanicOnError is a helper function to panic on Error 17 | func PanicOnError(err error) { 18 | if err != nil { 19 | logger.Errorf("%v", err) 20 | panic(err) 21 | } 22 | } 23 | 24 | // Exists returns if path exists 25 | func Exists(path string) (bool, error) { 26 | _, err := os.Stat(path) 27 | if err == nil { 28 | return true, nil 29 | } 30 | if os.IsNotExist(err) { 31 | return false, nil 32 | } 33 | return true, err 34 | } 35 | 36 | // CloseFile ... 37 | func CloseFile(file io.Closer) { 38 | err := file.Close() 39 | PanicOnError(err) 40 | } 41 | 42 | // GetFileSize ... 43 | func GetFileSize(file *os.File) (int64, error) { 44 | stat, err := file.Stat() 45 | if err != nil { 46 | return -1, err 47 | } 48 | return stat.Size(), nil 49 | } 50 | 51 | // Stringp returns a pointer to a string, a convenience for dealing with protbuf generated code 52 | func Stringp(s string) *string { 53 | return &s 54 | } 55 | 56 | // Int32p as above 57 | func Int32p(i int32) *int32 { 58 | return &i 59 | } 60 | 61 | // Float32p as above 62 | func Float32p(i float32) *float32 { 63 | return &i 64 | } 65 | 66 | // Int64p as above 67 | func Int64p(i int64) *int64 { 68 | return &i 69 | } 70 | 71 | // Boolp as above 72 | func Boolp(b bool) *bool { 73 | return &b 74 | } 75 | 76 | // FullPath returns the full path with ~ expanded and relative paths made abspath 77 | func FullPath(p string) (string, error) { 78 | if strings.HasPrefix(p, "~/") { 79 | usr, err := user.Current() 80 | if err != nil { 81 | return "", err 82 | } 83 | p = strings.Replace(p, "~/", "", 1) 84 | p = path.Join(usr.HomeDir, p) 85 | } 86 | p, err := filepath.Abs(p) 87 | if err != nil { 88 | return "", err 89 | } 90 | return p, nil 91 | } 92 | -------------------------------------------------------------------------------- /src/utils/utils_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | ) 7 | 8 | func TestFullPath(t *testing.T) { 9 | p, err := FullPath("reltest.txt") 10 | if err != nil { 11 | t.Error("Expected no error, got", err) 12 | } 13 | 14 | if !strings.HasPrefix(p, "/") { 15 | t.Error("expected an absolute path, got ", p) 16 | } 17 | if !strings.HasSuffix(p, "reltest.txt") { 18 | t.Error("expected path to end with filename, got ", p) 19 | } 20 | 21 | p, err = FullPath("~/hometest.txt") 22 | if err != nil { 23 | t.Error("Expected no error, got", err) 24 | } 25 | 26 | if !strings.HasPrefix(p, "/home/") && !strings.HasPrefix(p, "/Users/") { 27 | t.Error("expected an absolute path, got ", p) 28 | } 29 | if !strings.HasSuffix(p, "hometest.txt") { 30 | t.Error("expected path to end with filename, got ", p) 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /vendor/manifest: -------------------------------------------------------------------------------- 1 | { 2 | "version": 0, 3 | "dependencies": [ 4 | { 5 | "importpath": "github.com/AndreasBriese/bbloom", 6 | "repository": "https://github.com/AndreasBriese/bbloom", 7 | "revision": "", 8 | "branch": "master" 9 | }, 10 | { 11 | "importpath": "github.com/BurntSushi/toml", 12 | "repository": "https://github.com/BurntSushi/toml", 13 | "revision": "5c4df71dfe9ac89ef6287afc05e4c1b16ae65a1e", 14 | "branch": "master" 15 | }, 16 | { 17 | "importpath": "github.com/codegangsta/cli", 18 | "repository": "https://github.com/codegangsta/cli", 19 | "revision": "a2943485b110df8842045ae0600047f88a3a56a1", 20 | "branch": "master" 21 | }, 22 | { 23 | "importpath": "github.com/dgryski/go-farm", 24 | "repository": "https://github.com/dgryski/go-farm", 25 | "revision": "d1e51a4af19092715f4ce7d8257fe5bc8f8be727", 26 | "branch": "master" 27 | }, 28 | { 29 | "importpath": "github.com/dgryski/go-pcgr", 30 | "repository": "https://github.com/dgryski/go-pcgr", 31 | "revision": "2d8ee285e8a4737537c52590c8c68276296cf8e7", 32 | "branch": "master" 33 | }, 34 | { 35 | "importpath": "github.com/dgryski/go-topk", 36 | "repository": "https://github.com/dgryski/go-topk", 37 | "revision": "6b42aca27653f256668254d2f801f2be5e73d90f", 38 | "branch": "master" 39 | }, 40 | { 41 | "importpath": "github.com/fatih/color", 42 | "repository": "https://github.com/fatih/color", 43 | "revision": "9aae6aaa22315390f03959adca2c4d395b02fcef", 44 | "branch": "master" 45 | }, 46 | { 47 | "importpath": "github.com/gogo/protobuf/proto", 48 | "repository": "https://github.com/gogo/protobuf", 49 | "revision": "c57e439bad574c2e0877ff18d514badcfced004d", 50 | "branch": "master", 51 | "path": "/proto" 52 | }, 53 | { 54 | "importpath": "github.com/gogo/protobuf/proto/testdata", 55 | "repository": "https://github.com/gogo/protobuf", 56 | "revision": "c57e439bad574c2e0877ff18d514badcfced004d", 57 | "branch": "master", 58 | "path": "/proto/testdata" 59 | }, 60 | { 61 | "importpath": "github.com/golang/glog", 62 | "repository": "https://github.com/golang/glog", 63 | "revision": "fca8c8854093a154ff1eb580aae10276ad6b1b5f", 64 | "branch": "master" 65 | }, 66 | { 67 | "importpath": "github.com/golang/protobuf/proto", 68 | "repository": "https://github.com/golang/protobuf", 69 | "revision": "2402d76f3d41f928c7902a765dfc872356dd3aad", 70 | "branch": "master", 71 | "path": "/proto" 72 | }, 73 | { 74 | "importpath": "github.com/golang/protobuf/proto/testdata", 75 | "repository": "https://github.com/golang/protobuf", 76 | "revision": "2402d76f3d41f928c7902a765dfc872356dd3aad", 77 | "branch": "master", 78 | "path": "/proto/testdata" 79 | }, 80 | { 81 | "importpath": "github.com/martinpinto/liner", 82 | "repository": "https://github.com/martinpinto/liner", 83 | "revision": "488c6d3e1cd42b7f917c1eab570360f744fa508b", 84 | "branch": "master" 85 | }, 86 | { 87 | "importpath": "github.com/mattn/go-colorable", 88 | "repository": "https://github.com/mattn/go-colorable", 89 | "revision": "3dac7b4f76f6e17fb39b768b89e3783d16e237fe", 90 | "branch": "master" 91 | }, 92 | { 93 | "importpath": "github.com/mattn/go-isatty", 94 | "repository": "https://github.com/mattn/go-isatty", 95 | "revision": "56b76bdf51f7708750eac80fa38b952bb9f32639", 96 | "branch": "master" 97 | }, 98 | { 99 | "importpath": "github.com/njpatel/loggo", 100 | "repository": "https://github.com/njpatel/loggo", 101 | "revision": "5f24b0ca9bb52d28c4b215550d34e688d0ee2f3d", 102 | "branch": "master" 103 | }, 104 | { 105 | "importpath": "github.com/retailnext/hllpp", 106 | "repository": "https://github.com/retailnext/hllpp", 107 | "revision": "38a7bb71b483e855d35010808143beaf05b67f9d", 108 | "branch": "master" 109 | }, 110 | { 111 | "importpath": "github.com/skizzehq/count-min-log", 112 | "repository": "https://github.com/skizzehq/count-min-log", 113 | "revision": "", 114 | "branch": "master" 115 | }, 116 | { 117 | "importpath": "golang.org/x/crypto/ssh/terminal", 118 | "repository": "https://go.googlesource.com/crypto", 119 | "revision": "3760e016850398b85094c4c99e955b8c3dea5711", 120 | "branch": "master", 121 | "path": "/ssh/terminal" 122 | }, 123 | { 124 | "importpath": "golang.org/x/net/context", 125 | "repository": "https://go.googlesource.com/net", 126 | "revision": "c92cdcb05f66ce5b61460e23498d2dc5fcf69aaf", 127 | "branch": "master", 128 | "path": "/context" 129 | }, 130 | { 131 | "importpath": "golang.org/x/net/http2", 132 | "repository": "https://go.googlesource.com/net", 133 | "revision": "c92cdcb05f66ce5b61460e23498d2dc5fcf69aaf", 134 | "branch": "master", 135 | "path": "/http2" 136 | }, 137 | { 138 | "importpath": "golang.org/x/net/internal/timeseries", 139 | "repository": "https://go.googlesource.com/net", 140 | "revision": "c92cdcb05f66ce5b61460e23498d2dc5fcf69aaf", 141 | "branch": "master", 142 | "path": "/internal/timeseries" 143 | }, 144 | { 145 | "importpath": "golang.org/x/net/trace", 146 | "repository": "https://go.googlesource.com/net", 147 | "revision": "c92cdcb05f66ce5b61460e23498d2dc5fcf69aaf", 148 | "branch": "master", 149 | "path": "/trace" 150 | }, 151 | { 152 | "importpath": "golang.org/x/oauth2", 153 | "repository": "https://go.googlesource.com/oauth2", 154 | "revision": "2baa8a1b9338cf13d9eeb27696d761155fa480be", 155 | "branch": "master" 156 | }, 157 | { 158 | "importpath": "google.golang.org/cloud/compute/metadata", 159 | "repository": "https://code.googlesource.com/gocloud", 160 | "revision": "6f469c6f5e20fde4c2088ed8b80bfa7ae296e622", 161 | "branch": "master", 162 | "path": "/compute/metadata" 163 | }, 164 | { 165 | "importpath": "google.golang.org/cloud/internal", 166 | "repository": "https://code.googlesource.com/gocloud", 167 | "revision": "6f469c6f5e20fde4c2088ed8b80bfa7ae296e622", 168 | "branch": "master", 169 | "path": "/internal" 170 | }, 171 | { 172 | "importpath": "google.golang.org/grpc", 173 | "repository": "https://github.com/grpc/grpc-go", 174 | "revision": "5da22b92e9485c15a6b284d52cc54d6580864c99", 175 | "branch": "master" 176 | } 177 | ] 178 | } --------------------------------------------------------------------------------