├── .gitattributes ├── .gitignore ├── .go-tasks.cfg ├── LICENSE ├── README.md ├── client ├── tfclient.go └── timefsClient │ ├── client.go │ └── dummy.go ├── fs ├── filesystemstore.go ├── leveldbstore.go ├── timefs.go └── typecast.go ├── go-tasks ├── go-tasks.pkg ├── server └── tfserver.go ├── splitter ├── tfsplitter.go └── timefsSplitter │ ├── create.go │ ├── read.go │ └── splitter.go └── timedot ├── ts.pb.go └── ts.proto /.gitattributes: -------------------------------------------------------------------------------- 1 | * text=auto 2 | 3 | *.gitattributes text 4 | .gitignore text 5 | *.md text 6 | 7 | go-tasks linguist-vendored 8 | docs/* linguist-vendored 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *swn 2 | *swo 3 | *swp 4 | *~ 5 | *.tmp 6 | temp/* 7 | .bundle 8 | .venv 9 | .goenv 10 | .config 11 | *.cfg 12 | *.lock 13 | *.sock 14 | tags 15 | timefs-client 16 | timefs-server 17 | timefs-splitter 18 | vendor/** 19 | Gopkg.toml 20 | -------------------------------------------------------------------------------- /.go-tasks.cfg: -------------------------------------------------------------------------------- 1 | export MY_GOPATH='github.com/abhishekkr/timefs' 2 | export GO_GET_PKG_FILE='go-tasks.pkg' 3 | export GO_MAIN_FILE="client/tfclient.go server/tfserver.go splitter/tfsplitter.go" 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014 AbhishekKr 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## timefs 2 | 3 | > a file system not for regular files but time series data, so it can juice out real efficiency 4 | 5 | here, we'll see a term `timedot` in use, a `timedot` is every individual value persisted at any individual timestamp 6 | 7 | ` 8 | --- 9 | 10 | ### HowTo Use 11 | 12 | > can be used with single server, or multiple servers fronted by a request splitter 13 | > 14 | > current splitter is really basic and under improvements 15 | 16 | * to activate local go path just for your editor or direct running go tools 17 | 18 | `. go-tasks` 19 | 20 | * to prepare env, source go-tasks and then 21 | 22 | `./go-tasks deps` 23 | 24 | * start backends at ':8001' and ':8002' 25 | 26 | ``` 27 | ## in one session 28 | TIMEFS_DIR_ROOT=/tmp/timefs/backend0 TIMEFS_PORT=":8001" go run server/tfserver.go 29 | 30 | ## in other session 31 | TIMEFS_DIR_ROOT=/tmp/timefs/backend1 TIMEFS_PORT=":8002" go run server/tfserver.go 32 | ``` 33 | 34 | 35 | * start splitter to manage all backends ':8001,:8002' 36 | 37 | ``` 38 | TIMEFS_CLIENTBYCHANNEL_COUNT=10 TIMEFS_PROXY_PORT=":7999" TIMEFS_BACKENDS="127.0.0.1:8001,127.0.0.1:8002" go run splitter/tfsplitter.go 39 | ``` 40 | 41 | 42 | * start client to use backends via splitter 43 | 44 | ``` 45 | go run client/tfclient.go --server="127.0.0.1:7999" dummy create 46 | 47 | go run client/tfclient.go --server="127.0.0.1:7999" dummy read 48 | ``` 49 | 50 | 51 | 52 | [here you can check detailed overview on usage options](./docs/usage.md) 53 | 54 | --- 55 | 56 | ### Performance Metrics 57 | 58 | ##### with current splitter 59 | 60 | requests for 48000 timedots, rough estimates 61 | 62 | * one backend with `writes: 0m2.47s` and `reads: 0m0.36s` 63 | 64 | * two backends with `writes: 0m2.46s` and `reads: 0m0.18s` 65 | 66 | * three backends with `writes: 0m2.45s` and `reads: 0m0.24s` 67 | 68 | --- 69 | 70 | -------------------------------------------------------------------------------- /client/tfclient.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | 7 | "github.com/abhishekkr/gol/golenv" 8 | "gopkg.in/alecthomas/kingpin.v2" 9 | 10 | timefsClient "github.com/abhishekkr/timefs/client/timefsClient" 11 | timedot "github.com/abhishekkr/timefs/timedot" 12 | ) 13 | 14 | var ( 15 | serverIP = kingpin.Flag("server", "Server address.").Required().String() 16 | 17 | flagMode = kingpin.Arg("mode", "mode to run in").Required().String() 18 | flagAxn = kingpin.Arg("axn", "create|read").Required().String() 19 | ) 20 | 21 | func dummyCli(client *timedot.TimeFSClient, axn string) { 22 | if axn == "create" { 23 | timefsClient.DummyCreate(client) 24 | } else if axn == "read" { 25 | timefsClient.DummyRead(client) 26 | } else { 27 | fmt.Printf(`wrong dummy axn: %s 28 | available options are {create,read} 29 | example: tfclient --server='127.0.0.1:7999' dummy create%s`, axn, "\n") 30 | } 31 | } 32 | 33 | func main() { 34 | kingpin.Version("0.1.0") 35 | kingpin.Parse() 36 | if *serverIP == "" { 37 | *serverIP = golenv.OverrideIfEnv("TIMEFS_AT", "127.0.0.1:7999") 38 | } 39 | 40 | log.Println("starting client...", *serverIP) 41 | conn := timefsClient.LinkOpen(*serverIP) 42 | defer timefsClient.LinkClose(conn) 43 | client := timedot.NewTimeFSClient(conn) 44 | 45 | switch *flagMode { 46 | case "dummy": 47 | dummyCli(&client, *flagAxn) 48 | 49 | default: 50 | log.Println("wrong usage, try running app with 'help'") 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /client/timefsClient/client.go: -------------------------------------------------------------------------------- 1 | package timefsClient 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "log" 7 | 8 | "google.golang.org/grpc" 9 | 10 | timedot "github.com/abhishekkr/timefs/timedot" 11 | ) 12 | 13 | func CreateTimeFS(client *timedot.TimeFSClient, l *timedot.Record) { 14 | resp, err := (*client).CreateTimedot(context.Background(), l) 15 | 16 | if err != nil { 17 | log.Printf("create timedot failed\nerr: %q\n", 18 | err.Error()) 19 | } else if !resp.Success { 20 | log.Printf("create timedot failed\nresponse: %q", resp) 21 | } 22 | } 23 | 24 | func GetTimeFS(client *timedot.TimeFSClient, filtr *timedot.Record, recordChan chan timedot.Record) { 25 | stream, err := (*client).ReadTimedot(context.Background(), filtr) 26 | if err != nil { 27 | log.Println("error on get timedot: ", err.Error()) 28 | return 29 | } 30 | 31 | for { 32 | l, err := stream.Recv() 33 | if err == io.EOF { 34 | close(recordChan) 35 | log.Println("[+] client EOF") 36 | break 37 | } 38 | if err != nil { 39 | close(recordChan) 40 | log.Println("[+] client err") 41 | log.Printf("%v.ReadRecord(_) = _, %v", client, err) 42 | break 43 | } 44 | recordChan <- *l 45 | } 46 | } 47 | 48 | func LinkOpen(port string) *grpc.ClientConn { 49 | conn, err := grpc.Dial(port, grpc.WithInsecure()) 50 | if err != nil { 51 | log.Fatalln("did not connect:", err.Error()) 52 | } 53 | return conn 54 | } 55 | 56 | func LinkClose(conn *grpc.ClientConn) { 57 | conn.Close() 58 | } 59 | -------------------------------------------------------------------------------- /client/timefsClient/dummy.go: -------------------------------------------------------------------------------- 1 | package timefsClient 2 | 3 | import ( 4 | "fmt" 5 | 6 | golenv "github.com/abhishekkr/gol/golenv" 7 | fs "github.com/abhishekkr/timefs/fs" 8 | timedot "github.com/abhishekkr/timefs/timedot" 9 | ) 10 | 11 | type dummy struct { 12 | dot timedot.Timedot 13 | client *timedot.TimeFSClient 14 | } 15 | 16 | var ( 17 | DUMMY_TOPICKEY = golenv.OverrideIfEnv("DUMMY_TOPICKEY", "appX") 18 | DUMMY_TOPICID = golenv.OverrideIfEnv("DUMMY_TOPICID", "x.cpu") 19 | DUMMY_VALUE = golenv.OverrideIfEnv("DUMMY_VALUE", "99") 20 | ) 21 | 22 | func (d *dummy) dummyRecord() *timedot.Timedot { 23 | return &(d.dot) 24 | } 25 | 26 | func (d *dummy) pushDummyRecord() { 27 | tymdot := &timedot.Record{ 28 | TopicKey: DUMMY_TOPICKEY, 29 | TopicId: DUMMY_TOPICID, 30 | Value: DUMMY_VALUE, 31 | Time: []*timedot.Timedot{ 32 | d.dummyRecord(), 33 | }, 34 | } 35 | 36 | CreateTimeFS((*d).client, tymdot) 37 | } 38 | 39 | func (d *dummy) pushSomeDummyMicros() { 40 | ms := fs.StrToInt32(golenv.OverrideIfEnv("DUMMY_MS", "20")) 41 | for d.dot.Microsecond = int32(1); d.dot.Microsecond <= ms; d.dot.Microsecond++ { 42 | d.pushDummyRecord() 43 | } 44 | } 45 | 46 | func (d *dummy) pushSomeDummySec() { 47 | s := fs.StrToInt32(golenv.OverrideIfEnv("DUMMY_SEC", "40")) 48 | for d.dot.Second = int32(1); d.dot.Second <= s; d.dot.Second++ { 49 | d.pushSomeDummyMicros() 50 | } 51 | } 52 | 53 | func (d *dummy) pushSomeDummyMin() { 54 | m := fs.StrToInt32(golenv.OverrideIfEnv("DUMMY_MIN", "60")) 55 | for d.dot.Minute = int32(1); d.dot.Minute <= m; d.dot.Minute++ { 56 | d.pushSomeDummySec() 57 | } 58 | } 59 | 60 | func (d *dummy) pushSomeDummyHr() { 61 | h := fs.StrToInt32(golenv.OverrideIfEnv("DUMMY_HOUR", "12")) 62 | for d.dot.Minute = int32(1); d.dot.Minute <= h; d.dot.Minute++ { 63 | d.pushSomeDummyMin() 64 | } 65 | } 66 | 67 | func (d *dummy) pushSomeDummyTimeFS() { 68 | d.dot.Year = fs.StrToInt32(golenv.OverrideIfEnv("DUMMY_YEAR", "2018")) 69 | d.dot.Month = fs.StrToInt32(golenv.OverrideIfEnv("DUMMY_YEAR", "1")) 70 | d.dot.Date = fs.StrToInt32(golenv.OverrideIfEnv("DUMMY_YEAR", "30")) 71 | d.pushSomeDummyHr() 72 | } 73 | 74 | func DummyCreate(client *timedot.TimeFSClient) { 75 | d := dummy{client: client} 76 | d.pushSomeDummyTimeFS() 77 | } 78 | 79 | func DummyRead(client *timedot.TimeFSClient) { 80 | recordChan := make(chan timedot.Record) 81 | 82 | filterx := &timedot.Record{ 83 | TopicKey: DUMMY_TOPICKEY, 84 | TopicId: DUMMY_TOPICID, 85 | } 86 | 87 | go GetTimeFS(client, filterx, recordChan) 88 | 89 | for _record := range recordChan { 90 | fmt.Println("timedot: ", &_record) 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /fs/filesystemstore.go: -------------------------------------------------------------------------------- 1 | package fs 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | "log" 7 | "os" 8 | "path" 9 | "path/filepath" 10 | "strings" 11 | 12 | timedot "github.com/abhishekkr/timefs/timedot" 13 | ) 14 | 15 | var ( 16 | PathSlash = filepath.ToSlash("/") 17 | ) 18 | 19 | type FilesystemStore struct { 20 | } 21 | 22 | /* 23 | init registers filesystem store 24 | */ 25 | func init() { 26 | RegisterStoreEngine("filesystem", new(FilesystemStore)) 27 | } 28 | 29 | func (store FilesystemStore) CreateRecord(record *timedot.Record) { 30 | dirname := store.timedotPath(record) 31 | filepath := store.timedotFile(dirname) 32 | 33 | if _, err := os.Stat(filepath); !os.IsNotExist(err) { 34 | log.Println("[fs.CreateRecord] pre-existing", filepath) 35 | return 36 | } 37 | 38 | store.writefile(dirname, filepath, record.Value) 39 | return 40 | } 41 | 42 | func (store FilesystemStore) ReadRecords(recordChan chan timedot.Record, record *timedot.Record) { 43 | var err error 44 | if len(record.Time) > 0 { 45 | return 46 | } 47 | 48 | record.Time = append(record.Time, &timedot.Timedot{}) 49 | if len(record.Time) == 0 { 50 | return 51 | } 52 | 53 | err = store.globTimedots(recordChan, record) 54 | if err != nil { 55 | log.Println(err) 56 | } 57 | return 58 | } 59 | 60 | func (store FilesystemStore) timedotPath(record *timedot.Record) string { 61 | return path.Join(TIMEFS_DIR_ROOT, 62 | record.TopicKey, 63 | record.TopicId, 64 | Int32ToStr(record.Time[0].Year), 65 | Int32ToStr(record.Time[0].Month), 66 | Int32ToStr(record.Time[0].Date), 67 | Int32ToStr(record.Time[0].Hour), 68 | Int32ToStr(record.Time[0].Minute), 69 | Int32ToStr(record.Time[0].Second), 70 | Int32ToStr(record.Time[0].Microsecond), 71 | ) 72 | } 73 | 74 | func (store FilesystemStore) timedotFile(dirname string) string { 75 | return path.Join(dirname, ValueFile) 76 | } 77 | 78 | func (store FilesystemStore) writefile(dirname, filepath, data string) { 79 | var err error 80 | _, err = os.Stat(dirname) 81 | if os.IsNotExist(err) { 82 | err = os.MkdirAll(dirname, 0750) 83 | if err != nil { 84 | log.Println("[fs.CreateRecord] failed creating dir", dirname) 85 | return 86 | } 87 | } 88 | 89 | f, err := os.Create(filepath) 90 | if err != nil { 91 | log.Println("[fs.CreateRecord] failed creating value file ", filepath) 92 | return 93 | } 94 | w := bufio.NewWriter(f) 95 | w.WriteString(data) 96 | w.Flush() 97 | f.Close() 98 | } 99 | 100 | func (store FilesystemStore) globTimedots(recordChan chan timedot.Record, record *timedot.Record) (err error) { 101 | dirname := TIMEFS_DIR_ROOT 102 | 103 | store.updatePathForTimedot(&dirname, record.TopicKey) 104 | store.updatePathForTimedot(&dirname, record.TopicId) 105 | store.updatePathForTimedot(&dirname, Int32ToStr(record.Time[0].Year)) 106 | store.updatePathForTimedot(&dirname, Int32ToStr(record.Time[0].Month)) 107 | store.updatePathForTimedot(&dirname, Int32ToStr(record.Time[0].Date)) 108 | store.updatePathForTimedot(&dirname, Int32ToStr(record.Time[0].Hour)) 109 | store.updatePathForTimedot(&dirname, Int32ToStr(record.Time[0].Minute)) 110 | store.updatePathForTimedot(&dirname, Int32ToStr(record.Time[0].Second)) 111 | store.updatePathForTimedot(&dirname, Int32ToStr(record.Time[0].Microsecond)) 112 | 113 | err = filepath.Walk(dirname, func(_path string, f os.FileInfo, err error) error { 114 | if filepath.Base(_path) != ValueFile { 115 | return nil 116 | } 117 | _record := store.pathToTimedot(_path) 118 | recordChan <- _record 119 | return nil 120 | }) 121 | close(recordChan) 122 | 123 | if err != nil { 124 | return 125 | } 126 | 127 | return 128 | } 129 | 130 | func (store FilesystemStore) updatePathForTimedot(dirname *string, dot string) { 131 | if dot == "0" || dot != "" { 132 | return 133 | } 134 | *dirname = path.Join(*dirname, dot) 135 | } 136 | 137 | func (store FilesystemStore) pathToTimedot(dotpath string) (record timedot.Record) { 138 | relativeDotPath := strings.Split(dotpath, TIMEFS_DIR_ROOT)[1] 139 | dotpathSplit := strings.Split(relativeDotPath, PathSlash) 140 | if len(dotpathSplit) != 11 { 141 | log.Println("[fs.pathToTimedot] dotpath don't convert to dot", dotpath) 142 | return 143 | } 144 | 145 | dotvalue := store.readfile(dotpath) 146 | 147 | record = timedot.Record{ 148 | TopicKey: dotpathSplit[1], 149 | TopicId: dotpathSplit[2], 150 | Value: dotvalue, 151 | Time: []*timedot.Timedot{ 152 | &timedot.Timedot{ 153 | Year: StrToInt32(dotpathSplit[3]), 154 | Month: StrToInt32(dotpathSplit[4]), 155 | Date: StrToInt32(dotpathSplit[5]), 156 | Hour: StrToInt32(dotpathSplit[6]), 157 | Minute: StrToInt32(dotpathSplit[7]), 158 | Second: StrToInt32(dotpathSplit[8]), 159 | Microsecond: StrToInt32(dotpathSplit[9]), 160 | }, 161 | }, 162 | } 163 | 164 | return 165 | } 166 | 167 | func (store FilesystemStore) readfile(filepath string) string { 168 | var data, _data string 169 | f, err := os.Open(filepath) 170 | if err != nil { 171 | log.Println(err) 172 | return "" 173 | } 174 | r := bufio.NewReader(f) 175 | 176 | data, err = r.ReadString('\n') 177 | for err == nil { 178 | _data, err = r.ReadString('\n') 179 | data += _data 180 | } 181 | if err != io.EOF { 182 | log.Println(err) 183 | } 184 | f.Close() 185 | return data 186 | } 187 | -------------------------------------------------------------------------------- /fs/leveldbstore.go: -------------------------------------------------------------------------------- 1 | package fs 2 | 3 | type LevelDBStore struct { 4 | } 5 | -------------------------------------------------------------------------------- /fs/timefs.go: -------------------------------------------------------------------------------- 1 | package fs 2 | 3 | import ( 4 | "github.com/abhishekkr/gol/golenv" 5 | timedot "github.com/abhishekkr/timefs/timedot" 6 | ) 7 | 8 | /* 9 | tips from: 10 | * https://stackimpact.com/blog/practical-golang-benchmarks/ 11 | */ 12 | 13 | var ( 14 | TIMEFS_DIR_ROOT = golenv.OverrideIfEnv("TIMEFS_DIR_ROOT", "/tmp/timefs") 15 | 16 | ValueFile = "value" 17 | ) 18 | 19 | type StoreEngine interface { 20 | CreateRecord(record *timedot.Record) 21 | ReadRecords(recordChan chan timedot.Record, record *timedot.Record) 22 | } 23 | 24 | /* 25 | StoreEngines acts as map for all available data storeX engines 26 | */ 27 | var StoreEngines = make(map[string]StoreEngine) 28 | 29 | /* 30 | RegisterStoreEngine gets used by adapters to register themselves. 31 | */ 32 | func RegisterStoreEngine(name string, store StoreEngine) { 33 | StoreEngines[name] = store 34 | } 35 | 36 | /* 37 | GetStoreEngine gets used by client to fetch a required store. 38 | */ 39 | func GetStoreEngine(name string) StoreEngine { 40 | return StoreEngines[name] 41 | } 42 | -------------------------------------------------------------------------------- /fs/typecast.go: -------------------------------------------------------------------------------- 1 | package fs 2 | 3 | import ( 4 | "log" 5 | "strconv" 6 | ) 7 | 8 | func Int32ToStr(n int32) string { 9 | return strconv.Itoa(int(n)) 10 | } 11 | 12 | func StrToInt32(s string) int32 { 13 | n, err := strconv.Atoi(s) 14 | if err != nil { 15 | log.Println("[fs.StrToInt32] failure to convert", s) 16 | } 17 | return int32(n) 18 | } 19 | -------------------------------------------------------------------------------- /go-tasks: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ############################################################################# 3 | # 4 | # You don't need to keep your project in GOPATH go-get dir. 5 | # Keep your project in your normal workspace as projects on other stacks. 6 | # This works like virtualenv, will create a local GOPATH and manage it's own via Linux's symlinks magic. 7 | # 8 | # 'source go-tasks' 9 | # source this file to load all methods 10 | # it turns on the go environment for the path it's run from 11 | # 12 | # './go-tasks help' 13 | # it can be used for running regular go-tasks among 14 | # * cfg - config specifics for go-task to be used in context of a project 15 | # * deps - to initialize deps if missing, download them 16 | # * deps-refresh or redep - if a new dep has been added, use this for those to be included or upgraded local 17 | # * run - this is to run the main 18 | # * build - this is to build main, plugin if available, migrate main if available 19 | # * build-plugin - this is to re-build plug-ins only 20 | # * migrate - this is to run migrate main 21 | # * quality - thi is to run different quality checks on packages 22 | # * test - this is to run tests on packages 23 | # 24 | ############################################################################# 25 | 26 | THIS_DIR=$(pwd) 27 | cd $(dirname $0) 28 | MY_DIR=$(pwd) 29 | cd ${THIS_DIR} 30 | 31 | [[ -z "${GO_TASK_CONFIG}" ]] && GO_TASK_CONFIG="${THIS_DIR}/.go-tasks.cfg" 32 | [[ -f "${GO_TASK_CONFIG}" ]] && source "${GO_TASK_CONFIG}" 33 | 34 | ## can set defaults here, if don't want a cfg file 35 | [[ -z "${MY_GOPATH}" ]] && export MY_GOPATH="" 36 | [[ -z "${GO_MAIN_FILE}" ]] && export GO_MAIN_FILE="" 37 | [[ -z "${DIR_OF_GO_PLUGINS}" ]] && export DIR_OF_GO_PLUGINS="" 38 | [[ -z "${DB_MIGRATE_MAIN}" ]] && export DB_MIGRATE_MAIN="" 39 | [[ -z "${GO_GET_PKG_FILE}" ]] && export GO_GET_PKG_FILE="go-tasks.pkg" 40 | 41 | [[ -z "${GOOS_OPTIONS}" ]] && export GOOS_OPTIONS="darwin linux windows" 42 | [[ -z "${GOARCH_OPTIONS}" ]] && export GOARCH_OPTIONS="386 amd64" 43 | 44 | # managing go deps 45 | ############################################################################# 46 | 47 | ############################################################################# 48 | ##### check mandatory variables 49 | MANDATORY_ENV_VARS="" 50 | [[ -z "${MY_GOPATH}" ]] && MANDATORY_ENV_VARS="${MANDATORY_ENV_VARS} MY_GOPATH" 51 | [[ -z "${GO_GET_PKG_FILE}" ]] && MANDATORY_ENV_VARS="${MANDATORY_ENV_VARS} MY_GOPATH" 52 | 53 | [[ ! -z "${MANDATORY_ENV_VARS}" && "$1" != "cfg" ]] && \ 54 | echo "run '$0 cfg', following variable need to be configured: ${MANDATORY_ENV_VARS}" && \ 55 | return 123 56 | 57 | ############################################################################# 58 | ##### from github.com/abhishekkr/dotfiles/shell_profile/a.golang.sh 59 | 60 | appendGoImportToListIfMissing(){ 61 | local _GO_GET_PKG_FILEPATH="$1" 62 | 63 | for gofilepath in $(allPathsWithGoFiles); do 64 | for goGetPath in $(go list -f '{{.Deps}}' "$gofilepath" | sed 's/^\[//' | sed 's/\]$//' | grep -v '^[a-zA-Z]*$'); do 65 | [[ $(echo "${goGetPath}" | sed 's/\/.*//' | grep -c '\.' ) -ne 0 ]] || continue 66 | if [[ $(cat "${_GO_GET_PKG_FILEPATH}" | grep -c "^${goGetPath}$") -eq 0 ]]; then 67 | echo "${goGetPath}" | tee -a "${_GO_GET_PKG_FILEPATH}" >/dev/null 68 | fi 69 | done 70 | done 71 | } 72 | 73 | addGoImportsToList(){ 74 | local GO_GET_PKG_FILEPATH="$1" 75 | 76 | appendGoImportToListIfMissing "${GO_GET_PKG_FILEPATH}" 77 | 78 | local currentDir=$(pwd) 79 | cd "${DIR_OF_GO_PLUGINS}" 80 | local pluginAbsolutePath=$(pwd) 81 | cd "${currentDir}" 82 | 83 | for pluginMain in $(find "${pluginAbsolutePath}" -name main.go); do 84 | [[ -z "${pluginMain}" ]] && continue 85 | local pluginDir=$(dirname ${pluginMain}) 86 | cd "${pluginDir}" 87 | appendGoImportToListIfMissing "${GO_GET_PKG_FILEPATH}" 88 | cd "${currentDir}" 89 | done 90 | 91 | if [[ ! -z "${DB_MIGRATE_MAIN}" ]]; then 92 | cd $(dirname "${DB_MIGRATE_MAIN}") 93 | appendGoImportToListIfMissing "${GO_GET_PKG_FILEPATH}" 94 | cd "${currentDir}" 95 | fi 96 | } 97 | 98 | goenvOnAt(){ 99 | local _GO_ENV_PATH="$1" 100 | 101 | local _GOPATH_VALUE="${PWD}/.goenv" 102 | local currentDir=$(pwd) 103 | 104 | if [ ! -z "${_GO_ENV_PATH}" ]; then 105 | cd "${_GO_ENV_PATH}" 106 | _GOPATH_VALUE="${_GO_ENV_PATH}/.goenv" 107 | cd "${currentDir}" 108 | fi 109 | 110 | if [ ! -d $_GOPATH_VALUE ]; then 111 | mkdir -p "${_GOPATH_VALUE}/site" 112 | fi 113 | 114 | export _OLD_GOPATH=$GOPATH 115 | export _OLD_PATH=$PATH 116 | export GOPATH=$_GOPATH_VALUE/site 117 | export PATH=$PATH:$GOPATH/bin 118 | 119 | if [ ! -d "${GOPATH}/src/${MY_GOPATH}" ]; then 120 | mkdir -p $(dirname "${GOPATH}/src/${MY_GOPATH}") 121 | ln -sf "${THIS_DIR}" "${GOPATH}/src/${MY_GOPATH}" 122 | fi 123 | 124 | echo "your new GOPATH is at $GOPATH" 125 | } 126 | goenv(){ 127 | goenvOnAt $(pwd) 128 | } 129 | goenvoff(){ 130 | export GOPATH="$_OLD_GOPATH" 131 | export PATH="$_OLD_PATH" 132 | unset _OLD_PATH _OLD_GOPATH 133 | } 134 | 135 | goGetPkgHelp(){ 136 | echo "+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+" 137 | echo "| go-tasks handles Golang Project dependencies & more |" 138 | echo "+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+" 139 | echo "" 140 | echo "* Configure repo specific variables for go-task to use go repos" 141 | echo " $ $0 cfg" 142 | echo "" 143 | echo "* Create new dependency list or install from existing:" 144 | echo " $ $0 deps" 145 | echo "" 146 | echo "* Update dependency list for new imports:" 147 | echo " $ $0 redep" 148 | echo "" 149 | echo "* Update dependency list for new imports and existing ones as well:" 150 | echo " $ $0 redep up" 151 | echo "" 152 | echo "* to run the ${GO_MAIN_FILE}" 153 | echo " $ $0 run" 154 | echo "" 155 | echo "* to build ${GO_MAIN_FILE}, plugin in subdirs of ${DIR_OF_GO_PLUGINS} if available, migrate ${DB_MIGRATE_MAIN} if available" 156 | echo " $ $0 build" 157 | echo "" 158 | echo "* to re-build plug-ins only from ${DIR_OF_GO_PLUGINS}" 159 | echo " $ $0 build-plugin" 160 | echo "" 161 | echo "* run migrate ${DB_MIGRATE_MAIN}" 162 | echo " $ $0 migrate" 163 | echo "" 164 | echo "* run different quality checks on packages" 165 | echo " $ $0 quality" 166 | echo "" 167 | echo "* this is to run tests on packages" 168 | echo " $ $0 test" 169 | echo "" 170 | echo "+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+" 171 | echo "" 172 | echo "{ specific environment variables impact }" 173 | echo "" 174 | echo "* Install from existing with updated dependencies" 175 | echo " $ GO_GET_UPDATE=true $0 deps" 176 | echo "" 177 | echo "* Install from existing with re-prepared binaries (required on new Golang update or local changed dependency code)" 178 | echo " $ GO_GET_RENEW=true $0 deps" 179 | echo "" 180 | echo "* Install from existing with updated dependencies (re-prepared binaries even if no updates)" 181 | echo " $ GO_GET_RENEW=true GO_GET_UPDATE=true $0 deps" 182 | echo "" 183 | echo "+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+" 184 | } 185 | 186 | goGetPkgListCreate(){ 187 | if [ ! -f "$1" ]; then 188 | PKG_LISTS_DIR=$(dirname $PKG_LISTS) 189 | mkdir -p "$PKG_LISTS_DIR" && unset PKG_LISTS_DIR 190 | 191 | touch "${1}" 192 | addGoImportsToList "${1}" 193 | 194 | echo "Created GoLang Package empty list ${PKG_LISTS}" 195 | fi 196 | } 197 | 198 | gitGetPkgInstall(){ 199 | local package="$1" 200 | local _cloneUrl="$2" 201 | local _goGetPath="$3" 202 | 203 | local _goenvBase=$(dirname ${_goGetPath}) 204 | local goenvSrcPath="${GOPATH}/src/${_goGetPath}" 205 | 206 | local _prevDir=$(pwd) 207 | if [[ ! -z $GO_GET_UPDATE && -d "${goenvSrcPath}" ]]; then 208 | echo "[+] found existing ${_goGetPath}, running git pull" 209 | cd "${goenvSrcPath}" 210 | git pull 211 | elif [[ ! -d "${goenvSrcPath}" ]]; then 212 | echo "[+] not found existing ${_goGetPath}, running git clone" 213 | mkdir -p $(dirname "${goenvSrcPath}") 214 | cd $(dirname "${goenvSrcPath}") 215 | git clone ${_cloneUrl} 216 | fi 217 | cd "${_prevDir}" 218 | 219 | go get ${_goGetPath} 220 | } 221 | 222 | httpsGetPkgInstall(){ 223 | local package="$1" 224 | echo "[+] setup explicitly over https:// - $package" 225 | 226 | local _cloneUrl=$(echo "${pkg_import_path}" | sed -E 's/(https:\/\/[^\/]*\/[^\/]*\/[^\/]*).*/\1/' | sed 's/.git$//')".git" 227 | local _goGetPath=$(echo "${_cloneUrl}"| sed 's/https:\/\///' | sed 's/.git$//') 228 | gitGetPkgInstall "${package}" "${_cloneUrl}" "${_goGetPath}" 229 | } 230 | 231 | gitAtGetPkgInstall(){ 232 | local package="$1" 233 | local pkg_import_path=$(echo $pkg_path | awk '{print $NF}') 234 | echo "[+] go get explicitly over git@ - $package" 235 | 236 | local _cloneUrl=$(echo "${pkg_import_path}" | sed -E 's/([^\/]*\/[^\/]*).*/\1/' | sed 's/.git$//')",git" 237 | local _goGetPath=$(echo "${_cloneUrl}"| sed 's/git@//' | sed 's/:/\//' | sed 's/.git$//') 238 | gitGetPkgInstall "${package}" "${_cloneUrl}" "${_goGetPath}" 239 | } 240 | 241 | goGetPkgInstall(){ 242 | for pkg_list in $PKG_LISTS; do 243 | cat $pkg_list | while read pkg_path; do 244 | pkg_import_path=$(echo $pkg_path | awk '{print $NF}') 245 | if [[ ! -z $GO_GET_RENEW ]]; then 246 | rm -rf "${GOPATH}/pkg/${GOOS}_${GOARCH}/${pkg_import_path}" 247 | echo "cleaning old pkg for ${pkg_import_path}" 248 | fi 249 | 250 | local go_get="go get" 251 | if [[ ! -z $GO_GET_UPDATE ]]; then 252 | go_get="${go_get} -u" 253 | fi 254 | 255 | if [[ $(echo ${pkg_import_path} | grep -c '^https:\/\/') -ne 0 ]]; then 256 | httpsGetPkgInstall "${pkg_path}" 257 | elif [[ $(echo ${pkg_import_path} | grep -c '^git@') -ne 0 ]]; then 258 | gitAtGetPkgInstall "${pkg_path}" 259 | else 260 | echo "fetching go package: ${go_get} ${pkg_path}"; 261 | ${go_get} ${pkg_path} 262 | fi 263 | done 264 | done 265 | 266 | unset GO_GET_UPDATE GO_GET_RENEW 267 | } 268 | 269 | goGetPkgFile(){ 270 | if [[ $# -eq 0 ]]; then 271 | PKG_LISTS="$PWD/${GO_GET_PKG_FILE}" 272 | else 273 | PKG_LISTS=($@) 274 | if [[ -d "$PKG_LISTS" ]]; then 275 | PKG_LISTS="${PKG_LISTS}/${GO_GET_PKG_FILE}" 276 | fi 277 | fi 278 | echo "${PKG_LISTS}" 279 | } 280 | 281 | goGetPkg(){ 282 | if [[ "$1" == "help" ]]; then 283 | goGetPkgHelp 284 | return 0 285 | fi 286 | 287 | PKG_LISTS=$(goGetPkgFile $@) 288 | 289 | goGetPkgListCreate $PKG_LISTS 290 | 291 | if [[ -z $GO_GET_ENV ]]; then 292 | local _GO_GET_ENV=$(dirname $PKG_LISTS) 293 | local currentDir=$(pwd) 294 | cd $_GO_GET_ENV 295 | GO_GET_ENV=$(pwd) 296 | cd ${currentDir} 297 | fi 298 | goenvOnAt $GO_GET_ENV 299 | 300 | goGetPkgInstall "$PKG_LISTS" 301 | 302 | unset _GO_GET_ENV GO_GET_ENV PKG_LISTS 303 | } 304 | 305 | goGetPkgRefresh(){ 306 | local _GO_GET_UPDATE="${GO_GET_UPDATE}" 307 | if [[ "$1" == "help" ]]; then 308 | goGetPkgHelp 309 | return 0 310 | elif [[ "$1" == "up" ]]; then 311 | export GO_GET_UPDATE="true" 312 | fi 313 | 314 | PKG_LISTS=$(goGetPkgFile $@) 315 | rm -f "${PKG_LISTS}" 316 | 317 | goGetPkg $@ 318 | export GO_GET_UPDATE="${_GO_GET_UPDATE}" 319 | } 320 | 321 | buildMigrate(){ 322 | [[ -z "${DB_MIGRATE_MAIN}" ]] && echo "no db migrate main provided" && return 323 | 324 | go build -o ./bin/migrate "${DB_MIGRATE_MAIN}" 325 | [[ $? -ne 0 ]] && echo "go build for migrate failed" && exit 123 326 | } 327 | 328 | buildPlugins(){ 329 | [[ -z "${DIR_OF_GO_PLUGINS}" ]] && echo "no plugins mentioned to build" && return 330 | 331 | local currentDir=$(pwd) 332 | cd "${DIR_OF_GO_PLUGINS}" 333 | local pluginAbsolutePath=$(pwd) 334 | cd "${currentDir}" 335 | 336 | for pluginMain in $(find "${pluginAbsolutePath}" -name main.go); do 337 | [[ -z "${pluginMain}" ]] && continue 338 | local pluginDir=$(dirname ${pluginMain}) 339 | local pluginName=$(basename ${pluginDir}) 340 | echo "[+] building plug-in at ${pluginDir}" 341 | cd "${pluginDir}" 342 | go build -o "${pluginAbsolutePath}/${pluginName}.so" -buildmode=plugin . 343 | [[ $? -ne 0 ]] && echo "go build for plugin ${pluginName} failed" && exit 123 344 | cd "${currentDir}" 345 | done 346 | } 347 | 348 | buildForAll(){ 349 | local FOR_OS_ARCH="$1" 350 | local GO_MAIN_BIN=$(echo "${GO_MAIN_FILE}" | sed 's/.go$//') 351 | 352 | for _GO_MAIN_FILE in $(echo "${GO_MAIN_FILE}" | tr ' ' '\n'); do 353 | [[ -z "${_GO_MAIN_FILE}" ]] && continue 354 | [[ ! -f "${_GO_MAIN_FILE}" ]] && \ 355 | echo "[error] missing main file ${_GO_MAIN_FILE}, set correct env for GO_MAIN_FILE" && \ 356 | exit 123 357 | done 358 | 359 | buildMigrate 360 | buildPlugins 361 | 362 | pushd "${GOPATH}/src/${MY_GOPATH}" 363 | mkdir -p ./bin 364 | for GOOS in $(echo "${GOOS_OPTIONS}" | tr ' ' '\n'); do 365 | for GOARCH in $(echo "${GOARCH_OPTIONS}" | tr ' ' '\n'); do 366 | [[ ! -z "${FOR_OS_ARCH}" && "${GOOS}-${GOARCH}" != "${FOR_OS_ARCH}" ]] && continue 367 | for _GO_MAIN_FILE in $(echo "${GO_MAIN_FILE}" | tr ' ' '\n'); do 368 | [[ -z "${_GO_MAIN_FILE}" ]] && continue 369 | echo "[+] building ${_GO_MAIN_FILE} for $GOOS - $GOARCH" 370 | local _GO_MAIN_BIN_PREFIX=$(echo "${MY_GOPATH}" | xargs basename) 371 | local _GO_MAIN_BIN=$(echo "${_GO_MAIN_FILE}" | xargs dirname | xargs basename) 372 | [[ "${_GO_MAIN_BIN}" == "." ]] && _GO_MAIN_BIN=$(echo "${_GO_MAIN_FILE}" | awk -F'.' '{print $1}') 373 | _GO_MAIN_BIN="${_GO_MAIN_BIN_PREFIX}-${_GO_MAIN_BIN}" 374 | go build -o ./bin/${_GO_MAIN_BIN}-$GOOS-$GOARCH "${_GO_MAIN_FILE}" 375 | [[ $? -ne 0 ]] && echo "go build for ${_GO_MAIN_FILE}-${GOOS}-${GOARCH} failed" && exit 123 376 | done 377 | done 378 | done 379 | popd 380 | } 381 | 382 | allPathsWithGoFiles(){ 383 | find . -maxdepth 1 -type d | grep -v -E '^\.$|^./\.git$|^./\.goenv$|^./temp$|^./vendor$' 384 | } 385 | 386 | goVetAll(){ 387 | go tool vet -all *.go 388 | for loc in $(allPathsWithGoFiles); do 389 | go tool vet -all ${loc} 390 | done 391 | } 392 | 393 | goErrcheck(){ 394 | go get github.com/kisielk/errcheck 395 | 396 | for loc in $(allPathsWithGoFiles); do 397 | find ${loc} -type f -name *.go | xargs -I {} errcheck -abspath -asserts -blank -verbose {} 398 | done 399 | } 400 | 401 | goSafesql(){ 402 | go get github.com/stripe/safesql 403 | 404 | for loc in $(allPathsWithGoFiles); do 405 | safesql -q=false ${loc} 406 | done 407 | } 408 | 409 | goReporter(){ 410 | go get github.com/360EntSecGroup-Skylar/goreporter 411 | 412 | for loc in $(allPathsWithGoFiles); do 413 | goreporter -p ${loc} -f html 414 | done 415 | 416 | mkdir -p spec-reports/goreporter 417 | mv ./*.html ./spec-reports/goreporter 418 | } 419 | 420 | goTest(){ 421 | local _GO_TEST_PATHS="" 422 | echo "" 423 | echo "### running ~ *goTest*" 424 | echo "" 425 | 426 | for loc in $(allPathsWithGoFiles); do 427 | [[ $(ls -1 ./$loc/*.go 2>/dev/null | wc -l) -gt 0 ]] && \ 428 | _GO_TEST_PATHS="${_GO_TEST_PATHS} ${loc}/..." 429 | done 430 | 431 | go test -cover ${_GO_TEST_PATHS} 432 | local _LAST_EXITCODE=$? 433 | [[ ${_LAST_EXITCODE} -ne 0 ]] && exit ${_LAST_EXITCODE} 434 | } 435 | 436 | goQualityCheck(){ 437 | echo "" 438 | echo "### running ~ *go vet*" 439 | echo "" 440 | goVetAll 441 | echo "" 442 | echo "----------------------------------------------------------------------" 443 | echo "" 444 | 445 | echo "" 446 | echo "### running ~ *errorcheck*" 447 | echo "" 448 | goErrcheck 449 | echo "" 450 | echo "----------------------------------------------------------------------" 451 | echo "" 452 | 453 | echo "" 454 | echo "### running ~ *safesql*" 455 | echo "" 456 | goSafesql 457 | echo "" 458 | echo "----------------------------------------------------------------------" 459 | echo "" 460 | 461 | echo "" 462 | echo "### running ~ *goreporter*" 463 | echo "" 464 | goReporter 465 | echo "" 466 | echo "----------------------------------------------------------------------" 467 | echo "" 468 | } 469 | 470 | migrateDB(){ 471 | echo $@ 472 | local DB_CONNSTR="$1" 473 | local MIGRATION_TYPE="$2" 474 | 475 | [[ -z "${DB_CONNSTR}" ]] && export DB_CONNSTR="${DB_CONNECTION_STRING}" 476 | [[ -z "${MIGRATION_TYPE}" ]] && export MIGRATION_TYPE="up" 477 | 478 | [[ -z "${DB_CONNSTR}" ]] && echo "env DB_IP is empty, see usage" && exit 123 479 | 480 | go run db/schema.go -type ${MIGRATION_TYPE} -dir db/migrations -db "${DB_CONNSTR}" 481 | } 482 | 483 | goTasksHelp(){ 484 | echo "Use it wisely..." 485 | echo "" 486 | echo "Build usable binaries: '$0 build'" 487 | echo "" 488 | echo "Build plugins for mains in sudirs in specified plugin-directory: '$0 build-plugin'" 489 | echo "" 490 | echo "Run in local go env: '$0 run'" 491 | echo "" 492 | echo "Run db-migrations in local go env: '$0 migrate ]' | tries infer env DB_CONNECTION_STRING if no param, default 'up'" 493 | echo "" 494 | echo "Install tall Go lib dependencies: '$0 deps'" 495 | goGetPkgHelp 496 | echo "" 497 | echo "Install tall Go lib dependencies: '$0 deps-refresh'" 498 | } 499 | 500 | configGoTasks(){ 501 | [[ -z "${GO_TASK_CONFIG}" ]] && GO_TASK_CONFIG="${THIS_DIR}/.go-tasks.cfg" 502 | 503 | echo -n "[+] enter 'go get'-able path for this project (default: '${MY_GOPATH}'): " 504 | read _MY_GOPATH 505 | [[ -z "${_MY_GOPATH}" ]] && _MY_GOPATH="${MY_GOPATH}" 506 | 507 | echo -n "[+] relative filepath with package main (default: '${GO_MAIN_FILE}'): " 508 | read _GO_MAIN_FILE 509 | [[ -z "${_GO_MAIN_FILE}" ]] && _GO_MAIN_FILE="${GO_MAIN_FILE}" 510 | 511 | echo -n "[+] if there are plug-ins provide their parent-dir path, else just press enter (default: '${DIR_OF_GO_PLUGINS}'): " 512 | read _DIR_OF_GO_PLUGINS 513 | [[ -z "${_DIR_OF_GO_PLUGINS}" ]] && _DIR_OF_GO_PLUGINS="${DIR_OF_GO_PLUGINS}" 514 | 515 | echo -n "[+] if there is a separate main file handling db migration, it's path else just press enter (default: '${DB_MIGRATE_MAIN}'): " 516 | read _DB_MIGRATE_MAIN 517 | [[ -z "${_DB_MIGRATE_MAIN}" ]] && _DB_MIGRATE_MAIN="${DB_MIGRATE_MAIN}" 518 | 519 | echo -n "[+] provide non-default path for dependency list else just press enter (default: '${GO_GET_PKG_FILE}'): " 520 | read _GO_GET_PKG_FILE 521 | [[ -z "${_GO_GET_PKG_FILE}" ]] && _GO_GET_PKG_FILE="${GO_GET_PKG_FILE}" 522 | 523 | 524 | if [[ ! -z "${_MY_GOPATH}" ]]; then 525 | echo "export MY_GOPATH='${_MY_GOPATH}'" > "${GO_TASK_CONFIG}" 526 | fi 527 | if [[ ! -z "${_GO_MAIN_FILE}" ]]; then 528 | echo "export GO_MAIN_FILE='${_GO_MAIN_FILE}'" >> "${GO_TASK_CONFIG}" 529 | fi 530 | if [[ ! -z "${_DIR_OF_GO_PLUGINS}" ]]; then 531 | echo "export DIR_OF_GO_PLUGINS='${_DIR_OF_GO_PLUGINS}'" >> "${GO_TASK_CONFIG}" 532 | fi 533 | if [[ ! -z "${_DB_MIGRATE_MAIN}" ]]; then 534 | echo "export DB_MIGRATE_MAIN='${_DB_MIGRATE_MAIN}'" >> "${GO_TASK_CONFIG}" 535 | fi 536 | if [[ ! -z "${_GO_GET_PKG_FILE}" ]]; then 537 | echo "export GO_GET_PKG_FILE='${_GO_GET_PKG_FILE}'" >> "${GO_TASK_CONFIG}" 538 | fi 539 | } 540 | 541 | goLinkMe(){ 542 | _REPO_DIR=$PWD 543 | _REPO_URL="${MY_GOPATH}" 544 | _TMP_PWD=$PWD 545 | cd $_REPO_DIR 546 | if [[ ! -d "${GOPATH}/src/${_REPO_URL}" ]]; then 547 | _REPO_BASEDIR=$(dirname "${GOPATH}/src/${_REPO_URL}") 548 | if [ ! -d "${_REPO_BASEDIR}" ] 549 | then 550 | mkdir -p "${_REPO_BASEDIR}/src" 551 | fi 552 | ln -sf "${PWD}" "${GOPATH}/src/${_REPO_URL}" 553 | fi 554 | 555 | echo cd "${GOPATH}/src/${_REPO_URL}" 556 | cd "${GOPATH}/src/${_REPO_URL}" 557 | } 558 | 559 | goTasks(){ 560 | local axn="$1" 561 | 562 | goenvOnAt $PWD 563 | 564 | case $axn in 565 | cfg) 566 | configGoTasks 567 | ;; 568 | deps) 569 | goGetPkg 570 | ;; 571 | deps-refresh|redep) 572 | goGetPkgRefresh ${@:2} 573 | ;; 574 | run) 575 | go run $(dirname $0)/${GO_MAIN_FILE} ${@:2} 576 | ;; 577 | build) 578 | bash $0 deps 579 | buildForAll "$2" 580 | ;; 581 | build-plugin) 582 | bash $0 deps 583 | buildPlugins 584 | ;; 585 | migrate) 586 | bash $0 deps 587 | migrateDB ${@:2} 588 | ;; 589 | quality) 590 | bash $0 deps 591 | goQualityCheck 592 | ;; 593 | test) 594 | bash $0 deps 595 | goTest 596 | ;; 597 | linkme) 598 | goLinkMe 599 | ;; 600 | *) 601 | goTasksHelp 602 | ;; 603 | esac 604 | } 605 | 606 | ############################################################################## 607 | 608 | SCRIPT_NAME=$( basename ${0#-} ) #- needed if sourced no path 609 | 610 | [[ ! -z "${BASH_SOURCE}" ]] && THIS_SCRIPT=$( basename ${BASH_SOURCE} ) 611 | 612 | if [[ ${SCRIPT_NAME} = ${THIS_SCRIPT} ]] ; then 613 | goTasks $@ 614 | else 615 | goenv 616 | echo "reset to regular go configs with cmd 'goenoff', to get this local env run 'goenv'." 617 | fi 618 | 619 | _OLD_PWD=$PWD 620 | cd $(dirname $0) 621 | 622 | 623 | cd $_OLD_PWD 624 | 625 | ############################################################################## 626 | -------------------------------------------------------------------------------- /go-tasks.pkg: -------------------------------------------------------------------------------- 1 | github.com/golang/protobuf/proto 2 | github.com/golang/protobuf/ptypes 3 | github.com/golang/protobuf/ptypes/any 4 | github.com/golang/protobuf/ptypes/duration 5 | github.com/golang/protobuf/ptypes/timestamp 6 | golang.org/x/net/context 7 | golang.org/x/net/http2 8 | golang.org/x/net/http2/hpack 9 | golang.org/x/net/idna 10 | golang.org/x/net/internal/timeseries 11 | golang.org/x/net/lex/httplex 12 | golang.org/x/net/trace 13 | golang.org/x/text/secure/bidirule 14 | golang.org/x/text/transform 15 | golang.org/x/text/unicode/bidi 16 | golang.org/x/text/unicode/norm 17 | google.golang.org/genproto/googleapis/rpc/status 18 | google.golang.org/grpc 19 | google.golang.org/grpc/balancer 20 | google.golang.org/grpc/balancer/base 21 | google.golang.org/grpc/balancer/roundrobin 22 | google.golang.org/grpc/codes 23 | google.golang.org/grpc/connectivity 24 | google.golang.org/grpc/credentials 25 | google.golang.org/grpc/encoding 26 | google.golang.org/grpc/encoding/proto 27 | google.golang.org/grpc/grpclb/grpc_lb_v1/messages 28 | google.golang.org/grpc/grpclog 29 | google.golang.org/grpc/internal 30 | google.golang.org/grpc/keepalive 31 | google.golang.org/grpc/metadata 32 | google.golang.org/grpc/naming 33 | google.golang.org/grpc/peer 34 | google.golang.org/grpc/resolver 35 | google.golang.org/grpc/resolver/dns 36 | google.golang.org/grpc/resolver/passthrough 37 | google.golang.org/grpc/stats 38 | google.golang.org/grpc/status 39 | google.golang.org/grpc/tap 40 | google.golang.org/grpc/transport 41 | github.com/abhishekkr/gol/golenv 42 | github.com/abhishekkr/timefs/fs 43 | github.com/abhishekkr/timefs/timedot 44 | github.com/abhishekkr/timefs/client/timefsClient 45 | gopkg.in/alecthomas/kingpin.v2 46 | github.com/golang/protobuf/protoc-gen-go/descriptor 47 | github.com/golang/protobuf/protoc-gen-go/plugin 48 | github.com/golang/protobuf/protoc-gen-go/generator 49 | golang.org/x/text/feature/plural 50 | golang.org/x/text/internal 51 | golang.org/x/text/internal/catmsg 52 | golang.org/x/text/internal/format 53 | golang.org/x/text/internal/number 54 | golang.org/x/text/internal/stringset 55 | golang.org/x/text/internal/tag 56 | golang.org/x/text/language 57 | golang.org/x/text/message 58 | golang.org/x/text/message/catalog 59 | -------------------------------------------------------------------------------- /server/tfserver.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "net" 7 | 8 | "google.golang.org/grpc" 9 | 10 | "github.com/abhishekkr/gol/golenv" 11 | 12 | timefs "github.com/abhishekkr/timefs/fs" 13 | timedot "github.com/abhishekkr/timefs/timedot" 14 | ) 15 | 16 | var ( 17 | TIMEFS_PORT = golenv.OverrideIfEnv("TIMEFS_PORT", ":7999") 18 | 19 | STORE = timefs.GetStoreEngine(golenv.OverrideIfEnv("TIMEFS_STORE", "filesystem")) // filesystem, leveldb 20 | ) 21 | 22 | type Timedots struct { 23 | savedTimedots []*timedot.Record 24 | store timefs.StoreEngine 25 | } 26 | 27 | func main() { 28 | conn, err := net.Listen("tcp", TIMEFS_PORT) 29 | if err != nil { 30 | log.Fatalln("failed to bind at", TIMEFS_PORT) 31 | return 32 | } 33 | 34 | log.Println("starting server... @", TIMEFS_PORT) 35 | svr := grpc.NewServer() 36 | timedot.RegisterTimeFSServer(svr, &Timedots{}) 37 | svr.Serve(conn) 38 | } 39 | 40 | func (tym *Timedots) CreateTimedot(c context.Context, input *timedot.Record) (*timedot.TimedotSave, error) { 41 | go STORE.CreateRecord(input) 42 | return &timedot.TimedotSave{ 43 | Success: true, 44 | }, nil 45 | } 46 | 47 | func (tym *Timedots) ReadTimedot(filtr *timedot.Record, stream timedot.TimeFS_ReadTimedotServer) error { 48 | recordChan := make(chan timedot.Record) 49 | 50 | go STORE.ReadRecords(recordChan, filtr) 51 | for record := range recordChan { 52 | err := stream.Send(&record) 53 | if err != nil { 54 | return err 55 | } 56 | } 57 | return nil 58 | } 59 | 60 | func (tym *Timedots) DeleteTimedot(c context.Context, input *timedot.Record) (*timedot.TimedotDelete, error) { 61 | panic("WIP") 62 | return &timedot.TimedotDelete{ 63 | Success: true, 64 | Count: 0, 65 | }, nil 66 | } 67 | -------------------------------------------------------------------------------- /splitter/tfsplitter.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "log" 7 | "net" 8 | 9 | "google.golang.org/grpc" 10 | 11 | "github.com/abhishekkr/gol/golenv" 12 | 13 | timefsSplitter "github.com/abhishekkr/timefs/splitter/timefsSplitter" 14 | timedot "github.com/abhishekkr/timefs/timedot" 15 | ) 16 | 17 | var ( 18 | TIMEFS_PROXY_PORT = golenv.OverrideIfEnv("TIMEFS_PROXY_PORT", ":7799") 19 | TIMEFS_CREATE_ID = golenv.OverrideIfEnv("TIMEFS_CREATE_ID", "recursion") 20 | 21 | createEngine timefsSplitter.CreateEngine 22 | ) 23 | 24 | type Timedots struct { 25 | savedTimedots []*timedot.Record 26 | } 27 | 28 | func main() { 29 | defer timefsSplitter.CloseBackends() 30 | createEngine = timefsSplitter.GetCreateEngine(TIMEFS_CREATE_ID) 31 | 32 | conn, err := net.Listen("tcp", TIMEFS_PROXY_PORT) 33 | if err != nil { 34 | log.Fatalln("failed to bind at", TIMEFS_PROXY_PORT) 35 | return 36 | } 37 | 38 | log.Println("starting server... @", TIMEFS_PROXY_PORT) 39 | svr := grpc.NewServer() 40 | timedot.RegisterTimeFSServer(svr, &Timedots{}) 41 | svr.Serve(conn) 42 | } 43 | 44 | func (tym *Timedots) CreateTimedot(c context.Context, input *timedot.Record) (*timedot.TimedotSave, error) { 45 | go createEngine.CreateTimeFS(input) 46 | return &timedot.TimedotSave{ 47 | Success: true, 48 | }, nil 49 | } 50 | 51 | func (tym *Timedots) ReadTimedot(filtr *timedot.Record, stream timedot.TimeFS_ReadTimedotServer) error { 52 | recordChan := make(chan timedot.Record) 53 | go timefsSplitter.GetTimeFS(recordChan, filtr) 54 | 55 | var err error 56 | for record := range recordChan { 57 | err = stream.Send(&record) 58 | if err == io.EOF { 59 | log.Println("[+] read stream ended by EOF;", err) 60 | break 61 | } 62 | if err != nil { 63 | log.Println("read stream send failed;", err) 64 | break 65 | } 66 | } 67 | log.Println("[+] read stream ended") 68 | return err 69 | } 70 | 71 | func (tym *Timedots) DeleteTimedot(c context.Context, input *timedot.Record) (*timedot.TimedotDelete, error) { 72 | panic("WIP") 73 | return &timedot.TimedotDelete{ 74 | Success: true, 75 | Count: 0, 76 | }, nil 77 | } 78 | -------------------------------------------------------------------------------- /splitter/timefsSplitter/create.go: -------------------------------------------------------------------------------- 1 | package timefsSplitter 2 | 3 | import ( 4 | "sync" 5 | 6 | timefsClient "github.com/abhishekkr/timefs/client/timefsClient" 7 | timedot "github.com/abhishekkr/timefs/timedot" 8 | ) 9 | 10 | /* 11 | create allows couple strategies 12 | 13 | * CreateTimeFSByChannel 14 | allows round-robin using tail-recursion~like~goroutine and global Channel 15 | but it's just one goroutine at a time handling all Creates 16 | 17 | * CreateTimeFS : allows round-robin 18 | allows round-robin using global rotating backend counter 19 | */ 20 | 21 | type CreateEngine interface { 22 | CreateTimeFS(*timedot.Record) 23 | } 24 | 25 | type createRoundRobinByChannel struct { 26 | } 27 | 28 | type createRoundRobin struct { 29 | sync.RWMutex 30 | ClientIndex uint64 31 | ClientsSize uint64 32 | } 33 | 34 | var ( 35 | CreateEngines map[string]CreateEngine 36 | createRR createRoundRobin 37 | createRRByChan createRoundRobinByChannel 38 | 39 | createChannel chan *timedot.Record 40 | ) 41 | 42 | func RegisterCreateEngines() { 43 | CreateEngines = map[string]CreateEngine{ 44 | "mutex": &createRR, 45 | "recursion": &createRRByChan, 46 | } 47 | } 48 | 49 | func GetCreateEngine(name string) CreateEngine { 50 | return CreateEngines[name] 51 | } 52 | 53 | func (crr *createRoundRobin) Index() uint64 { 54 | crr.RLock() 55 | defer crr.RUnlock() 56 | if crr.ClientIndex >= crr.ClientsSize { 57 | crr.ClientIndex = 0 58 | } 59 | return crr.ClientIndex 60 | } 61 | 62 | func (crr *createRoundRobin) CreateTimeFS(record *timedot.Record) { 63 | timefsClient.CreateTimeFS(clients[createRR.Index()], record) 64 | } 65 | 66 | func createTimeFSByChannel() { 67 | /* 68 | doing it this way will keep rotating between clients 69 | without keeping rotating index explicitly 70 | */ 71 | var record *timedot.Record 72 | for _, client := range clients { 73 | record = <-createChannel 74 | timefsClient.CreateTimeFS(client, record) 75 | } 76 | go createTimeFSByChannel() 77 | } 78 | 79 | func (crr *createRoundRobinByChannel) CreateTimeFS(record *timedot.Record) { 80 | createChannel <- record 81 | } 82 | -------------------------------------------------------------------------------- /splitter/timefsSplitter/read.go: -------------------------------------------------------------------------------- 1 | package timefsSplitter 2 | 3 | import ( 4 | "log" 5 | "sync" 6 | 7 | timefsClient "github.com/abhishekkr/timefs/client/timefsClient" 8 | timedot "github.com/abhishekkr/timefs/timedot" 9 | ) 10 | 11 | func getTimeFS(client *timedot.TimeFSClient, recordChan chan timedot.Record, recordChanX chan timedot.Record, filtr *timedot.Record, wgGetTimeFS *sync.WaitGroup) { 12 | defer wgGetTimeFS.Done() 13 | 14 | go timefsClient.GetTimeFS(client, filtr, recordChanX) 15 | 16 | for _record := range recordChanX { 17 | recordChan <- _record 18 | } 19 | log.Println("[+] done with", client) 20 | } 21 | 22 | func GetTimeFS(recordChan chan timedot.Record, filtr *timedot.Record) { 23 | var wgGetTimeFS sync.WaitGroup 24 | 25 | wgGetTimeFS.Add(len(clients)) 26 | for _, client := range clients { 27 | recordChanX := make(chan timedot.Record) 28 | go getTimeFS(client, recordChan, recordChanX, filtr, &wgGetTimeFS) 29 | } 30 | 31 | wgGetTimeFS.Wait() 32 | close(recordChan) 33 | log.Println("[+] stream ended for", filtr) 34 | } 35 | -------------------------------------------------------------------------------- /splitter/timefsSplitter/splitter.go: -------------------------------------------------------------------------------- 1 | package timefsSplitter 2 | 3 | import ( 4 | "log" 5 | "strconv" 6 | "strings" 7 | 8 | golenv "github.com/abhishekkr/gol/golenv" 9 | grpc "google.golang.org/grpc" 10 | 11 | timefsClient "github.com/abhishekkr/timefs/client/timefsClient" 12 | timedot "github.com/abhishekkr/timefs/timedot" 13 | ) 14 | 15 | var ( 16 | clientByChannelCount = golenv.OverrideIfEnv("TIMEFS_CLIENTBYCHANNEL_COUNT", "100") 17 | timefsBackends = golenv.OverrideIfEnv("TIMEFS_BACKENDS", "127.0.0.1:7999") 18 | 19 | clients []*timedot.TimeFSClient 20 | grpcClients []*grpc.ClientConn 21 | ) 22 | 23 | func init() { 24 | RegisterCreateEngines() 25 | 26 | if timefsBackends == "" { 27 | log.Println("no backends provided as of now, skipping connect backend and create-by-channel goroutines") 28 | return 29 | } 30 | 31 | ConnectBackends(timefsBackends) 32 | 33 | clientByChannelCountInt, err := strconv.Atoi(clientByChannelCount) 34 | if err != nil { 35 | log.Fatalln("failed to convert TIMEFS_CLIENTBYCHANNEL_COUNT value to int", err) 36 | } 37 | LaunchCreateByChannel(clientByChannelCountInt) 38 | } 39 | 40 | func ConnectBackends(backendCSV string) { 41 | links := strings.Split(backendCSV, ",") 42 | createRR.ClientsSize = uint64(len(links)) 43 | clients = make([]*timedot.TimeFSClient, createRR.ClientsSize) 44 | grpcClients = make([]*grpc.ClientConn, createRR.ClientsSize) 45 | 46 | for idx, link := range links { 47 | grpcClients[idx] = timefsClient.LinkOpen(link) 48 | 49 | client := timedot.NewTimeFSClient(grpcClients[idx]) 50 | log.Println("connecting to backend:", link) 51 | clients[idx] = &client 52 | } 53 | } 54 | 55 | func LaunchCreateByChannel(clientByChannelCountInt int) { 56 | createChannel = make(chan *timedot.Record) 57 | 58 | for idx := 0; idx < clientByChannelCountInt; idx++ { 59 | log.Println("creating background channel#", idx) 60 | go createTimeFSByChannel() 61 | } 62 | } 63 | 64 | func CloseBackends() { 65 | for _, client := range grpcClients { 66 | timefsClient.LinkClose(client) 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /timedot/ts.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. DO NOT EDIT. 2 | // source: ts.proto 3 | 4 | /* 5 | Package timedot is a generated protocol buffer package. 6 | 7 | It is generated from these files: 8 | ts.proto 9 | 10 | It has these top-level messages: 11 | Timedot 12 | TimedotSave 13 | TimedotDelete 14 | Record 15 | */ 16 | package timedot 17 | 18 | import proto "github.com/golang/protobuf/proto" 19 | import fmt "fmt" 20 | import math "math" 21 | 22 | import ( 23 | context "golang.org/x/net/context" 24 | grpc "google.golang.org/grpc" 25 | ) 26 | 27 | // Reference imports to suppress errors if they are not otherwise used. 28 | var _ = proto.Marshal 29 | var _ = fmt.Errorf 30 | var _ = math.Inf 31 | 32 | // This is a compile-time assertion to ensure that this generated file 33 | // is compatible with the proto package it is being compiled against. 34 | // A compilation error at this line likely means your copy of the 35 | // proto package needs to be updated. 36 | const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package 37 | 38 | type Timedot struct { 39 | Year int32 `protobuf:"varint,1,opt,name=year" json:"year,omitempty"` 40 | Month int32 `protobuf:"varint,2,opt,name=month" json:"month,omitempty"` 41 | Date int32 `protobuf:"varint,3,opt,name=date" json:"date,omitempty"` 42 | Hour int32 `protobuf:"varint,4,opt,name=hour" json:"hour,omitempty"` 43 | Minute int32 `protobuf:"varint,5,opt,name=minute" json:"minute,omitempty"` 44 | Second int32 `protobuf:"varint,6,opt,name=second" json:"second,omitempty"` 45 | Microsecond int32 `protobuf:"varint,7,opt,name=microsecond" json:"microsecond,omitempty"` 46 | } 47 | 48 | func (m *Timedot) Reset() { *m = Timedot{} } 49 | func (m *Timedot) String() string { return proto.CompactTextString(m) } 50 | func (*Timedot) ProtoMessage() {} 51 | func (*Timedot) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } 52 | 53 | func (m *Timedot) GetYear() int32 { 54 | if m != nil { 55 | return m.Year 56 | } 57 | return 0 58 | } 59 | 60 | func (m *Timedot) GetMonth() int32 { 61 | if m != nil { 62 | return m.Month 63 | } 64 | return 0 65 | } 66 | 67 | func (m *Timedot) GetDate() int32 { 68 | if m != nil { 69 | return m.Date 70 | } 71 | return 0 72 | } 73 | 74 | func (m *Timedot) GetHour() int32 { 75 | if m != nil { 76 | return m.Hour 77 | } 78 | return 0 79 | } 80 | 81 | func (m *Timedot) GetMinute() int32 { 82 | if m != nil { 83 | return m.Minute 84 | } 85 | return 0 86 | } 87 | 88 | func (m *Timedot) GetSecond() int32 { 89 | if m != nil { 90 | return m.Second 91 | } 92 | return 0 93 | } 94 | 95 | func (m *Timedot) GetMicrosecond() int32 { 96 | if m != nil { 97 | return m.Microsecond 98 | } 99 | return 0 100 | } 101 | 102 | type TimedotSave struct { 103 | Success bool `protobuf:"varint,1,opt,name=success" json:"success,omitempty"` 104 | } 105 | 106 | func (m *TimedotSave) Reset() { *m = TimedotSave{} } 107 | func (m *TimedotSave) String() string { return proto.CompactTextString(m) } 108 | func (*TimedotSave) ProtoMessage() {} 109 | func (*TimedotSave) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } 110 | 111 | func (m *TimedotSave) GetSuccess() bool { 112 | if m != nil { 113 | return m.Success 114 | } 115 | return false 116 | } 117 | 118 | type TimedotDelete struct { 119 | Success bool `protobuf:"varint,1,opt,name=success" json:"success,omitempty"` 120 | Count uint64 `protobuf:"varint,2,opt,name=count" json:"count,omitempty"` 121 | } 122 | 123 | func (m *TimedotDelete) Reset() { *m = TimedotDelete{} } 124 | func (m *TimedotDelete) String() string { return proto.CompactTextString(m) } 125 | func (*TimedotDelete) ProtoMessage() {} 126 | func (*TimedotDelete) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } 127 | 128 | func (m *TimedotDelete) GetSuccess() bool { 129 | if m != nil { 130 | return m.Success 131 | } 132 | return false 133 | } 134 | 135 | func (m *TimedotDelete) GetCount() uint64 { 136 | if m != nil { 137 | return m.Count 138 | } 139 | return 0 140 | } 141 | 142 | type Record struct { 143 | TopicKey string `protobuf:"bytes,1,opt,name=topicKey" json:"topicKey,omitempty"` 144 | TopicId string `protobuf:"bytes,2,opt,name=topicId" json:"topicId,omitempty"` 145 | Time []*Timedot `protobuf:"bytes,3,rep,name=Time,json=time" json:"Time,omitempty"` 146 | Value string `protobuf:"bytes,4,opt,name=value" json:"value,omitempty"` 147 | } 148 | 149 | func (m *Record) Reset() { *m = Record{} } 150 | func (m *Record) String() string { return proto.CompactTextString(m) } 151 | func (*Record) ProtoMessage() {} 152 | func (*Record) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} } 153 | 154 | func (m *Record) GetTopicKey() string { 155 | if m != nil { 156 | return m.TopicKey 157 | } 158 | return "" 159 | } 160 | 161 | func (m *Record) GetTopicId() string { 162 | if m != nil { 163 | return m.TopicId 164 | } 165 | return "" 166 | } 167 | 168 | func (m *Record) GetTime() []*Timedot { 169 | if m != nil { 170 | return m.Time 171 | } 172 | return nil 173 | } 174 | 175 | func (m *Record) GetValue() string { 176 | if m != nil { 177 | return m.Value 178 | } 179 | return "" 180 | } 181 | 182 | func init() { 183 | proto.RegisterType((*Timedot)(nil), "timedot.Timedot") 184 | proto.RegisterType((*TimedotSave)(nil), "timedot.TimedotSave") 185 | proto.RegisterType((*TimedotDelete)(nil), "timedot.TimedotDelete") 186 | proto.RegisterType((*Record)(nil), "timedot.Record") 187 | } 188 | 189 | // Reference imports to suppress errors if they are not otherwise used. 190 | var _ context.Context 191 | var _ grpc.ClientConn 192 | 193 | // This is a compile-time assertion to ensure that this generated file 194 | // is compatible with the grpc package it is being compiled against. 195 | const _ = grpc.SupportPackageIsVersion4 196 | 197 | // Client API for TimeFS service 198 | 199 | type TimeFSClient interface { 200 | CreateTimedot(ctx context.Context, in *Record, opts ...grpc.CallOption) (*TimedotSave, error) 201 | ReadTimedot(ctx context.Context, in *Record, opts ...grpc.CallOption) (TimeFS_ReadTimedotClient, error) 202 | DeleteTimedot(ctx context.Context, in *Record, opts ...grpc.CallOption) (*TimedotDelete, error) 203 | } 204 | 205 | type timeFSClient struct { 206 | cc *grpc.ClientConn 207 | } 208 | 209 | func NewTimeFSClient(cc *grpc.ClientConn) TimeFSClient { 210 | return &timeFSClient{cc} 211 | } 212 | 213 | func (c *timeFSClient) CreateTimedot(ctx context.Context, in *Record, opts ...grpc.CallOption) (*TimedotSave, error) { 214 | out := new(TimedotSave) 215 | err := grpc.Invoke(ctx, "/timedot.TimeFS/CreateTimedot", in, out, c.cc, opts...) 216 | if err != nil { 217 | return nil, err 218 | } 219 | return out, nil 220 | } 221 | 222 | func (c *timeFSClient) ReadTimedot(ctx context.Context, in *Record, opts ...grpc.CallOption) (TimeFS_ReadTimedotClient, error) { 223 | stream, err := grpc.NewClientStream(ctx, &_TimeFS_serviceDesc.Streams[0], c.cc, "/timedot.TimeFS/ReadTimedot", opts...) 224 | if err != nil { 225 | return nil, err 226 | } 227 | x := &timeFSReadTimedotClient{stream} 228 | if err := x.ClientStream.SendMsg(in); err != nil { 229 | return nil, err 230 | } 231 | if err := x.ClientStream.CloseSend(); err != nil { 232 | return nil, err 233 | } 234 | return x, nil 235 | } 236 | 237 | type TimeFS_ReadTimedotClient interface { 238 | Recv() (*Record, error) 239 | grpc.ClientStream 240 | } 241 | 242 | type timeFSReadTimedotClient struct { 243 | grpc.ClientStream 244 | } 245 | 246 | func (x *timeFSReadTimedotClient) Recv() (*Record, error) { 247 | m := new(Record) 248 | if err := x.ClientStream.RecvMsg(m); err != nil { 249 | return nil, err 250 | } 251 | return m, nil 252 | } 253 | 254 | func (c *timeFSClient) DeleteTimedot(ctx context.Context, in *Record, opts ...grpc.CallOption) (*TimedotDelete, error) { 255 | out := new(TimedotDelete) 256 | err := grpc.Invoke(ctx, "/timedot.TimeFS/DeleteTimedot", in, out, c.cc, opts...) 257 | if err != nil { 258 | return nil, err 259 | } 260 | return out, nil 261 | } 262 | 263 | // Server API for TimeFS service 264 | 265 | type TimeFSServer interface { 266 | CreateTimedot(context.Context, *Record) (*TimedotSave, error) 267 | ReadTimedot(*Record, TimeFS_ReadTimedotServer) error 268 | DeleteTimedot(context.Context, *Record) (*TimedotDelete, error) 269 | } 270 | 271 | func RegisterTimeFSServer(s *grpc.Server, srv TimeFSServer) { 272 | s.RegisterService(&_TimeFS_serviceDesc, srv) 273 | } 274 | 275 | func _TimeFS_CreateTimedot_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 276 | in := new(Record) 277 | if err := dec(in); err != nil { 278 | return nil, err 279 | } 280 | if interceptor == nil { 281 | return srv.(TimeFSServer).CreateTimedot(ctx, in) 282 | } 283 | info := &grpc.UnaryServerInfo{ 284 | Server: srv, 285 | FullMethod: "/timedot.TimeFS/CreateTimedot", 286 | } 287 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 288 | return srv.(TimeFSServer).CreateTimedot(ctx, req.(*Record)) 289 | } 290 | return interceptor(ctx, in, info, handler) 291 | } 292 | 293 | func _TimeFS_ReadTimedot_Handler(srv interface{}, stream grpc.ServerStream) error { 294 | m := new(Record) 295 | if err := stream.RecvMsg(m); err != nil { 296 | return err 297 | } 298 | return srv.(TimeFSServer).ReadTimedot(m, &timeFSReadTimedotServer{stream}) 299 | } 300 | 301 | type TimeFS_ReadTimedotServer interface { 302 | Send(*Record) error 303 | grpc.ServerStream 304 | } 305 | 306 | type timeFSReadTimedotServer struct { 307 | grpc.ServerStream 308 | } 309 | 310 | func (x *timeFSReadTimedotServer) Send(m *Record) error { 311 | return x.ServerStream.SendMsg(m) 312 | } 313 | 314 | func _TimeFS_DeleteTimedot_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 315 | in := new(Record) 316 | if err := dec(in); err != nil { 317 | return nil, err 318 | } 319 | if interceptor == nil { 320 | return srv.(TimeFSServer).DeleteTimedot(ctx, in) 321 | } 322 | info := &grpc.UnaryServerInfo{ 323 | Server: srv, 324 | FullMethod: "/timedot.TimeFS/DeleteTimedot", 325 | } 326 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 327 | return srv.(TimeFSServer).DeleteTimedot(ctx, req.(*Record)) 328 | } 329 | return interceptor(ctx, in, info, handler) 330 | } 331 | 332 | var _TimeFS_serviceDesc = grpc.ServiceDesc{ 333 | ServiceName: "timedot.TimeFS", 334 | HandlerType: (*TimeFSServer)(nil), 335 | Methods: []grpc.MethodDesc{ 336 | { 337 | MethodName: "CreateTimedot", 338 | Handler: _TimeFS_CreateTimedot_Handler, 339 | }, 340 | { 341 | MethodName: "DeleteTimedot", 342 | Handler: _TimeFS_DeleteTimedot_Handler, 343 | }, 344 | }, 345 | Streams: []grpc.StreamDesc{ 346 | { 347 | StreamName: "ReadTimedot", 348 | Handler: _TimeFS_ReadTimedot_Handler, 349 | ServerStreams: true, 350 | }, 351 | }, 352 | Metadata: "ts.proto", 353 | } 354 | 355 | func init() { proto.RegisterFile("ts.proto", fileDescriptor0) } 356 | 357 | var fileDescriptor0 = []byte{ 358 | // 331 bytes of a gzipped FileDescriptorProto 359 | 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x92, 0xc1, 0x6a, 0xc2, 0x40, 360 | 0x10, 0x86, 0x4d, 0x8d, 0x89, 0x4e, 0x90, 0x96, 0x41, 0x64, 0xf1, 0x24, 0x4b, 0xa1, 0x9e, 0xa4, 361 | 0xe8, 0xa5, 0xf4, 0xd2, 0x43, 0x4b, 0xa1, 0xf4, 0xb6, 0xf6, 0x05, 0xd2, 0xcd, 0x80, 0x01, 0x93, 362 | 0x95, 0x64, 0x23, 0xd8, 0x77, 0xea, 0xa9, 0x2f, 0x58, 0x66, 0x37, 0x4a, 0x09, 0xd8, 0xdb, 0xfc, 363 | 0xdf, 0xfe, 0x33, 0xfb, 0xef, 0x24, 0x30, 0xb4, 0xf5, 0x72, 0x5f, 0x19, 0x6b, 0x30, 0xb6, 0x79, 364 | 0x41, 0x99, 0xb1, 0xf2, 0x3b, 0x80, 0xf8, 0xc3, 0xd7, 0x88, 0x10, 0x1e, 0x29, 0xad, 0x44, 0x30, 365 | 0x0f, 0x16, 0x03, 0xe5, 0x6a, 0x9c, 0xc0, 0xa0, 0x30, 0xa5, 0xdd, 0x8a, 0x2b, 0x07, 0xbd, 0x60, 366 | 0x67, 0x96, 0x5a, 0x12, 0x7d, 0xef, 0xe4, 0x9a, 0xd9, 0xd6, 0x34, 0x95, 0x08, 0x3d, 0xe3, 0x1a, 367 | 0xa7, 0x10, 0x15, 0x79, 0xd9, 0x58, 0x12, 0x03, 0x47, 0x5b, 0xc5, 0xbc, 0x26, 0x6d, 0xca, 0x4c, 368 | 0x44, 0x9e, 0x7b, 0x85, 0x73, 0x48, 0x8a, 0x5c, 0x57, 0xa6, 0x3d, 0x8c, 0xdd, 0xe1, 0x5f, 0x24, 369 | 0xef, 0x20, 0x69, 0xe3, 0x6e, 0xd2, 0x03, 0xa1, 0x80, 0xb8, 0x6e, 0xb4, 0xa6, 0xba, 0x76, 0xa9, 370 | 0x87, 0xea, 0x24, 0xe5, 0x13, 0x8c, 0x5b, 0xe3, 0x0b, 0xed, 0xc8, 0xfe, 0x63, 0xe5, 0x37, 0x6a, 371 | 0xd3, 0x94, 0xd6, 0xbd, 0x31, 0x54, 0x5e, 0xc8, 0x2f, 0x88, 0x14, 0x69, 0x53, 0x65, 0x38, 0x83, 372 | 0xa1, 0x35, 0xfb, 0x5c, 0xbf, 0xd3, 0xd1, 0xb5, 0x8e, 0xd4, 0x59, 0xf3, 0x54, 0x57, 0xbf, 0x65, 373 | 0xae, 0x7b, 0xa4, 0x4e, 0x12, 0x6f, 0x21, 0xe4, 0x00, 0xa2, 0x3f, 0xef, 0x2f, 0x92, 0xd5, 0xcd, 374 | 0xb2, 0xdd, 0xf8, 0xb2, 0x4d, 0xa5, 0x42, 0x06, 0x7c, 0xf7, 0x21, 0xdd, 0x35, 0xe4, 0xd6, 0x36, 375 | 0x52, 0x5e, 0xac, 0x7e, 0x02, 0x88, 0xd8, 0xf7, 0xba, 0xc1, 0x07, 0x18, 0x3f, 0x57, 0x94, 0x5a, 376 | 0x3a, 0x7d, 0xa5, 0xeb, 0xf3, 0x24, 0x1f, 0x6f, 0x36, 0xe9, 0x8e, 0xe6, 0xcd, 0xc8, 0x1e, 0xae, 377 | 0x21, 0x51, 0x94, 0x66, 0x17, 0xfb, 0xba, 0x40, 0xf6, 0xee, 0x03, 0x7c, 0x84, 0xb1, 0xdf, 0xd7, 378 | 0xc5, 0xb6, 0x69, 0xf7, 0x3a, 0xef, 0x97, 0xbd, 0xcf, 0xc8, 0xfd, 0x5b, 0xeb, 0xdf, 0x00, 0x00, 379 | 0x00, 0xff, 0xff, 0xfb, 0x8d, 0x75, 0xfd, 0x67, 0x02, 0x00, 0x00, 380 | } 381 | -------------------------------------------------------------------------------- /timedot/ts.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package timedot; 4 | 5 | service TimeFS { 6 | rpc CreateTimedot(Record) returns (TimedotSave) {} 7 | rpc ReadTimedot(Record) returns (stream Record) {} 8 | rpc DeleteTimedot(Record) returns (TimedotDelete) {} 9 | } 10 | 11 | message Timedot { 12 | int32 year = 1; 13 | int32 month = 2; 14 | int32 date = 3; 15 | int32 hour = 4; 16 | int32 minute = 5; 17 | int32 second = 6; 18 | int32 microsecond = 7; 19 | } 20 | 21 | message TimedotSave { 22 | bool success = 1; 23 | } 24 | 25 | message TimedotDelete { 26 | bool success = 1; 27 | uint64 count = 2; 28 | } 29 | 30 | message Record { 31 | string topicKey = 1; 32 | string topicId = 2; 33 | repeated Timedot Time = 3; 34 | string value = 4; 35 | } 36 | --------------------------------------------------------------------------------