├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── LICENSE ├── README.md ├── dbtasks.go ├── dbtasks_del.go ├── dbtasks_get.go ├── dbtasks_packet.go ├── dbtasks_push.go ├── documents ├── goshare-over-http.md ├── goshare-over-zeromq.md ├── goshare-philosophy.md ├── goshare-quick-start.md └── goshare-user-concepts.md ├── flag_handler.go ├── go-get-pkg.txt ├── go-tasks.sh ├── go0mq.go ├── gohttp.go ├── goshare.go ├── goshare_config.json.sample ├── httpd ├── goshareF1.go └── public │ ├── concept.html │ ├── help-http.html │ ├── help-zmq.html │ ├── index.html │ ├── quickstart.html │ └── status.html ├── requestor └── packager.go ├── tests ├── go0mq_client.go ├── gohttp_client.go └── million_go0mq.go └── zxtra ├── goshare_daemon.go └── goshare_server.go /.gitignore: -------------------------------------------------------------------------------- 1 | *swo 2 | *swp 3 | *~ 4 | *.tmp 5 | temp/* 6 | .goenv 7 | bin/* 8 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.3 5 | - 1.2 6 | - tip 7 | 8 | install: 9 | - ./go-tasks.sh deps 10 | 11 | script: 12 | - ./go-tasks.sh test 13 | 14 | notifications: 15 | irc: "chat.freenode.net##timeseries" -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## Changelog 2 | 3 | ### v0.8.5 4 | 5 | * [db-tasks] moved to use golkeyval{,NS,TSDS} for switch-able db back-end 6 | * golint-ified goshare base code 7 | 8 | 9 | ### v0.8.4 10 | 11 | * [read,delete] enable {def,ns,tsds}-{,csv,json} to get multiple keys, this also define multi-val type for "read" output 12 | 13 | #### v0.8.3 14 | 15 | * ParentNS feature provided, provide Parent NameSpace for all keys (in any key-type) to be used while Push/Read/Delete 16 | * dbtasks moved to use Packet{}, abstract-ing out how data is received on multiple communication channels 17 | 18 | #### v0.8.2 19 | 20 | * [push] encoding for ns/tsds may contain a root parent-ns config to pre-pend for all ASCII multi-val 21 | 22 | --- 23 | 24 | #### v0.8.1 25 | 26 | * added json support 27 | * fixed multi-val support for non-tsds key-types 28 | 29 | #### v0.8.0 30 | 31 | * huge refactor and design fixes 32 | 33 | #### *Good To Have* 34 | 35 | > 36 | > * fetch all one-step {,all with val::} children in a particular key namespace, don't fetch all with-val:: children in a particular key namespace, 'cuz that can be done by doing ns and looking at all keys 37 | > 38 | > * (logging) "debug:default" log triggers only at failure status; "verbose" with only DBRest call logged again 39 | > 40 | > * [read] features for tsds like: {latest,this}_{year,month,week,day,hour,hour,min,dot} 41 | > 42 | > * [read] for tsds: mean, median, max, min, from_X_to_Y, radius_X_of_Y, more_than_X, less_than_X, matching_to_X, not_matching_to_X 43 | > 44 | 45 | #### *Design Decisions* 46 | 47 | > 48 | > * if task-type tokens need to grow to more than 3, move to msgpack for entire packet sent to dbtasks 49 | > 50 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014 AbhishekKr 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Go Share 2 | 3 | ```ASCII 4 | __ 5 | ____ _____ _____/ /_ ____ _________ 6 | / __ `/ __ \ / ___/ __ \/ __ `/ ___/ _ \ 7 | / /_/ / /_/ / (__ ) / / / /_/ / / / __/ 8 | \__, /\____/ /____/_/ /_/\__,_/_/ \___/ 9 | /____/ 10 | 11 | ``` 12 | 13 | [Tasks in Queue at Trello Board](https://trello.com/b/ZjDMRGQN/goshare) 14 | 15 | distributed under [MIT License](http://opensource.org/licenses/MIT) 16 | 17 | #### Go Share any data among the nodes. Over HTTP or ZeroMQ. 18 | 19 | * GOShare eases up communication over HTTP GET param based interaction. 20 | * ZeroMQ REQ/REP based synchronous communication model. 21 | 22 | it's "go get"-able 23 | 24 | ``` go get "github.com/abhishekkr/goshare" ``` 25 | 26 | *** 27 | 28 | #### Make Distributable Binary 29 | 30 | ```bash 31 | ./go-tasks.sh bin 32 | ``` 33 | 34 | This will create two distributable binaries *./bin/goshare_service* & *./bin/goshare_daemon*. Here *./bin/goshare_service* works as shown in README's 'Tryout' section. 35 | 36 | Whereas *./bin/goshare_daemon* can be used as a system service daemon, with following command line flags (along with flags mentioned in 'Tryout' section for ports and db-path) 37 | > 38 | > * start: ``` ./bin/goshare_daemon -daemon=start ``` 39 | > * stop: ``` ./bin/goshare_daemon -daemon=stop ``` 40 | > * status: ``` ./bin/goshare_daemon -daemon=status ``` 41 | > 42 | > *this dumps daemon's current status to /tmp/goshare_daemon.status and pid to /tmp/goshare_daemon.pid* 43 | > *the status and pid file path can be changed with flags '-daemon-log=' & '-daemon-pid=' respectively* 44 | > 45 | 46 | 47 | #### Tryout: 48 | 49 | ```Shell 50 | go run zxtra/goshare_daemon.go -DBPath=/tmp/GOTSDB 51 | ``` 52 | 53 | By default it runs HTTP daemon at port 9999 and ZeroMQ daemon at 9797/9898, 54 | make it run on another port using following required flags 55 | 56 | ```Shell 57 | go run zxtra/goshare_daemon.go -DBPath=/tmp/GOTSDB -port=8080 -req-port=8000 -rep-port=8001 58 | ``` 59 | 60 | ```ASCII 61 | Dummy Clients Using It 62 | 63 | * go run zxtra/gohttp_client.go 64 | 65 | * go run zxtra/go0mq_client.go 66 | 67 | 68 | for custom Port: 8080 for HTTP; Port: 8000/8001 for ZeroMQ 69 | 70 | * go run zxtra/gohttp_client.go -port=8080 71 | 72 | * go run zxtra/go0mq_client.go -req-port=8000 -rep-port=8001 73 | ``` 74 | 75 | > 76 | > To utilize it "zxtra/gohttp_client.go" and "zxtra/go0mq_client.go" can be referred on how to utilize capabilities of GoShare. 77 | > 78 | 79 | *** 80 | 81 | #### Structure: 82 | 83 | > "goshare"'s methods to adapt these in your code: 84 | > 85 | > * GoShare() : it runs HTTP and ZeroMQ daemon in parallel goroutines 86 | > > has optional flags customization of: 87 | > > * DBPath: path for LevelDB or other used (default: /tmp/GO.DB) 88 | > > * DBEngine: DB backend used (default: leveldb,; other options: sqlite3) 89 | > > * port: port to bind HTTP daemon (default: 9999) 90 | > > * req-port, rep-port: ports to bind ZeroMQ REQ/REP daemon (default: 9797, 9898) 91 | > 92 | > * GoShareHTTP(<levigo DB handle>, <http port as int>) : it runs HTTP daemon 93 | > 94 | > * GoShareZMQ(<levigo DB handle>, <req-port as int>, <rep-port as int>) : it runs ZMQ daemon 95 | > 96 | 97 | *** 98 | 99 | Now visit the the link asked by it and get the help page. 100 | 101 | ##### Dependency 102 | * [go lang](http://golang.org/doc/install) (obviously, the heart and soul of the app) 103 | * [leveldb](http://en.wikipedia.org/wiki/LevelDB) (we are using for datastore, it's awesome) 104 | * [levigo](https://github.com/jmhodges/levigo/blob/master/README.md) (the go library utilized to access leveldb) 105 | * [zeroMQ](http://zeromq.org/) (the supercharged Sockets giving REQuest/REPly power) 106 | * [gozmq](https://github.com/alecthomas/gozmq) GoLang ZeroMQ Bindings used here 107 | * [levigoNS](https://github.com/abhishekkr/levigoNS) NameSpace KeyVal capabilities around leveldb via levigo 108 | * [levigoTSDS](https://github.com/abhishekkr/levigoTSDS) TimeSeries KeyVal capabilties around leveldb via levigoNS 109 | * [gol](https://github.com/abhishekkr/gol) Set of common utility functionalities 110 | 111 | [![baby-gopher](https://raw2.github.com/drnic/babygopher-site/gh-pages/images/babygopher-badge.png)](http://www.babygopher.org) 112 | -------------------------------------------------------------------------------- /dbtasks.go: -------------------------------------------------------------------------------- 1 | package goshare 2 | 3 | /* 4 | [PATTERN] 5 | action {read, push, delete} 6 | type {default, ns, tsds, now} 7 | 8 | ## message_array here is devoided of axn and key_type 9 | non-tsds {key&val, :type-data} 10 | tsds(-*) {tdot&key&val, tdot&:type-data} 11 | */ 12 | 13 | /* 14 | DBTasks can be provided standard Packet Data Array from any communication Protocol. 15 | Communications handled on byte streams can use it by passing standard-ized packet-array 16 | It prepares Packet and passes on to TasksOnPacket. 17 | 0MQ directly utilizes it. 18 | */ 19 | func DBTasks(packetArray []string) ([]byte, bool) { 20 | packet := CreatePacket(packetArray) 21 | return DBTasksOnPacket(packet) 22 | } 23 | 24 | /* 25 | DBTasksOnPacket can utilize fromulated Packet. 26 | Communication can directly create packet and pass it here. 27 | HTTP directly utilizes it directly. 0MQ indirectly. 28 | */ 29 | func DBTasksOnPacket(packet Packet) ([]byte, bool) { 30 | response := "" 31 | axnStatus := false 32 | 33 | switch packet.DBAction { 34 | case "read": 35 | // returns axn error if key has empty value, if you gotta store then store, don't keep placeholders 36 | response = ReadFromPacket(packet) 37 | if response != "" { 38 | axnStatus = true 39 | } 40 | 41 | case "push": 42 | axnStatus = PushFromPacket(packet) 43 | 44 | case "delete": 45 | axnStatus = DeleteFromPacket(packet) 46 | } 47 | 48 | return []byte(response), axnStatus 49 | } 50 | -------------------------------------------------------------------------------- /dbtasks_del.go: -------------------------------------------------------------------------------- 1 | package goshare 2 | 3 | /* 4 | DelKey deletes val for a given key, returns status. 5 | */ 6 | func DelKey(key string) bool { 7 | return tsds.DelKey(key) 8 | } 9 | 10 | /* 11 | DelKeyNS deletes given key's namespace and all its values, returns status. 12 | */ 13 | func DelKeyNS(key string) bool { 14 | return tsds.DeleteNSRecursive(key) 15 | } 16 | 17 | /* 18 | DelKeyTSDS deletes all keys under given namespace, same as NS. 19 | As here TimeSeries is a NameSpace 20 | */ 21 | func DelKeyTSDS(key string) bool { 22 | return tsds.DeleteTSDS(key) 23 | } 24 | 25 | /* 26 | DeleteFuncByKeyType calls a delete action for a key based on task-type. 27 | */ 28 | func DeleteFuncByKeyType(keyType string) FunkAxnParamKey { 29 | switch keyType { 30 | case "tsds": 31 | return DelKeyTSDS 32 | 33 | case "ns": 34 | return DelKeyNS 35 | 36 | default: 37 | return DelKey 38 | 39 | } 40 | } 41 | 42 | /* 43 | DeleteFromPacket can handle multi-keys delete action, 44 | it acts on packet data. 45 | */ 46 | func DeleteFromPacket(packet Packet) bool { 47 | status := true 48 | axnFunk := DeleteFuncByKeyType(packet.KeyType) 49 | for _, _key := range packet.KeyList { 50 | status = status && axnFunk(_key) 51 | } 52 | 53 | return status 54 | } 55 | -------------------------------------------------------------------------------- /dbtasks_get.go: -------------------------------------------------------------------------------- 1 | package goshare 2 | 3 | import golhashmap "github.com/abhishekkr/gol/golhashmap" 4 | 5 | /* 6 | ReadKey gets value of given key. 7 | */ 8 | func ReadKey(key string) golhashmap.HashMap { 9 | var hashmap golhashmap.HashMap 10 | hashmap = make(golhashmap.HashMap) 11 | val := tsds.GetVal(key) 12 | if val == "" { 13 | return hashmap 14 | } 15 | hashmap[key] = val 16 | return hashmap 17 | } 18 | 19 | /* 20 | ReadKeyNS gets value for all descendents of given key's namespace. 21 | */ 22 | func ReadKeyNS(key string) golhashmap.HashMap { 23 | return tsds.ReadNSRecursive(key) 24 | } 25 | 26 | /* 27 | ReadKeyTSDS gets value for the asked time-frame key, aah same NS. 28 | */ 29 | func ReadKeyTSDS(key string) golhashmap.HashMap { 30 | return tsds.ReadTSDS(key) 31 | } 32 | 33 | /* 34 | ReadFuncByKeyType calls a read task on task-type. 35 | */ 36 | func ReadFuncByKeyType(keyType string) FunkAxnParamKeyReturnMap { 37 | switch keyType { 38 | case "tsds": 39 | return ReadKeyTSDS 40 | 41 | case "ns": 42 | return ReadKeyNS 43 | 44 | default: 45 | return ReadKey 46 | 47 | } 48 | } 49 | 50 | /* 51 | ReadFromPacket calls ReadFuncByKeyType for multi-keys based on provided packet. 52 | */ 53 | func ReadFromPacket(packet Packet) string { 54 | var response string 55 | var hashmap golhashmap.HashMap 56 | hashmap = make(golhashmap.HashMap) 57 | 58 | axnFunk := ReadFuncByKeyType(packet.KeyType) 59 | for _, _key := range packet.KeyList { 60 | hashmap = axnFunk(_key) 61 | if len(hashmap) == 0 { 62 | continue 63 | } 64 | response += responseByValType(packet.ValType, hashmap) 65 | } 66 | 67 | return response 68 | } 69 | 70 | /* transform response by ValType, if none default:csv */ 71 | func responseByValType(valType string, responseMap golhashmap.HashMap) string { 72 | var response string 73 | 74 | switch valType { 75 | case "csv", "json": 76 | hashmapEngine := golhashmap.GetHashMapEngine(valType) 77 | response = hashmapEngine.FromHashMap(responseMap) 78 | 79 | default: 80 | response = golhashmap.HashMapToCSV(responseMap) 81 | } 82 | return response 83 | } 84 | -------------------------------------------------------------------------------- /dbtasks_packet.go: -------------------------------------------------------------------------------- 1 | package goshare 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | golhashmap "github.com/abhishekkr/gol/golhashmap" 8 | gollist "github.com/abhishekkr/gol/gollist" 9 | goltime "github.com/abhishekkr/gol/goltime" 10 | ) 11 | 12 | /* 13 | Packet for modelling data passed to GoShare into a structure of possible fields. 14 | */ 15 | type Packet struct { 16 | DBAction string 17 | TaskType string 18 | 19 | KeyType string // key: default, namespace key: ns, timeseries key: tsds, timeseries for goshare time: now 20 | ValType string // single: default, csv, json 21 | 22 | HashMap golhashmap.HashMap 23 | KeyList []string 24 | 25 | ParentNS string // allowed for ns|tsds|now 26 | TimeDot goltime.Timestamp 27 | } 28 | 29 | /* 30 | FunkAxnParamKeyVal is a function type which get passed two string parameters 31 | and returns one boolean. Like Push Key-Val calls. 32 | */ 33 | type FunkAxnParamKeyVal func(key string, val string) bool 34 | 35 | /* 36 | FunkAxnParamKey is a function type which get passed one string parameter 37 | and returns one boolean. Like Del Key tasks. 38 | */ 39 | type FunkAxnParamKey func(key string) bool 40 | 41 | /* 42 | FunkAxnParamKeyReturnMap is a function type which get passed one string parameters 43 | and returns one hashmap. Like Get Key tasks. 44 | */ 45 | type FunkAxnParamKeyReturnMap func(key string) golhashmap.HashMap 46 | 47 | /* 48 | CreatePacket formulates Packet structure from passed message array. 49 | */ 50 | func CreatePacket(packetArray []string) Packet { 51 | packet := Packet{} 52 | packet.HashMap = make(golhashmap.HashMap) 53 | 54 | lenPacketArray := len(packetArray) 55 | if lenPacketArray < 3 { 56 | packet.DBAction = "ERROR" 57 | return packet 58 | } 59 | 60 | packet.DBAction = packetArray[0] 61 | packet.TaskType = packetArray[1] 62 | dataStartsFrom := 2 63 | 64 | taskTypeTokens := strings.Split(packet.TaskType, "-") 65 | packet.KeyType = taskTypeTokens[0] 66 | if packet.KeyType == "tsds" && packet.DBAction == "push" { 67 | if lenPacketArray < 9 { 68 | packet.DBAction = "ERROR" 69 | return packet 70 | } 71 | packet.TimeDot = goltime.CreateTimestamp(packetArray[2:8]) 72 | dataStartsFrom += 6 73 | } 74 | 75 | if len(taskTypeTokens) > 1 { 76 | packet.ValType = taskTypeTokens[1] 77 | 78 | if len(taskTypeTokens) == 3 { 79 | // if packet requirement grows more than 3, that's the limit 80 | // go get 'msgpack' to handle it instead... 81 | thirdTokenFeature(&packet, packetArray, &dataStartsFrom, taskTypeTokens[2]) 82 | } 83 | } 84 | 85 | decodeData(&packet, packetArray[dataStartsFrom:]) 86 | return packet 87 | } 88 | 89 | /* thirdTokenFeature handles special 3rd token feature, to populate Packet data. */ 90 | func thirdTokenFeature(packet *Packet, packetArray []string, dataStartsFrom *int, token string) { 91 | switch token { 92 | case "parent": 93 | packet.ParentNS = packetArray[*dataStartsFrom] 94 | (*dataStartsFrom)++ 95 | } 96 | } 97 | 98 | /* 99 | decodeData handles Packet formation based on DBAction. 100 | Handles TimeDot and pre-pending of ParentNS. 101 | */ 102 | func decodeData(packet *Packet, messageArray []string) { 103 | switch packet.DBAction { 104 | case "read", "delete": 105 | packet.KeyList = decodeKeyData(packet.ValType, messageArray) 106 | if packet.ParentNS != "" { 107 | PrefixKeyParentNamespace(packet) 108 | } 109 | 110 | case "push": 111 | packet.HashMap = decodeKeyValData(packet.ValType, messageArray) 112 | if packet.ParentNS != "" { 113 | PrefixKeyValParentNamespace(packet) 114 | } 115 | 116 | default: 117 | packet.DBAction = "ERROR" 118 | } 119 | } 120 | 121 | /* 122 | decodeKeyData handles Packet formation based on valType for GET, DELETE. 123 | */ 124 | func decodeKeyData(valType string, messageArray []string) []string { 125 | switch valType { 126 | case "csv", "json": 127 | multiValue := strings.Join(messageArray, "\n") 128 | listEngine := gollist.GetListEngine(valType) 129 | return listEngine.ToList(multiValue) 130 | 131 | default: 132 | return []string{messageArray[0]} 133 | } 134 | } 135 | 136 | /* 137 | decodeKeyValData handles Packet formation based on valType for PUSH. 138 | */ 139 | func decodeKeyValData(valType string, messageArray []string) golhashmap.HashMap { 140 | var hashmap golhashmap.HashMap 141 | hashmap = make(golhashmap.HashMap) 142 | 143 | switch valType { 144 | case "csv", "json": 145 | multiValue := strings.Join(messageArray, "\n") 146 | hashmapEngine := golhashmap.GetHashMapEngine(valType) 147 | hashmap = hashmapEngine.ToHashMap(multiValue) 148 | 149 | default: 150 | key := messageArray[0] 151 | value := strings.Join(messageArray[1:], " ") 152 | hashmap[key] = value 153 | } 154 | return hashmap 155 | } 156 | 157 | /* 158 | PrefixKeyParentNamespace prefixes Parent Namespaces to all keys in List 159 | if val for 'parentNamespace'. 160 | */ 161 | func PrefixKeyParentNamespace(packet *Packet) { 162 | var newList []string 163 | newList = make([]string, len(packet.KeyList)) 164 | 165 | parentNamespace := packet.ParentNS 166 | for idx, key := range packet.KeyList { 167 | newList[idx] = fmt.Sprintf("%s:%s", parentNamespace, key) 168 | } 169 | packet.KeyList = newList 170 | } 171 | 172 | /* 173 | PrefixKeyValParentNamespace prefixes Parent Namespaces to all key-val in HashMap 174 | if it has val for 'parentNamespace'. 175 | */ 176 | func PrefixKeyValParentNamespace(packet *Packet) { 177 | var newHmap golhashmap.HashMap 178 | newHmap = make(golhashmap.HashMap) 179 | 180 | parentNamespace := packet.ParentNS 181 | for key, val := range packet.HashMap { 182 | newHmap[fmt.Sprintf("%s:%s", parentNamespace, key)] = val 183 | } 184 | packet.HashMap = newHmap 185 | } 186 | -------------------------------------------------------------------------------- /dbtasks_push.go: -------------------------------------------------------------------------------- 1 | package goshare 2 | 3 | import "strings" 4 | 5 | /* 6 | PushKeyVal pushes a given set of Key-Val. 7 | */ 8 | func PushKeyVal(key string, val string) bool { 9 | return tsds.PushKeyVal(key, val) 10 | } 11 | 12 | /* 13 | PushKeyValNS pushes a given Namespace-Key and its value. 14 | */ 15 | func PushKeyValNS(key string, val string) bool { 16 | return tsds.PushNS(key, val) 17 | } 18 | 19 | /* 20 | PushKeyValNowTSDS pushes a key namespace-d with current time. 21 | */ 22 | func PushKeyValNowTSDS(key string, val string) bool { 23 | return tsds.PushNowTSDS(key, val) 24 | } 25 | 26 | /* 27 | PushKeyValTSDS pushes a key namespace-d with goltime.Timestamp. 28 | */ 29 | func PushKeyValTSDS(packet Packet) bool { 30 | status := true 31 | for _key, _val := range packet.HashMap { 32 | _val = strings.Replace(_val, "\n", " ", -1) 33 | status = status && tsds.PushTSDS(_key, _val, packet.TimeDot) 34 | } 35 | return status 36 | } 37 | 38 | /* 39 | PushFuncByKeyType returns func handle according to KeyType. 40 | */ 41 | func PushFuncByKeyType(keyType string) FunkAxnParamKeyVal { 42 | switch keyType { 43 | case "now": 44 | return PushKeyValNowTSDS 45 | 46 | case "ns": 47 | return PushKeyValNS 48 | 49 | default: 50 | return PushKeyVal 51 | 52 | } 53 | } 54 | 55 | /* 56 | PushFromPacket handles push task based on provided Packet. 57 | */ 58 | func PushFromPacket(packet Packet) bool { 59 | status := true 60 | switch packet.KeyType { 61 | case "tsds": 62 | PushKeyValTSDS(packet) 63 | 64 | default: 65 | axnFunk := PushFuncByKeyType(packet.KeyType) 66 | for _key, _val := range packet.HashMap { 67 | _val = strings.Replace(_val, "\n", " ", -1) 68 | status = status && axnFunk(_key, _val) 69 | } 70 | } 71 | 72 | return status 73 | } 74 | -------------------------------------------------------------------------------- /documents/goshare-over-http.md: -------------------------------------------------------------------------------- 1 | ## Using GoShare over HTTP 2 | 3 | #### By HTTP Method, send HTTP Request to "/db" as 4 | 5 | * POST/PUT method for "push" db-action 6 | * GET method for "read" db-action 7 | * DELETE method for "delete" db-action 8 | 9 | --- 10 | 11 | #### By Route, send GET method HTTP Requests to 12 | 13 | * "/put" for "push" db-action 14 | * "/get" for "read" db-action 15 | * "/del" for "delete" db-action 16 | 17 | --- 18 | 19 | URL form fields can be added for 20 | 21 | * "type": task-type {default,ns.tsds,now}-{,default,csv,json}-{,parentNS}; like ns-json or tsds-csv-parentNS 22 | * "key": for key to be used in "default" key-type 23 | * "val": for value to be used in "default" key-type on "push" db-action 24 | * "dbdata" : to specify data-string for val-types like csv, json 25 | * "parentNS": to specify ParentNamespace for all/any keys to specified in request 26 | * "year", "month", "day", "hour", "min", "sec" to specify all values of a timedot for key-type "tsds" 27 | 28 | > 29 | > for POST/PUT HTTP Method on /db, the field values can be provided via POST Body or URL fields 30 | > 31 | 32 | --- 33 | 34 | #### Examples 35 | 36 | all '%s' seen in URLs demand a suitable value for that field there 37 | 38 | * read value for a key 'anything' ``` http://:/get?type=default&key=anything ``` 39 | 40 | * push with full namespace of 'any:thing" value 'huh' ``` http://:/put?type=ns&key=any:thing&val=huh ``` 41 | 42 | * delete all self and child namespace for 'any' and 'all' ``` http://:/del?type=ns-csv&dbdata=any,all ``` 43 | 44 | * push 'up' for 'state' as timeseries for given time point ``` http://:/put?key=state&val=up&type=tsds&year=%s&month=%s&day=%s&hour=%s&min=%s&sec=%s ``` 45 | 46 | * push namespaced-key "name:first" with value "bob" ``` http://:/put?dbdata={\"name:first\":\"bob\"}&type=ns-json ``` 47 | 48 | * push timeseries value for timedot of being stored of key 'a' value 'A', key 'b' value 'B' ``` http://:/put?dbdata=a,A%0D%0Ab,B&type=now-csv ``` 49 | 50 | * to use a parent-namespace helper (prepend all keys with it) for keys ``` &parentNS=%s ``` 51 | 52 | --- 53 | -------------------------------------------------------------------------------- /documents/goshare-over-zeromq.md: -------------------------------------------------------------------------------- 1 | ## Using GoShare over HTTP 2 | 3 | #### By Word Stream 4 | 5 | ```ASCII 6 | 7 | [DB-Action] [Task-Type] ([Time-Dot]) ([Parent-NameSpace]) {[Key ([Val...])] OR [DB-Data...]} 8 | 9 | ``` 10 | 11 | the specifics for these components have been explained under [User Concepts]() before 12 | 13 | > * if you are coding in Golang, you can directly utilize ZmqRequest from "[golzmq](http://github.com/abhishekkr/gol/golzmq)" * 14 | > * otherwise, just prepare the word-stream and pass it as bytes in your favorite ZeroMQ Request library * 15 | 16 | --- 17 | 18 | #### Word Stream for major usable scenarios 19 | 20 | 21 | > simple Key Values 22 | 23 | * Push a simple Key Value ``` push default kernel linux ``` 24 | 25 | * Push multiple Key Value as CSV ``` push default-csv kernel,linux ``` 26 | 27 | * Push multiple Key Value as JSON ``` push default-json {"kernel":"linux"} ``` 28 | 29 | * Read a simple key, default response into CSV ``` read default kernel ``` 30 | 31 | * Read a simple key into JSON response ``` read default-json ["kernel"] ``` 32 | 33 | * Read multiple keys from JSON into JSON response ``` read default-json ["kernel","os"] ``` 34 | 35 | * Delete a simple key ``` delete default kernel ``` 36 | 37 | * Delete multiple keys from CSV ``` delete default-csv kernel,os ``` 38 | 39 | 40 | > Namespaced Key Values 41 | 42 | * Push a namespace key ``` push ns software:internet:browser chrome ``` 43 | 44 | * Push multiple namespace keys from CSV ``` push ns-csv software:internet:browser,chrome\nsoftware:internet:downloader,wget ``` 45 | 46 | * Push a namespace key with parent-namespace ``` push ns-default-parentNS software internet:browser chrome ``` 47 | 48 | * Push multiple namespace keys from JSON with parent-namespace ``` push ns-json-parentNS {"software:internet:browser":"chrome","software:internet:downloader":"wget"} ``` 49 | 50 | * Read all namespace value under given key, default as csv ``` read ns "software:internet" ``` 51 | 52 | * Read all namespace value under given key, fixed as csv ``` read ns-csv "software:internet" ``` 53 | 54 | * Read all namespace value under given key, response as JSON ``` read ns-json-parentNS software ["internet"] ``` 55 | 56 | * Delete all namespace value under given key ``` delete ns software:internet ``` 57 | 58 | 59 | > TimeSeries Key Values 60 | 61 | * Push a tsds key val ``` push tsds 2014 5 25 11 53 10 node:mymachine:memfree 4 ``` 62 | 63 | * Push multiple tsds keys from CSV ``` push tsds-csv 2014 5 25 11 53 10 node:mymachine:memfree,4\nnode:mymachine:diskfree,20 ``` 64 | 65 | * Push multiple NOW timeseries keys from JSON at TimeDot of getting stored ``` push now-json {"node:mymachine:memfree":"4","node:mymachine:diskfree":"20"} ``` 66 | 67 | * Push multiple tsds keys from JSON with parent-namespace ``` push tsds-json-parentNS 2014 5 25 11 53 10 node:mymachine {"memfree":"4","diskfree":"20"} ``` 68 | 69 | * Read all tsds values under given key, response as JSON ``` read tsds-json ["node:mymachine"] ``` 70 | 71 | * Read all tsds value under given key, response as JSON ``` read tsds-json-parentNS node ["mymachine","othermachine"] ``` 72 | 73 | * Delete all tsds value under given key ``` delete tsds node ``` 74 | 75 | --- 76 | -------------------------------------------------------------------------------- /documents/goshare-philosophy.md: -------------------------------------------------------------------------------- 1 | ## GoShare Philosophy 2 | 3 | ##### *(in short)* **GoShare can be used to have *persistent* Time-Series, Namespace-KeyVal or typical Key-Val data-store over HTTP and ZeroMQ communication capability as independent server (or built-in for Golang applications).** 4 | 5 | --- 6 | 7 | #### What the hell is it? 8 | GoShare is an engine for Time-Series Data-Store, a datastore tailor-fit built to suit the requirement of storing varying states of any type of attribute along the timeline. 9 | It's not a database built for generic purpose used for TimeSeries data, but a datastore which will keep on improving to serve only this kind of requirement from core up. 10 | 11 | --- 12 | 13 | #### *Uh*, which solution would even use it? 14 | It's kind of datastore which is perfect for requirements of a real-time monitoring system. Any kind of analytics system which requires change of attribute values along the stretch of a timeline could just plug-in and use it with a made for each other ease. 15 | There could be a lot more usable case for a timeseries datastore, these are the one I mainly ideated it for. 16 | 17 | --- 18 | 19 | #### *In the name of God*, why re-invent the wheel? 20 | There are not a page long list of alternatives for this. 21 | 22 | When I ideated it, the only solutions available were not self-dependent (using Zookeeper, MongoDB). Hence they were not actually alternative of an independent (born-for-it) Time-Series Data-Store. Also, I don't consider (stupid) closed-source projects for my FOSS initiatives. 23 | I needed such independent datastore for another monitoring project of mine where I didn't wanted to have more than one services to worry about for the core of my montitoring service. That would have like Watchmen for Watchmen for... 24 | Then later a project started (["InfluxDB"](http://influxdb.org/)) when I already started this. Which is in some way similar to this project but with much larger workforce behind it. It's also OpenSource, written in Golang. 25 | 26 | I didn't re-invented the entire wheel just the improved upon the design and made it suit the road. The core of this datastore sits and inter-woven key-val store ["leveldb"](https://code.google.com/p/leveldb/) (it's an awesome, made for performance key-val store mechanism by some (x)Googlers which also the core of super-awesome [Riak](http://basho.com/riak/)). 27 | 28 | I also wanted to give it an awesome super communication channel which is not there in other alternative last I checked. Currently it allows communication ZeroMQ Req/Rep connections along with HTTP calls. 29 | 30 | For data encoding, it supports key-val, csv and json for now. Msgpack (Protobuf and Cap'n'Proto) shall be in within a month or two (by July/2014). 31 | 32 | If you are thinking on "why CSV support". The main reason for that is since it was ideated with monitoring solution in mind, which aim on supporting agents on node supporting plugins for monitoring data. Plug-ins supported for written in any technology, even a bash script doing sed+grep over status files with most easier encoding available as csv. Idea is supporting wide range at features where it will help better & quicker adoption. 33 | 34 | Then we've the power of ZeroMQ to communicate with it (as mentioned before) enabling the awesomeness of supercharged socket communication with minimal formality possible. 35 | 36 | --- 37 | 38 | #### It's an Engine, so where is the Train? 39 | The repo itself provides an extra piece of code allowing creation of service/daemon binaries to start using it and trying the power real quick. 40 | 41 | As for a proper Train driving this engine, in parallel I've started design of [MomentDB](https://github.com/abhishekkr/momentdb) which will utilize it and enable having multiple style load-balanced, replicated and fail-over mechanisms for it. Just hold on for few months, design is almost done... development spike has started in pieces. 42 | 43 | The monitoring system which lead to its creation, has also seen the light of spiking. It can be kept track of at [ChaacMonitoring](https://github.com/ChaacMonitoring), but there is a very dumb structure there till now. It will show you GoShare in action but not of much utlization. 44 | 45 | --- 46 | 47 | #### It's not, but it is... 48 | 49 | It's not built for typical key-val or namespace key-val data storage but on the way of its construction. It also allows to use all the available data-encoding (and the cooler efficient binary ones to come) over HTTP and also awesome ZeroMQ for Key-Val and NameSpace Key-Val persistent storage. 50 | 51 | --- 52 | 53 | [Get QuickStart at GoShare](https://github.com/abhishekkr/goshare/wiki/QuickStart) 54 | -------------------------------------------------------------------------------- /documents/goshare-quick-start.md: -------------------------------------------------------------------------------- 1 | ## GoShare QuickStart 2 | 3 | #### Get the freakin' code 4 | 5 | * have git client, awesome it's easy ``` git clone https://github.com/abhishekkr/goshare ``` 6 | 7 | * have svn client, it's fine ``` svn co https://github.com/abhishekkr/goshare ``` 8 | 9 | * ahhh... have internet, download and uncompress ``` https://github.com/abhishekkr/goshare/archive/v0.8.4.tar.gz ``` 10 | 11 | --- 12 | 13 | #### Build your binaries 14 | 15 | Make cloned/checkout/downloaded 'goshare' code directory as your current directory. 16 | 17 | * have bash/zsh shell, cool ``` ./go-tasks.sh bin ``` 18 | 19 | * have golang at least 20 | > 21 | > * fetch all dependencies mentioned in 'go-get-pkg.txt' 22 | > * run at cli ``` mkdir ./bin ; cd ./bin ; go build ../zxtra/goshare_service.go ``` 23 | > 24 | 25 | --- 26 | 27 | #### Start it up 28 | 29 | 30 | * if you have a '/tmp' location with write access for current user 31 | > * run ``` ./bin/goshare_server ``` 32 | 33 | * otherwise 34 | > * run ``` ./bin/goshare_server -dbpath=$ANY_LOCATION_FORDB_CREATION ``` 35 | 36 | > 37 | > * HTTP listener at 0.0.0.0:9999 38 | > * ZeroMQ Request/Reply listener at 9898, 9797 39 | > 40 | > when started in non-daemon mode, these details get printed on command line as well 41 | > 42 | 43 | Now you can just run ``` go run zxtra/gohttp_client.go ``` to fire some read, push and delete calls to GoShare. Or make some of your own. 44 | 45 | > 46 | > You'll notice in a particular requirement -dbpath flag has been used to provide location of DB to be passed to GoShare. There are similar other flags available, which are: 47 | > 48 | > * dbpath : path to create database at; Default: "/tmp/GO.DB" 49 | > * http-uri : IP to connect while opening http port, Default: "0.0.0.0" 50 | > * http-port : Port to open http server connection for GoShare, Default: "9999" 51 | > * req-port : First port to bind ZeroMQ Request/Reply socket server, Default: "9797" 52 | > * rep-port : Second port to bind ZeroMQ Request/Reply socket server, Default: "9898" 53 | > * cpuprofile : file to log cpu profile data 54 | > 55 | > * config : json config file to provide all other flag value to override 56 | > 57 | 58 | > 59 | > The same set of configuration changes (or a portion of it) can be applied by a configuration file in JSON format, written like 60 | > ```JSON 61 | > {"http-port": "8080"} 62 | > ``` 63 | > 64 | 65 | So, here configuration applied by JSON file will override configuration applied by flags. If flags are not provided then the default value for them will be assigned to them. 66 | 67 | --- 68 | 69 | #### Ceate, Read, Delete 70 | 71 | These actions have been covered in detail at: 72 | * Create : [wiki]*to-be-written* 73 | * Read : [wiki]*to-be-written* 74 | * Delete : [wiki]*to-be-written* 75 | 76 | For this QuickStart section, let's try (assumption you have curl, else visit same URLs in your fav Browser you spoilt kid): 77 | 78 | ```SHELL 79 | # 80 | curl http://127.0.0.1:9999/get?type=default&key=name 81 | 82 | # 83 | curl http://127.0.0.1:9999/put?type=default&key=name&val=ledzep 84 | 85 | # 86 | curl http://127.0.0.1:9999/get?type=default&key=name 87 | 88 | # 89 | curl http://127.0.0.1:9999/put?type=ns&key=name:full&val=LedZep 90 | curl http://127.0.0.1:9999/put?type=ns-json&dbdata={\"name:first\":\"Led\",\"name:last\":\"Zep\"} 91 | 92 | # 93 | curl http://127.0.0.1:9999/get?type=ns&key=name 94 | 95 | # 96 | curl http://127.0.0.1:9999/del?type=ns&key=name 97 | 98 | # 99 | curl http://127.0.0.1:9999/get?type=ns&key=name 100 | 101 | # 102 | curl http://127.0.0.1:9999/put?type=now-json&dbdata={\"node01:webservice:state\":\"up\",\"node01:memfree\":\"4256783\"} 103 | 104 | # 105 | curl http://127.0.0.1:9999/get?type=ns&key=node01:webservice 106 | curl http://127.0.0.1:9999/get?type=ns&key=node01 107 | 108 | # 109 | curl http://127.0.0.1:9999/del?type=ns&key=node01 110 | 111 | # 112 | curl http://127.0.0.1:9999/get?type=ns&key=node01 113 | ``` 114 | 115 | GoShare's HTTP link also allows using POST/PUT for Create and DELETE for Delete tasks as method rather than route based approach. So, to each their own satisfaction. 116 | 117 | --- 118 | -------------------------------------------------------------------------------- /documents/goshare-user-concepts.md: -------------------------------------------------------------------------------- 1 | ## GoShare's Concept for Users 2 | 3 | A better look into how to use GoShare for your usability scenario. 4 | 5 | Learn how to use it over: 6 | * [HTTP]() at [wiki: GoShare over HTTP]() 7 | * [ZeroMQ]() at [wiki: GoShare over ZeroMQ]() 8 | 9 | --- 10 | 11 | ## GoShare Request 12 | 13 | it's constructed of following pieces 14 | 15 | ```ASCII 16 | 17 | [DB-Action] [Task-Type] ([Time-Dot]) ([Parent-NameSpace]) {[Key ([Val])] OR [DB-Data]} 18 | 19 | ``` 20 | 21 | --- 22 | 23 | ### DB-Action 24 | 25 | * 'push': to push provided key-val depending on task-type and Parent-Namespace 26 | * 'read': to read and return values for provided keys depending on task-type and Parent-Namespace 27 | * 'delete': to delete values for provided keys depending on task-type and Parent-Namespace 28 | 29 | --- 30 | 31 | ### Task-Type 32 | 33 | Task-Type is a string value telling goshare how to understand the data for the request and how to prepare response as well. 34 | 35 | Task-Type string can have maximum 3 tokens, separated by a hyphen '-'. The structure of its is 36 | 37 | ```ASCII 38 | 39 | KeyType-ValType-Helpers 40 | 41 | ``` 42 | 43 | 44 | #### What all '**KeyType**' key-val can be created? 45 | 46 | this helps GoShare understand what kin'of Keys are we dealing with from the following 47 | 48 | * 'default': create a 'key' for particula 'value' 49 | * 'ns': create a 'namespace' with a particular value, reachable via all parents 50 | * 'tsds': create a 'namespace for provided timedot[1]', reachable like 'ns' filtered from larger time unit to smaller 51 | * 'now': create a 'namespace for a timedot[1] mapped to time of its getting pushed', reachable like 'tsds' 52 | 53 | Here, 'now' KeyType is only usable for 'push' DB-Action. 54 | 55 | 56 | #### What all '**ValType**' data can be handled from Request? 57 | 58 | * same for ZeroMQ and HTTP 59 | > 60 | > * nil: opts for 'default' format 61 | > * 'default': handles Request-Default-Format[2], used for single key handling 62 | > * 'csv': handles just one data-string in Request-CSV-format[3], better for multiple keys 63 | > * 'json': handles just one data-string in Request-JSON-format[4], better for multiple keys 64 | > 65 | 66 | 67 | #### What all '**Helpers**' are available 68 | 69 | it let's you have some sanity around sent DB-Data and prepare it using the value from these helpers 70 | 71 | * 'parentNS': this let's you use only child-key-names in DB-Data and send Parent-Namespace for them separately 72 | > for example DB-Data 'name:first,anon' with ParentNS 'people', translates to 'people:name:first,anon' in DB 73 | 74 | when used, Second token for Task-Type can't be left blank for default, so use 'default' as ValType 75 | 76 | 77 | ##### Examples: 78 | 79 | * 'default': 80 | > push request will just provide a key and val as separate tokens, request depict status 81 | > read/delete request will have just a key, response will be csv of 'key,val' 82 | 83 | * 'default:default': 84 | > same like one before it for 'default' 85 | 86 | * 'ns-default'; 87 | > push request like 'default', but creates all parent namespace required for key 88 | > read/delete like 'default', but will clean up all key-vals under provided key 89 | 90 | * 'tsds-json': 91 | > push request will just provide a key-val as dictionary in JSON, request depict status, will also have all fields requird for 'timedot'[1] and keys will be namespace by them 92 | > read/delete request will have keys as list in JSON, response will be JSON of '{"key":"val",..}' 93 | 94 | * 'tsds-default-parentNS': 95 | > push request like 'ns-default', will have all fields required for 'timedot'[1] and key will be namespace by them 96 | > read/delete like 'ns-default', just uses key with power of parentNS 97 | 98 | * 'ns-csv-parentNS' 99 | > push request like 'ns-csv', will have all fields required for 'timedot'[1] and key will be namespace by them 100 | > read/delete like 'ns-csv', just uses key with power of parentNS 101 | 102 | --- 103 | 104 | ## GoShare Response 105 | 106 | 107 | #### What all '**ValType**' data can be handled for Response? 108 | 109 | * for "Push" (Create/Update) and Delete calls 110 | > 111 | > * on ZeroMQ: empty-string for Success, error-string for Failure 112 | > * on HTTP: "Success" for Success, error-string for Failure 113 | > 114 | 115 | * for Read calls (same on ZeroMQ and HTTP) 116 | > 117 | > * nil, 'default': opts for 'csv' Response-CSV-Format format 118 | > * 'csv': handles just one data-string in CSV-format, better for cli apps 119 | > * 'json': handles just one data-string in JSON-format, better for not-cli apps 120 | > 121 | > ** it uses the same ValType token for response in which the request has been sent ** 122 | > ** so if you want response in json, request in json... it's like want respect, better give respect ;) ** 123 | > 124 | 125 | --- 126 | 127 | ##### Clarifications if required: 128 | 129 | > 130 | > * [1] 'timedot': 131 | > > it's an identifiable point in time by provided Year, Month, Day, Hour, Minute and Second. 132 | > 133 | > * [2] 'Request-Default-Format': 134 | > > for '*PUSH*': it's two different fields for Key and Val, as in above ASCII-gram can be seen ```[Key ([Val])]``` 135 | > > for '*Read/Delete*': it's just one field for Key ```'[key]'``` 136 | > 137 | > * [3] 'Request-CSV-format': 138 | > > for '*PUSH*': one ('key,val') or multiple ('key,val\nkey2,val2') available in CSV format 139 | > > for '*Read/Delete*': one ('key') or multiple ('key1,key2,key3,key4') available in CSV format 140 | > 141 | > * [4] 'Request-JSON-format': 142 | > > for '*PUSH*': one ('{"key":"val"}') or multiple ('{"key":"val","key2":"val2"}') available in JSON format 143 | > > for '*Read/Delete*': one ('["key"]') or multiple ('["key1","key2","key3","key4"]') available in JSON format 144 | > 145 | 146 | * [A] Create/Update is same thing for GoShare. Just overwrites. 147 | -------------------------------------------------------------------------------- /flag_handler.go: -------------------------------------------------------------------------------- 1 | package goshare 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | 7 | "github.com/abhishekkr/gol/golconfig" 8 | ) 9 | 10 | // flags 11 | var ( 12 | flagConfig = flag.String("config", "", "the path to overriding config file") 13 | 14 | flagDBEngine = flag.String("DBEngine", "leveldb", "the type of KeyVal DB backend to be used") 15 | flagNSEngine = flag.String("NSEngine", "delimited", "the type of NameSpace DB backend to be used") 16 | flagTSEngine = flag.String("TSEngine", "namespace", "the type of TimeSeries backend to be used") 17 | 18 | flagDBPath = flag.String("DBPath", "/tmp/GO.DB", "the path to DB") 19 | flagServerUri = flag.String("server-uri", "0.0.0.0", "what Port to Run HTTP Server at") 20 | flagHTTPPort = flag.String("http-port", "9999", "what Port to Run HTTP Server at") 21 | flagRepPorts = flag.String("rep-ports", "9898,9797", "what PORT to run ZMQ REP at") 22 | flagCPUProfile = flag.String("cpuprofile", "", "write cpu profile to file") 23 | ) 24 | 25 | /* assignIfEmpty assigns val to *key only if it's empty */ 26 | func assignIfEmpty(mapper golconfig.FlatConfig, key string, val string) { 27 | if mapper[key] == "" { 28 | mapper[key] = val 29 | } 30 | } 31 | 32 | /* 33 | ConfigFromFlags configs from values provided to flags. 34 | */ 35 | func ConfigFromFlags() golconfig.FlatConfig { 36 | flag.Parse() 37 | 38 | var config golconfig.FlatConfig 39 | config = make(golconfig.FlatConfig) 40 | if *flagConfig != "" { 41 | configFile := golconfig.GetConfigurator("json") 42 | configFile.ConfigFromFile(*flagConfig, &config) 43 | } 44 | 45 | assignIfEmpty(config, "DBEngine", *flagDBEngine) 46 | assignIfEmpty(config, "NSEngine", *flagNSEngine) 47 | assignIfEmpty(config, "TSEngine", *flagTSEngine) 48 | assignIfEmpty(config, "DBPath", *flagDBPath) 49 | assignIfEmpty(config, "server-uri", *flagServerUri) 50 | assignIfEmpty(config, "http-port", *flagHTTPPort) 51 | assignIfEmpty(config, "rep-ports", *flagRepPorts) 52 | assignIfEmpty(config, "cpuprofile", *flagCPUProfile) 53 | 54 | fmt.Println("GoShare config:") 55 | for cfg, val := range config { 56 | fmt.Printf("[ %v : %v ]\n", cfg, val) 57 | } 58 | return config 59 | } 60 | -------------------------------------------------------------------------------- /go-get-pkg.txt: -------------------------------------------------------------------------------- 1 | -tags zmq_4_x github.com/alecthomas/gozmq 2 | github.com/abhishekkr/gol 3 | github.com/abhishekkr/gol/golkeyval 4 | github.com/abhishekkr/gol/golkeyvalNS 5 | github.com/abhishekkr/gol/golkeyvalTSDS 6 | github.com/abhishekkr/gol/golhashmap 7 | github.com/abhishekkr/gol/gollist 8 | github.com/abhishekkr/gol/gollog 9 | github.com/abhishekkr/gol/goltime 10 | -tags zmq_4_x github.com/abhishekkr/gol/golzmq 11 | -tags zmq_4_x github.com/abhishekkr/goshare 12 | github.com/VividCortex/godaemon 13 | -------------------------------------------------------------------------------- /go-tasks.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # managing go deps 4 | ############################################################################# 5 | ##### from github.com/abhishekkr/dotfiles/shell_profile/a.golang.sh 6 | goenv_on_at(){ 7 | if [ $# -eq 0 ]; then 8 | _GOPATH_VALUE="${PWD}/.goenv" 9 | else 10 | cd "$1" ; _GOPATH_VALUE="${1}/.goenv" ; cd - 11 | fi 12 | if [ ! -d $_GOPATH_VALUE ]; then 13 | mkdir -p "${_GOPATH_VALUE}/site" 14 | fi 15 | export _OLD_GOPATH=$GOPATH 16 | export _OLD_PATH=$PATH 17 | export GOPATH=$_GOPATH_VALUE/site 18 | export PATH=$PATH:$GOPATH/bin 19 | 20 | echo "your new GOPATH is at $GOPATH" 21 | } 22 | alias goenv_on="goenv_on_at \$PWD" 23 | alias goenv_off="export GOPATH=$_OLD_GOPATH ; export PATH=$_OLD_PATH ; unset _OLD_PATH ; unset _OLD_GOPATH" 24 | 25 | go_get_pkg_help(){ 26 | echo "go_get_pkg handles your Golang Project dependencies." 27 | echo "* Create new dependency list or install from existing:" 28 | echo " $ go_get_pkg" 29 | echo "* Install from existing with updated dependencies" 30 | echo " $ GO_GET_UPDATE=true go_get_pkg" 31 | echo "* Install from existing with re-prepared binaries (required on new Golang update or local changed dependency code)" 32 | echo " $ GO_GET_RENEW=true go_get_pkg" 33 | echo "* Install from existing with updated dependencies (re-prepared binaries even if no updates)" 34 | echo " $ GO_GET_RENEW=true GO_GET_UPDATE=true go_get_pkg" 35 | } 36 | go_get_pkg_list_create(){ 37 | if [ ! -f "$1" ]; then 38 | PKG_LISTS_DIR=$(dirname $PKG_LISTS) 39 | mkdir -p "$PKG_LISTS_DIR" && unset PKG_LISTS_DIR 40 | touch "${1}" 41 | echo "Created GoLang Package empty list ${PKG_LISTS}" 42 | echo "Start adding package paths as separate lines." 43 | return 0 44 | fi 45 | return 1 46 | } 47 | go_get_pkg_install(){ 48 | for pkg_list in $PKG_LISTS; do 49 | cat $pkg_list | while read pkg_path; do 50 | echo "fetching golag package: go get ${pkg_path}"; 51 | pkg_import_path=$(echo $pkg_path | awk '{print $NF}') 52 | if [[ ! -z $GO_GET_RENEW ]]; then 53 | rm -rf "${GOPATH}/pkg/${GOOS}_${GOARCH}/${pkg_import_path}" 54 | echo "cleaning old pkg for ${pkg_import_path}" 55 | fi 56 | if [[ -z $GO_GET_UPDATE ]]; then 57 | echo $pkg_path | xargs go get 58 | else 59 | echo $pkg_path | xargs go get -u 60 | fi 61 | done 62 | done 63 | 64 | unset GO_GET_UPDATE GO_GET_RENEW 65 | } 66 | go_get_pkg(){ 67 | if [[ "$1" == "help" ]]; then 68 | go_get_pkg_help 69 | return 0 70 | fi 71 | 72 | if [[ $# -eq 0 ]]; then 73 | PKG_LISTS="$PWD/go-get-pkg.txt" 74 | else 75 | PKG_LISTS=($@) 76 | if [[ -d "$PKG_LISTS" ]]; then 77 | PKG_LISTS="${PKG_LISTS}/go-get-pkg.txt" 78 | fi 79 | fi 80 | go_get_pkg_list_create $PKG_LISTS 81 | if [[ $? -eq 0 ]]; then 82 | return 0 83 | fi 84 | 85 | if [[ -z $GO_GET_ENV ]]; then 86 | _GO_GET_ENV=$(dirname $PKG_LISTS) 87 | GO_GET_ENV=$(cd $_GO_GET_ENV ; pwd ; cd - >/dev/null) 88 | fi 89 | goenv_on_at $GO_GET_ENV 90 | 91 | go_get_pkg_install "$PKG_LISTS" 92 | 93 | unset _GO_GET_ENV GO_GET_ENV PKG_LISTS 94 | } 95 | ############################################################################## 96 | 97 | 98 | _OLD_PWD=$PWD 99 | cd $(dirname $0) 100 | 101 | if [[ "$1" == "deps" ]]; then 102 | go_get_pkg 103 | 104 | elif [[ "$1" == "gorun" ]]; then 105 | go run -tags zmq_4_x "$2" 106 | 107 | elif [[ "$1" == "test" ]]; then 108 | $0 build 109 | echo 110 | echo "~~~~~Test Pieces~~~~~" 111 | go test ./... 112 | echo 113 | echo "~~~~~Test Features~~~~~" 114 | for feature_test in `ls ./tests/go*_client.go`; do 115 | echo ">> Testing: "$feature_test 116 | ./bin/goshare_daemon -daemon=start -dbpath=/tmp/GOSHARE.TEST.DB 117 | go run $feature_test 118 | ./bin/goshare_daemon -daemon=stop 119 | rm -rf /tmp/GOSHARE.TEST.DB 120 | done 121 | 122 | elif [[ "$1" == "wiki" ]]; then 123 | $0 bin 124 | echo 125 | echo "~~~~~Visit wiki at GoShare HTTP~~~~~" 126 | echo "~~~~~ http://0.0.0.0:9999 ~~~~~" 127 | ./bin/goshare_server 128 | 129 | elif [[ "$1" == "build" ]]; then 130 | bash $0 deps 131 | goenv_on_at $PWD 132 | mkdir -p ./bin 133 | cd ./bin 134 | for go_code_to_build in `ls ../zxtra/goshare_*.go`; do 135 | echo "Building: "$go_code_to_build 136 | go build -tags zmq_4_x $go_code_to_build 137 | done 138 | 139 | else 140 | echo "Use it wisely..." 141 | echo "Build usable binaries: '$0 build'" 142 | echo "Install tall Go lib dependencies: '$0 deps'" 143 | echo "Run all Tests: '$0 test'" 144 | 145 | fi 146 | 147 | cd $_OLD_PWD 148 | -------------------------------------------------------------------------------- /go0mq.go: -------------------------------------------------------------------------------- 1 | package goshare 2 | 3 | import ( 4 | "fmt" 5 | "runtime" 6 | "strings" 7 | 8 | zmq "github.com/alecthomas/gozmq" 9 | 10 | golzmq "github.com/abhishekkr/gol/golzmq" 11 | ) 12 | 13 | /* 14 | goShareZmqRep handles Read/Push/Delete tasks diversion based on task-type. 15 | */ 16 | func goShareZmqRep(socket *zmq.Socket) { 17 | var errResponse string 18 | for { 19 | msg, _ := socket.Recv(0) 20 | messageArray := strings.Fields(string(msg)) 21 | responseBytes, axnStatus := DBTasks(messageArray) 22 | 23 | if axnStatus { 24 | socket.Send([]byte(responseBytes), 0) 25 | } else { 26 | errResponse = fmt.Sprintf("Error for request sent: %s", msg) 27 | socket.Send([]byte(errResponse), 0) 28 | } 29 | } 30 | } 31 | 32 | /* 33 | GoShareZMQ starts a Daemon communicating of provided array ports over ZMQ Reply. 34 | */ 35 | func GoShareZMQ(ip string, replyPorts []int) { 36 | fmt.Printf("starting ZeroMQ REP/REQ at %v\n", replyPorts) 37 | runtime.GOMAXPROCS(runtime.NumCPU()) 38 | 39 | socket := golzmq.ZmqReplySocket(ip, replyPorts) 40 | goShareZmqRep(socket) 41 | } 42 | -------------------------------------------------------------------------------- /gohttp.go: -------------------------------------------------------------------------------- 1 | package goshare 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "runtime" 7 | "strings" 8 | "time" 9 | 10 | golhashmap "github.com/abhishekkr/gol/golhashmap" 11 | goltime "github.com/abhishekkr/gol/goltime" 12 | "github.com/abhishekkr/goshare/httpd" 13 | ) 14 | 15 | /* 16 | DBRest enables HTTP to formulate DBTasks call from HTTP Request. 17 | */ 18 | func DBRest(httpMethod string, w http.ResponseWriter, req *http.Request) { 19 | var ( 20 | dbAction string 21 | responseBytes []byte 22 | axnStatus bool 23 | ) 24 | 25 | switch httpMethod { 26 | case "GET": 27 | dbAction = "read" 28 | 29 | case "POST", "PUT": 30 | dbAction = "push" 31 | 32 | case "DELETE": 33 | dbAction = "delete" 34 | 35 | default: 36 | // log_this corrupt request 37 | return 38 | } 39 | 40 | packet := PacketFromHTTPRequest(dbAction, req) 41 | if packet.DBAction != "" { 42 | responseBytes, axnStatus = DBTasksOnPacket(packet) 43 | DBRestResponse(w, req, responseBytes, axnStatus) 44 | } 45 | } 46 | 47 | /* 48 | DBRestResponse sends proper response back to client based on success, data or error. 49 | */ 50 | func DBRestResponse(w http.ResponseWriter, req *http.Request, responseBytes []byte, axnStatus bool) { 51 | if !axnStatus { 52 | errorMsg := fmt.Sprintf("FATAL Error: (DBTasks) %q", req.Form) 53 | http.Error(w, errorMsg, http.StatusInternalServerError) 54 | 55 | } else if len(responseBytes) == 0 { 56 | w.Write([]byte("Success")) 57 | 58 | } else { 59 | w.Write(responseBytes) 60 | 61 | } 62 | } 63 | 64 | /* 65 | PacketFromHTTPRequest return Packet identifiable by DBTasksOnAction. 66 | */ 67 | func PacketFromHTTPRequest(dbAction string, req *http.Request) Packet { 68 | packet := Packet{} 69 | packet.HashMap = make(golhashmap.HashMap) 70 | packet.DBAction = dbAction 71 | 72 | req.ParseForm() 73 | taskType := req.FormValue("type") 74 | if taskType == "" { 75 | taskType = "default" 76 | } 77 | packet.TaskType = taskType 78 | taskTypeTokens := strings.Split(taskType, "-") 79 | packet.KeyType = taskTypeTokens[0] 80 | if packet.KeyType == "tsds" && packet.DBAction == "push" { 81 | packet.TimeDot = goltime.TimestampFromHTTPRequest(req) 82 | } 83 | 84 | if len(taskTypeTokens) > 1 { 85 | packet.ValType = taskTypeTokens[1] 86 | 87 | if len(taskTypeTokens) == 3 { 88 | thirdTokenFeatureHTTP(&packet, req) 89 | } 90 | } 91 | 92 | dbdata := req.FormValue("dbdata") 93 | key := req.FormValue("key") 94 | if key != "" { 95 | dbdata = fmt.Sprintf("%s\n%s", key, req.FormValue("val")) 96 | } else if dbdata == "" { 97 | return Packet{} 98 | } 99 | decodeData(&packet, strings.Split(dbdata, "\n")) 100 | 101 | return packet 102 | } 103 | 104 | /* 105 | thirdTokenFeatureHTTP handles third token in taskType. 106 | */ 107 | func thirdTokenFeatureHTTP(packet *Packet, req *http.Request) { 108 | parentNS := req.FormValue("parentNS") 109 | if parentNS != "" { 110 | packet.ParentNS = parentNS 111 | } 112 | } 113 | 114 | /* 115 | DBRestHandler handles DB Call for HTTP Request Method at '/db'. 116 | */ 117 | func DBRestHandler(w http.ResponseWriter, req *http.Request) { 118 | w.Header().Set("Content-Type", "text/plain") 119 | req.ParseForm() 120 | 121 | DBRest(req.Method, w, req) 122 | } 123 | 124 | /* 125 | GetReadKey HTTP GET DB-GET call handler at '/get'. 126 | */ 127 | func GetReadKey(w http.ResponseWriter, req *http.Request) { 128 | w.Header().Set("Content-Type", "text/plain") 129 | req.ParseForm() 130 | 131 | DBRest("GET", w, req) 132 | } 133 | 134 | /* 135 | GetPushKey HTTP GET DB-POST call handler at '/put'. 136 | */ 137 | func GetPushKey(w http.ResponseWriter, req *http.Request) { 138 | w.Header().Set("Content-Type", "text/plain") 139 | req.ParseForm() 140 | 141 | DBRest("POST", w, req) 142 | } 143 | 144 | /* 145 | GetDeleteKey HTTP GET DB-POST call handler at 'del'. 146 | */ 147 | func GetDeleteKey(w http.ResponseWriter, req *http.Request) { 148 | w.Header().Set("Content-Type", "text/plain") 149 | req.ParseForm() 150 | 151 | DBRest("DELETE", w, req) 152 | } 153 | 154 | /* 155 | GoShareHTTP handles all valid HTTP Requests for DBTasks, documentation 156 | and playground(WIP). 157 | */ 158 | func GoShareHTTP(httpuri string, httpport int) { 159 | runtime.GOMAXPROCS(runtime.NumCPU()) 160 | 161 | http.HandleFunc("/", abkhttpd.Index) 162 | http.HandleFunc("/quickstart", abkhttpd.QuickStart) 163 | http.HandleFunc("/help-http", abkhttpd.HelpHTTP) 164 | http.HandleFunc("/help-zmq", abkhttpd.HelpZMQ) 165 | http.HandleFunc("/concept", abkhttpd.Concept) 166 | http.HandleFunc("/status", abkhttpd.Status) 167 | 168 | http.HandleFunc("/db", DBRestHandler) 169 | http.HandleFunc("/get", GetReadKey) 170 | http.HandleFunc("/put", GetPushKey) 171 | http.HandleFunc("/del", GetDeleteKey) 172 | 173 | srv := &http.Server{ 174 | Addr: fmt.Sprintf("%s:%d", httpuri, httpport), 175 | Handler: http.DefaultServeMux, 176 | ReadTimeout: time.Duration(5) * time.Second, 177 | } 178 | 179 | fmt.Printf("access your goshare at http://%s:%d\n", httpuri, httpport) 180 | err := srv.ListenAndServe() 181 | fmt.Println("Game Over:", err) 182 | } 183 | -------------------------------------------------------------------------------- /goshare.go: -------------------------------------------------------------------------------- 1 | package goshare 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "runtime" 8 | "runtime/pprof" 9 | "strconv" 10 | "time" 11 | 12 | golconfig "github.com/abhishekkr/gol/golconfig" 13 | golerror "github.com/abhishekkr/gol/golerror" 14 | golkeyvalTSDS "github.com/abhishekkr/gol/golkeyvalTSDS" 15 | gollist "github.com/abhishekkr/gol/gollist" 16 | ) 17 | 18 | var ( 19 | tsds golkeyvalTSDS.TSDSDBEngine 20 | ) 21 | 22 | /* banner just brand print */ 23 | func banner() { 24 | fmt.Println("**************************************************") 25 | fmt.Println(" ___ ____ ___ __ _ __") 26 | fmt.Println(" | | | | | | / \\ | ) |") 27 | fmt.Println(" | =| | | =~ |==| |==| |==| |= |=") 28 | fmt.Println(" |__| |__| ___| | | | | | \\ |__") 29 | fmt.Println("") 30 | fmt.Println("**************************************************") 31 | } 32 | 33 | /* 34 | DoYouWannaContinue checking if you still wanna keep the goshare up. 35 | */ 36 | func DoYouWannaContinue() { 37 | var input string 38 | for { 39 | fmt.Println("Do you wanna exit. (yes|no):\n\n") 40 | 41 | fmt.Scanf("%s", &input) 42 | 43 | if input == "yes" || input == "y" { 44 | break 45 | } 46 | } 47 | } 48 | 49 | /* 50 | goshareDB returns golkeyval DBEngine for it. 51 | */ 52 | func goshareDB(config golconfig.FlatConfig) { 53 | if config["TSEngine"] == "namespace" { 54 | tsds = golkeyvalTSDS.GetNamespaceEngine(config) 55 | } else { 56 | panic("Unhandled TimeSeries Engine required.") 57 | } 58 | } 59 | 60 | /* 61 | GoShareEngine putting together base engine for GoShare as per config. 62 | dbpath, server_uri, httpport, rep_port, *string 63 | */ 64 | func GoShareEngine(config golconfig.FlatConfig) { 65 | runtime.GOMAXPROCS(runtime.NumCPU()) 66 | 67 | // remember it will be same DB instance shared across goshare package 68 | goshareDB(config) 69 | 70 | if config["cpuprofile"] != "" { 71 | f, err := os.Create(config["cpuprofile"]) 72 | if err != nil { 73 | log.Fatal(err) 74 | } 75 | pprof.StartCPUProfile(f) 76 | go func() { 77 | time.Sleep(100 * time.Second) 78 | pprof.StopCPUProfile() 79 | }() 80 | } 81 | 82 | _httpPort, errHTTPPort := strconv.Atoi(config["http-port"]) 83 | _replyPorts, errReplyPorts := gollist.CSVToNumbers(config["rep-ports"]) 84 | if errHTTPPort == nil && errReplyPorts == nil { 85 | go GoShareHTTP(config["server-uri"], _httpPort) 86 | go GoShareZMQ(config["server-uri"], _replyPorts) 87 | } else { 88 | golerror.Boohoo("Port parameters to bind, error-ed while conversion to number.", true) 89 | } 90 | } 91 | 92 | /* 93 | GoShare is daddy-o of goshare instance. 94 | */ 95 | func GoShare() { 96 | banner() 97 | runtime.GOMAXPROCS(runtime.NumCPU()) 98 | GoShareEngine(ConfigFromFlags()) 99 | DoYouWannaContinue() 100 | } 101 | -------------------------------------------------------------------------------- /goshare_config.json.sample: -------------------------------------------------------------------------------- 1 | { 2 | "dbpath": "/tmp/some.db", 3 | "server-uri": "0.0.0.0", 4 | "rep-ports": "9898,9897,9896,9895,9894,9893,9892,9891,9890", 5 | "http-port": "9001" 6 | } 7 | -------------------------------------------------------------------------------- /httpd/goshareF1.go: -------------------------------------------------------------------------------- 1 | package abkhttpd 2 | 3 | import ( 4 | "html/template" 5 | "net/http" 6 | ) 7 | 8 | func Index(w http.ResponseWriter, req *http.Request) { 9 | w.Header().Set("Content-Type", "text/html") 10 | 11 | t, err := template.ParseFiles("httpd/public/index.html") 12 | if err != nil { 13 | w.Write([]byte(err.Error())) 14 | return 15 | } 16 | t.Execute(w, nil) 17 | } 18 | 19 | func QuickStart(w http.ResponseWriter, req *http.Request) { 20 | w.Header().Set("Content-Type", "text/html") 21 | 22 | t, err := template.ParseFiles("httpd/public/quickstart.html") 23 | if err != nil { 24 | w.Write([]byte(err.Error())) 25 | return 26 | } 27 | t.Execute(w, nil) 28 | } 29 | 30 | func HelpHTTP(w http.ResponseWriter, req *http.Request) { 31 | w.Header().Set("Content-Type", "text/html") 32 | 33 | t, err := template.ParseFiles("httpd/public/help-http.html") 34 | if err != nil { 35 | w.Write([]byte(err.Error())) 36 | return 37 | } 38 | t.Execute(w, nil) 39 | } 40 | 41 | func HelpZMQ(w http.ResponseWriter, req *http.Request) { 42 | w.Header().Set("Content-Type", "text/html") 43 | 44 | t, err := template.ParseFiles("httpd/public/help-zmq.html") 45 | if err != nil { 46 | w.Write([]byte(err.Error())) 47 | return 48 | } 49 | t.Execute(w, nil) 50 | } 51 | 52 | func Concept(w http.ResponseWriter, req *http.Request) { 53 | w.Header().Set("Content-Type", "text/html") 54 | 55 | t, err := template.ParseFiles("httpd/public/concept.html") 56 | if err != nil { 57 | w.Write([]byte(err.Error())) 58 | return 59 | } 60 | t.Execute(w, nil) 61 | } 62 | 63 | func Status(w http.ResponseWriter, req *http.Request) { 64 | w.Header().Set("Content-Type", "text/html") 65 | 66 | t, err := template.ParseFiles("httpd/public/status.html") 67 | if err != nil { 68 | w.Write([]byte(err.Error())) 69 | return 70 | } 71 | t.Execute(w, nil) 72 | } 73 | -------------------------------------------------------------------------------- /httpd/public/concept.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | GoShare User Concepts 5 | 6 | 7 |

GoShare ~ persistent (TimeSeries/Namespace/Simple) KeyVal storage

8 | 9 | 

 10 |                         __                      ___
 11 |    ____  ____     _____/ /_  ____  ____,,__    /      ___   .  .   __  __  __  _____  ____
 12 |   / __ `/ __ \   / ___/ __ \/ __ `/ ___/ _ \  |      /   \  |\ |  /   |   |  |   |   |
 13 |  / /_/ / /_/ /  (__  ) / / / /_/ / /  /  __/  |     |     | | \| |    |-- |--'   |   '---,
 14 |  \__, /\____/  /____/_/ /_/\__,_/_/   \___/   |      \___/  |  |  \__ |__ |      |   ____|
 15 | /____/                                         \___
 16 | 
 17 |
18 | 19 |

Go Share User Concepts

20 | Home | QuickStart | over HTTP | over ZeroMQ | Status 21 |
22 |
23 |

A better look into how to use GoShare for your usability scenario.

24 | 25 |

Learn how to use it over: 26 | * HTTP at wiki: GoShare over HTTP 27 | * ZeroMQ at wiki: GoShare over ZeroMQ

28 | 29 |
30 | 31 |

GoShare Request

32 | 33 |

it's constructed of following pieces

34 | 35 |

36 | [DB-Action] [Task-Type] ([Time-Dot]) ([Parent-NameSpace]) {[Key ([Val])] OR [DB-Data]} 37 |

38 | 39 |
40 | 41 |

DB-Action

42 | 43 |
    44 |
  • 'push': to push provided key-val depending on task-type and Parent-Namespace
    • 45 |
  • 'read': to read and return values for provided keys depending on task-type and Parent-Namespace
    • 46 |
  • 'delete': to delete values for provided keys depending on task-type and Parent-Namespace
    • 47 |
48 | 49 |
50 | 51 |

Task-Type

52 | 53 |

Task-Type is a string value telling goshare how to understand the data for the request and how to prepare response as well.

54 | 55 |

Task-Type string can have maximum 3 tokens, separated by a hyphen '-'. The structure of its is

56 | 57 |

KeyType-ValType-Helpers

58 | 59 |

What all 'KeyType' key-val can be created?

60 | 61 |

this helps GoShare understand what kin'of Keys are we dealing with from the following

62 | 63 |
    64 |
  • 'default': create a 'key' for particula 'value'
    • 65 |
  • 'ns': create a 'namespace' with a particular value, reachable via all parents
    • 66 |
  • 'tsds': create a 'namespace for provided timedot[1]', reachable like 'ns' filtered from larger time unit to smaller
    • 67 |
  • 'now': create a 'namespace for a timedot[1] mapped to time of its getting pushed', reachable like 'tsds'
    • 68 |
69 | 70 |

Here, 'now' KeyType is only usable for 'push' DB-Action.

71 | 72 |

What all 'ValType' data can be handled from Request?

73 | 74 | same for ZeroMQ and HTTP 75 | 76 |
    77 |
  • nil: opts for 'default' format
    • 78 |
  • 'default': handles Request-Default-Format[2], used for single key handling
    • 79 |
  • 'csv': handles just one data-string in Request-CSV-format[3], better for multiple keys
    • 80 |
  • 'json': handles just one data-string in Request-JSON-format[4], better for multiple keys 81 |
    • 82 |
83 | 84 | 85 |

What all 'Helpers' are available

86 | 87 | it let's you have some sanity around sent DB-Data and prepare it using the value from these helpers 88 | 89 |
    90 |
  • 'parentNS': this let's you use only child-key-names in DB-Data and send Parent-Namespace for them separately 91 | 92 | 93 |
    94 |

    for example DB-Data 'name:first,anon' with ParentNS 'people', translates to 'people:name:first,anon' in DB

    • 95 |
96 | when used, Second token for Task-Type can't be left blank for default, so use 'default' as ValType

97 | 98 | 99 | Examples: 100 | 101 |
    102 |

  • 'default':

    103 | 104 |
    105 |

    push request will just provide a key and val as separate tokens, request depict status 106 | read/delete request will have just a key, response will be csv of 'key,val'

    107 |
    • 108 |

  • 'default:default':

    109 | 110 |
    111 |

    same like one before it for 'default'

    112 |
    • 113 |

  • 'ns-default';

    114 | 115 |
    116 |

    push request like 'default', but creates all parent namespace required for key 117 | read/delete like 'default', but will clean up all key-vals under provided key

    118 |
    • 119 |

  • 'tsds-json':

    120 | 121 |
    122 |

    push request will just provide a key-val as dictionary in JSON, request depict status, will also have all fields requird for 'timedot'[1] and keys will be namespace by them 123 | read/delete request will have keys as list in JSON, response will be JSON of '{"key":"val",..}'

    124 |
    • 125 |

  • 'tsds-default-parentNS':

    126 | 127 |
    128 |

    push request like 'ns-default', will have all fields required for 'timedot'[1] and key will be namespace by them 129 | read/delete like 'ns-default', just uses key with power of parentNS

    130 |
    • 131 |

  • 'ns-csv-parentNS'

    132 | 133 |
    134 |

    push request like 'ns-csv', will have all fields required for 'timedot'[1] and key will be namespace by them 135 | read/delete like 'ns-csv', just uses key with power of parentNS

    136 |
    • 137 |
138 | 139 |
140 | 141 |

GoShare Response

142 | 143 | What all 'ValType' data can be handled for Response? 144 | 145 |
    146 |

  • for "Push" (Create/Update) and Delete calls 147 | 148 | 149 |

    150 |

    151 |
    152 | 153 |
    154 |
    • on ZeroMQ: empty-string for Success, error-string for Failure
      • 155 |
    • on HTTP: "Success" for Success, error-string for Failure
    156 |
    • 157 |

  • for Read calls (same on ZeroMQ and HTTP) 158 | 159 | 160 |

    161 |

    162 |
    163 | 164 |
    165 |
    • nil, 'default': opts for 'csv' Response-CSV-Format format
      • 166 |
    • 'csv': handles just one data-string in CSV-format, better for cli apps
      • 167 |
    • 'json': handles just one data-string in JSON-format, better for not-cli apps
    168 | 169 |
    • 170 |
171 |

it uses the same ValType token for response in which the request has been sent
172 | so if you want response in json, request in json... it's like want respect, better give respect ;)

173 | 174 |
175 | 176 |

Clarifications if required:

177 | 178 |
    179 |

  • [1] 'timedot':

    180 | 181 |
    182 |

    it's an identifiable point in time by provided Year, Month, Day, Hour, Minute and Second.

    183 |
    • 184 |

  • [2] 'Request-Default-Format':

    185 | 186 |
    187 |

    for 'PUSH': it's two different fields for Key and Val, as in above ASCII-gram can be seen [Key ([Val])] 188 | for 'Read/Delete': it's just one field for Key '[key]'

    189 |
    • 190 |

  • [3] 'Request-CSV-format':

    191 | 192 |
    193 |

    for 'PUSH': one ('key,val') or multiple ('key,val\nkey2,val2') available in CSV format 194 | for 'Read/Delete': one ('key') or multiple ('key1,key2,key3,key4') available in CSV format

    195 |
    • 196 |

  • [4] 'Request-JSON-format':

    197 | 198 |
    199 |

    for 'PUSH': one ('{"key":"val"}') or multiple ('{"key":"val","key2":"val2"}') available in JSON format 200 | for 'Read/Delete': one ('["key"]') or multiple ('["key1","key2","key3","key4"]') available in JSON format

    201 |
    • 202 | 203 | 204 |
  • [A] Create/Update is same thing for GoShare. Just overwrites.
    • 205 |
206 |
207 | 208 | 209 | -------------------------------------------------------------------------------- /httpd/public/help-http.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | GoShare HTTP 5 | 6 | 7 |

GoShare ~ persistent (TimeSeries/Namespace/Simple) KeyVal storage

8 | 9 | 

10 |                         __                         _____ _____ ___
11 |    ____  ____     _____/ /_  ____  ____,,__   |  |   |     |   |  \
12 |   / __ `/ __ \   / ___/ __ \/ __ `/ ___/ _ \  |__|   |     |   |__/
13 |  / /_/ / /_/ /  (__  ) / / / /_/ / /  /  __/  |  |   |     |   |
14 |  \__, /\____/  /____/_/ /_/\__,_/_/   \___/   |  |   |     |   |
15 | /____/
16 | 
17 |
18 | 19 |

Go Share any data among the nodes. Over HTTP.

20 | Home | QuickStart | User Concepts | over ZeroMQ | Status 21 |
22 |
23 |

By HTTP Method, send HTTP Request to "/db" as

24 | 25 |
    26 |
  • POST/PUT method for "push" db-action
    • 27 |
  • GET method for "read" db-action
    • 28 |
  • DELETE method for "delete" db-action
    • 29 |
30 | 31 |
32 | 33 | By Route, send GET method HTTP Requests to 34 | 35 |
    36 |
  • "/put" for "push" db-action
    • 37 |
  • "/get" for "read" db-action
    • 38 |
  • "/del" for "delete" db-action
    • 39 |
40 | 41 |
42 | 43 |

URL form fields can be added for

44 | 45 |
    46 |
  • "type": task-type {default,ns.tsds,now}-{,default,csv,json}-{,parentNS}; like ns-json or tsds-csv-parentNS
    • 47 |
  • "key": for key to be used in "default" key-type
    • 48 |
  • "val": for value to be used in "default" key-type on "push" db-action
    • 49 |
  • "dbdata" : to specify data-string for val-types like csv, json
    • 50 |
  • "parentNS": to specify ParentNamespace for all/any keys to specified in request
    • 51 |
  • "year", "month", "day", "hour", "min", "sec" to specify all values of a timedot for key-type "tsds"
    • 52 |
53 | 54 |

>

55 | 56 |
57 |

for POST/PUT HTTP Method on /db, the field values can be provided via POST Body or URL fields

58 |
59 | 60 |
61 | 62 | Examples 63 | 64 |

all '%s' seen in URLs demand a suitable value for that field there

65 | 66 |
    67 |

  • read value for a key 'anything' http://<IP>:<PORT>/get?type=default&key=anything

    • 68 |

  • push with full namespace of 'any:thing" value 'huh' http://<IP>:<PORT>/put?type=ns&key=any:thing&val=huh

    • 69 |

  • delete all self and child namespace for 'any' and 'all' http://<IP>:<PORT>/del?type=ns-csv&dbdata=any,all

    • 70 |

  • push 'up' for 'state' as timeseries for given time point http://<IP>:<PORT>/put?key=state&val=up&type=tsds&year=%s&month=%s&day=%s&hour=%s&min=%s&sec=%s

    • 71 |

  • push namespaced-key "name:first" with value "bob" http://<IP>:<PORT>/put?dbdata={\"name:first\":\"bob\"}&type=ns-json

    • 72 |

  • push timeseries value for timedot of being stored of key 'a' value 'A', key 'b' value 'B' http://<IP>:<PORT>/put?dbdata=a,A%0D%0Ab,B&type=now-csv

    • 73 |

  • to use a parent-namespace helper (prepend all keys with it) for keys <ANY-OTHER-URL>&parentNS=%s

    • 74 |
75 | 76 |
77 |
78 | 79 | 80 | 81 | -------------------------------------------------------------------------------- /httpd/public/help-zmq.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | GoShare ZeroMQ 5 | 6 | 7 |

GoShare ~ persistent (TimeSeries/Namespace/Simple) KeyVal storage

8 | 9 | 

10 |                         __                    _____ _    _   __
11 |    ____  ____     _____/ /_  ____  ____,,__      /  |\  /|  /  \
12 |   / __ `/ __ \   / ___/ __ \/ __ `/ ___/ _ \    /   | \/ | |    |
13 |  / /_/ / /_/ /  (__  ) / / / /_/ / /  /  __/   /    |    | |  \\|
14 |  \__, /\____/  /____/_/ /_/\__,_/_/   \___/   /____ |    |  \__/\\
15 | /____/
16 | 
17 |
18 | 19 |

Go Share any data among the nodes. Over ZeroMQ.

20 | Home | QuickStart | User Concepts | over HTTP | Status 21 |
22 |
23 |
By Word Stream 24 | 25 | 26 | [DB-Action] [Task-Type] ([Time-Dot]) ([Parent-NameSpace]) {[Key ([Val...])] OR [DB-Data...]} 27 | 28 | 29 |

the specifics for these components have been explained under User Concepts before

30 | 31 |
32 |
    33 |
  • if you are coding in Golang, you can directly utilize ZmqRequest from "golzmq" *
    • 34 |
  • otherwise, just prepare the word-stream and pass it as bytes in your favorite ZeroMQ Request library *
    • 35 |
36 |
37 | 38 |
39 | 40 | Word Stream for major usable scenarios 41 | 42 |
43 |

simple Key Values

44 |
45 | 46 |
    47 |

  • Push a simple Key Value push default kernel linux

    • 48 |

  • Push multiple Key Value as CSV push default-csv kernel,linux

    • 49 |

  • Push multiple Key Value as JSON push default-json {"kernel":"linux"}

    • 50 |

  • Read a simple key, default response into CSV read default kernel

    • 51 |

  • Read a simple key into JSON response read default-json ["kernel"]

    • 52 |

  • Read multiple keys from JSON into JSON response read default-json ["kernel","os"]

    • 53 |

  • Delete a simple key delete default kernel

    • 54 |

  • Delete multiple keys from CSV delete default-csv kernel,os

    • 55 |
56 | 57 |
58 |

Namespaced Key Values

59 |
60 | 61 |
    62 |

  • Push a namespace key push ns software:internet:browser chrome

    • 63 |

  • Push multiple namespace keys from CSV push ns-csv software:internet:browser,chrome\nsoftware:internet:downloader,wget

    • 64 |

  • Push a namespace key with parent-namespace push ns-default-parentNS software internet:browser chrome

    • 65 |

  • Push multiple namespace keys from JSON with parent-namespace push ns-json-parentNS {"software:internet:browser":"chrome","software:internet:downloader":"wget"}

    • 66 |

  • Read all namespace value under given key, default as csv read ns "software:internet"

    • 67 |

  • Read all namespace value under given key, fixed as csv read ns-csv "software:internet"

    • 68 |

  • Read all namespace value under given key, response as JSON read ns-json-parentNS software ["internet"]

    • 69 |

  • Delete all namespace value under given key delete ns software:internet

    • 70 |
71 | 72 |
73 |

TimeSeries Key Values

74 |
75 | 76 |
    77 |

  • Push a tsds key val push tsds 2014 5 25 11 53 10 node:mymachine:memfree 4

    • 78 |

  • Push multiple tsds keys from CSV push tsds-csv 2014 5 25 11 53 10 node:mymachine:memfree,4\nnode:mymachine:diskfree,20

    • 79 |

  • Push multiple NOW timeseries keys from JSON at TimeDot of getting stored push now-json {"node:mymachine:memfree":"4","node:mymachine:diskfree":"20"}

    • 80 |

  • Push multiple tsds keys from JSON with parent-namespace push tsds-json-parentNS 2014 5 25 11 53 10 node:mymachine {"memfree":"4","diskfree":"20"}

    • 81 |

  • Read all tsds values under given key, response as JSON read tsds-json ["node:mymachine"]

    • 82 |

  • Read all tsds value under given key, response as JSON read tsds-json-parentNS node ["mymachine","othermachine"]

    • 83 |

  • Delete all tsds value under given key delete tsds node

    • 84 |
85 | 86 |
87 | 88 | 89 | 90 | -------------------------------------------------------------------------------- /httpd/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Go Share 5 | 6 | 7 |

GoShare ~ persistent (TimeSeries/Namespace/Simple) KeyVal storage

8 | 9 | 

10 |                         __
11 |    ____  ____     _____/ /_  ____  ____,,__
12 |   / __ `/ __ \   / ___/ __ \/ __ `/ ___/ _ \
13 |  / /_/ / /_/ /  (__  ) / / / /_/ / /  /  __/
14 |  \__, /\____/  /____/_/ /_/\__,_/_/   \___/
15 | /____/
16 | 
17 |
18 | 19 |

GoShare Philosophy

20 | QuickStart | User Concepts | over HTTP | over ZeroMQ | Status 21 |
22 |
23 | 24 |
25 | (in short) GoShare can be used to have persistent Time-Series, Namespace-KeyVal or typical Key-Val data-store over HTTP and ZeroMQ communication capability as independent server (or built-in for Golang applications). 26 | 27 |
28 | 29 | What the hell is it? 30 | 31 |

GoShare is an engine for Time-Series Data-Store, a datastore tailor-fit built to suit the requirement of storing varying states of any type of attribute along the timeline. 32 | It's not a database built for generic purpose used for TimeSeries data, but a datastore which will keep on improving to serve only this kind of requirement from core up.

33 | 34 |
35 | 36 | Uh, which solution would even use it? 37 | 38 |

It's kind of datastore which is perfect for requirements of a real-time monitoring system. Any kind of analytics system which requires change of attribute values along the stretch of a timeline could just plug-in and use it with a made for each other ease. 39 | There could be a lot more usable case for a timeseries datastore, these are the one I mainly ideated it for.

40 | 41 |
42 | 43 | In the name of God, why re-invent the wheel? 44 | 45 |

There are not a page long list of alternatives for this.

46 | 47 |

When I ideated it, the only solutions available were not self-dependent (using Zookeeper, MongoDB). Hence they were not actually alternative of an independent (born-for-it) Time-Series Data-Store. Also, I don't consider (stupid) closed-source projects for my FOSS initiatives. 48 | I needed such independent datastore for another monitoring project of mine where I didn't wanted to have more than one services to worry about for the core of my montitoring service. That would have like Watchmen for Watchmen for... 49 | Then later a project started ("InfluxDB") when I already started this. Which is in some way similar to this project but with much larger workforce behind it. It's also OpenSource, written in Golang.

50 | 51 |

I didn't re-invented the entire wheel just the improved upon the design and made it suit the road. The core of this datastore sits and inter-woven key-val store "leveldb" (it's an awesome, made for performance key-val store mechanism by some (x)Googlers which also the core of super-awesome Riak).

52 | 53 |

I also wanted to give it an awesome super communication channel which is not there in other alternative last I checked. Currently it allows communication ZeroMQ Req/Rep connections along with HTTP calls.

54 | 55 |

For data encoding, it supports key-val, csv and json for now. Msgpack (Protobuf and Cap'n'Proto) shall be in within a month or two (by July/2014).

56 | 57 |

If you are thinking on "why CSV support". The main reason for that is since it was ideated with monitoring solution in mind, which aim on supporting agents on node supporting plugins for monitoring data. Plug-ins supported for written in any technology, even a bash script doing sed+grep over status files with most easier encoding available as csv. Idea is supporting wide range at features where it will help better & quicker adoption.

58 | 59 |

Then we've the power of ZeroMQ to communicate with it (as mentioned before) enabling the awesomeness of supercharged socket communication with minimal formality possible.

60 | 61 |
62 | 63 | It's an Engine, so where is the Train? 64 | 65 |

The repo itself provides an extra piece of code allowing creation of service/daemon binaries to start using it and trying the power real quick.

66 | 67 |

As for a proper Train driving this engine, in parallel I've started design of MomentDB which will utilize it and enable having multiple style load-balanced, replicated and fail-over mechanisms for it. Just hold on for few months, design is almost done... development spike has started in pieces.

68 | 69 |

The monitoring system which lead to its creation, has also seen the light of spiking. It can be kept track of at ChaacMonitoring, but there is a very dumb structure there till now. It will show you GoShare in action but not of much utlization.

70 | 71 |
72 | 73 | It's not, but it is... 74 | 75 |

It's not built for typical key-val or namespace key-val data storage but on the way of its construction. It also allows to use all the available data-encoding (and the cooler efficient binary ones to come) over HTTP and also awesome ZeroMQ for Key-Val and NameSpace Key-Val persistent storage.

76 | 77 |
78 | 79 |

Get QuickStart at GoShare

80 |
81 | 82 | 83 | 84 | -------------------------------------------------------------------------------- /httpd/public/quickstart.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | GoShare Quickstart 5 | 6 | 7 |

GoShare ~ persistent (TimeSeries/Namespace/Simple) KeyVal storage

8 | 9 | 

 10 |                         __
 11 |    ____  ____     _____/ /_  ____  ____,,__    ___          _____   __     .   __  _____    _     __   _____
 12 |   / __ `/ __ \   / ___/ __ \/ __ `/ ___/ _ \  |   |  |   |    |    |    | /   /      |     / \   |  /    |
 13 |  / /_/ / /_/ /  (__  ) / / / /_/ / /  /  __/  |   |  |   |    |    |    |=    --,    |    |---|  |-.'    |
 14 |  \__, /\____/  /____/_/ /_/\__,_/_/   \___/   |__\\_ |___|, __|__  |__  | \.  __/    |    |   |  |  \    |
 15 | /____/
 16 | 
 17 |
18 | 19 |

Go Share : QuickStart

20 | Home | User Concepts | over HTTP | over ZeroMQ | Status 21 |
22 |
23 | 24 |
25 | 26 | Get the freakin' code 27 | 28 |
    29 |

  • have git client, awesome it's easy git clone https://github.com/abhishekkr/goshare

    • 30 |

  • have svn client, it's fine svn co https://github.com/abhishekkr/goshare

    • 31 |

  • have internet, ahhh... download and uncompress https://github.com/abhishekkr/goshare/archive/v0.8.4.tar.gz

    • 32 |
33 | 34 |
35 | 36 | Build your binaries 37 | 38 |

Make cloned/checkout/downloaded 'goshare' code directory as your current directory.

39 | 40 |
    41 |

  • have bash/zsh shell, cool ./go-tasks.sh bin

    • 42 |

  • have golang at least 43 | 44 | 45 |

    46 |

    47 |
    48 | 49 |
    50 |
    • fetch all dependencies mentioned in 'go-get-pkg.txt'
      • 51 |
    • run at cli mkdir ./bin ; cd ./bin ; go build ../zxtra/goshare_service.go
    52 |
    • 53 |
54 | 55 |
56 | 57 | Start it up 58 | 59 |
    60 |

  • if you have a '/tmp' location with write access for current user

    61 | 62 |
    63 |
    • run ./bin/goshare_server
    64 |
    • 65 |

  • otherwise

    66 | 67 |
    68 |
    • run ./bin/goshare_server -dbpath=$ANY_LOCATION_FORDB_CREATION
    69 |
    • 70 |
71 | 72 |
73 |
    74 |
  • HTTP listener at 0.0.0.0:9999
    • 75 |
  • ZeroMQ Request/Reply listener at 9898, 9797
    • 76 |
77 | 78 |

when started in non-daemon mode, these details get printed on command line as well

79 |
80 | 81 |

Now you can just run go run zxtra/gohttp_client.go to fire some read, push and delete calls to GoShare. Or make some of your own.

82 | 83 |
84 |

You'll notice in a particular requirement -dbpath flag has been used to provide location of DB to be passed to GoShare. There are similar other flags available, which are:

85 | 86 |
    87 |
  • dbpath : path to create database at; Default: "/tmp/GO.DB"
    • 88 |
  • http-uri : IP to connect while opening http port, Default: "0.0.0.0"
    • 89 |
  • http-port : Port to open http server connection for GoShare, Default: "9999"
    • 90 |
  • req-port : First port to bind ZeroMQ Request/Reply socket server, Default: "9797"
    • 91 |
  • rep-port : Second port to bind ZeroMQ Request/Reply socket server, Default: "9898"
    • 92 |
  • cpuprofile : file to log cpu profile data
    • 93 |
  • config : json config file to provide all other flag value to override
    • 94 |
95 |
96 | 97 |
98 |

The same set of configuration changes (or a portion of it) can be applied by a configuration file in JSON format, written like 99 | JSON 100 | {"http-port": "8080"} 101 |

102 |
103 | 104 |

So, here configuration applied by JSON file will override configuration applied by flags. If flags are not provided then the default value for them will be assigned to them.

105 | 106 |
107 | 108 | Ceate, Read, Delete 109 | 110 |

These actions have been covered in detail at: 111 | * Create : [wiki]to-be-written 112 | * Read : [wiki]to-be-written 113 | * Delete : [wiki]to-be-written

114 | 115 |

For this QuickStart section, let's try (assumption you have curl, else visit same URLs in your this Browser you spoilt kid ):

116 | 117 |

# 118 | curl http://127.0.0.1:9999/get?type=default&key=name

119 | 120 |

# 121 | curl http://127.0.0.1:9999/put?type=default&key=name&val=ledzep

122 | 123 |

# 124 | curl http://127.0.0.1:9999/get?type=default&key=name

125 | 126 |

# 127 | curl http://127.0.0.1:9999/put?type=ns&key=name:full&val=LedZep

128 | 129 |

# 130 | curl http://127.0.0.1:9999/put?type=ns-json&dbdata={\"name:first\":\"Led\",\"name:last\":\"Zep\"}

131 | 132 |

# 133 | curl http://127.0.0.1:9999/get?type=ns&key=name

134 | 135 |

# 136 | curl http://127.0.0.1:9999/del?type=ns&key=name

137 | 138 |

# 139 | curl http://127.0.0.1:9999/get?type=ns&key=name

140 | 141 |

# 142 | curl http://127.0.0.1:9999/put?type=now-json&dbdata={\"node01:webservice:state\":\"up\",\"node01:memfree\":\"4256783\"}

143 | 144 |

# 145 | curl http://127.0.0.1:9999/get?type=ns&key=node01:webservice

146 | 147 |

# 148 | curl http://127.0.0.1:9999/get?type=ns&key=node01

149 | 150 |

# 151 | curl http://127.0.0.1:9999/del?type=ns&key=node01

152 | 153 |

# 154 | curl http://127.0.0.1:9999/get?type=ns&key=node01

155 | 156 |

GoShare's HTTP link also allows using POST/PUT for Create and DELETE for Delete tasks as method rather than route based approach. So, to each their own satisfaction.

157 | 158 |
159 | 160 | 161 | 162 | -------------------------------------------------------------------------------- /httpd/public/status.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | GoShare Status 5 | 6 | 7 |

GoShare ~ persistent (TimeSeries/Namespace/Simple) KeyVal storage

8 | 9 | 

10 |                         __                    ____
11 |    ____  ____     _____/ /_  ____  ____,,__   |     |        |
12 |   / __ `/ __ \   / ___/ __ \/ __ `/ ___/ _ \  |___ ===  __  ===       __,
13 |  / /_/ / /_/ /  (__  ) / / / /_/ / /  /  __/     |  |  /  |  |  |  |  \__
14 |  \__, /\____/  /____/_/ /_/\__,_/_/   \___/   ___|  |_ |_/\  |_ \__|, ,_/
15 | /____/
16 | 
17 |
18 | 19 |

Go Share. Some status of this and that.

20 | Home | QuickStart | User Concepts | over HTTP | over ZeroMQ 21 | 22 |

W.I.P.

23 |

Blimey!

{{.Eh}} Eh!.

24 | 25 | HOME 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /requestor/packager.go: -------------------------------------------------------------------------------- 1 | package goshare_requestor 2 | 3 | import ( 4 | "fmt" 5 | 6 | golhashmap "github.com/abhishekkr/gol/golhashmap" 7 | gollist "github.com/abhishekkr/gol/gollist" 8 | goshare "github.com/abhishekkr/goshare" 9 | ) 10 | 11 | /* 12 | Create request bytes from Packet 13 | [DB-Action] [Task-Type] ([Time-Dot]) ([Parent-NameSpace]) {[Key ([Val...])] OR [DB-Data...]} 14 | */ 15 | func RequestPacketBytes(packet *goshare.Packet) []byte { 16 | var request string 17 | task_type := taskTypeFromPacket(packet) 18 | dbdata := dbDataFromPacket(packet) 19 | 20 | request = fmt.Sprintf("%s %s", packet.DBAction, task_type) 21 | if packet.KeyType == "tsds" { 22 | request = fmt.Sprintf("%s %s", request, packet.TimeDot) 23 | } 24 | if packet.ParentNS != "" { 25 | request = fmt.Sprintf("%s %s", request, packet.ParentNS) 26 | } 27 | request = fmt.Sprintf("%s %s", request, dbdata) 28 | 29 | return []byte(request) 30 | } 31 | 32 | /* formulate Task-Type from other info in Packet */ 33 | func taskTypeFromPacket(packet *goshare.Packet) (task_type string) { 34 | if packet.ValType == "" { 35 | packet.ValType = "default" 36 | } 37 | 38 | task_type = fmt.Sprintf("%s-%s", packet.KeyType, packet.ValType) 39 | 40 | if packet.ParentNS != "" { 41 | task_type = fmt.Sprintf("%s-parentNS", task_type) 42 | } 43 | return task_type 44 | } 45 | 46 | /* formulate DBData from other info in Packet */ 47 | func dbDataFromPacket(packet *goshare.Packet) string { 48 | if packet.ValType == "default" { 49 | return dbDataForDefaultVal(packet) 50 | } 51 | return dbDataForEncodedVal(packet) 52 | } 53 | 54 | /* formulate dbdata for default val-type FOR dbDataFromPacket */ 55 | func dbDataForDefaultVal(packet *goshare.Packet) (dbdata string) { 56 | switch packet.DBAction { 57 | case "push": 58 | for _key, _val := range packet.HashMap { 59 | dbdata = fmt.Sprintf("%s %s", _key, _val) 60 | break 61 | } 62 | case "read", "delete": 63 | for _, _val := range packet.KeyList { 64 | dbdata = _val 65 | break 66 | } 67 | default: 68 | //to-be-log 69 | } 70 | return dbdata 71 | } 72 | 73 | /* formulate dbdata for non-default val-type FOR dbDataFromPacket */ 74 | func dbDataForEncodedVal(packet *goshare.Packet) (dbdata string) { 75 | switch packet.DBAction { 76 | case "push": 77 | hmap_engine := golhashmap.GetHashMapEngine(packet.ValType) 78 | dbdata = hmap_engine.FromHashMap(packet.HashMap) 79 | case "read", "delete": 80 | list_engine := gollist.GetListEngine(packet.ValType) 81 | dbdata = list_engine.FromList(packet.KeyList) 82 | default: 83 | //to-be-log 84 | } 85 | return dbdata 86 | } 87 | -------------------------------------------------------------------------------- /tests/go0mq_client.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "strings" 7 | "time" 8 | 9 | golassert "github.com/abhishekkr/gol/golassert" 10 | golhashmap "github.com/abhishekkr/gol/golhashmap" 11 | golzmq "github.com/abhishekkr/gol/golzmq" 12 | 13 | goshare "github.com/abhishekkr/goshare" 14 | goshare_requestor "github.com/abhishekkr/goshare/requestor" 15 | ) 16 | 17 | var ( 18 | request_port01 = flag.Int("req-port01", 9797, "what Socket PORT to run at") 19 | request_port02 = flag.Int("req-port02", 9898, "what Socket PORT to run at") 20 | zmqSock = golzmq.ZmqRequestSocket("127.0.0.1", []int{*request_port01, *request_port02}) 21 | expected, result string 22 | err error 23 | ) 24 | 25 | // for key-type default 26 | func TestDefaultKeyType() { 27 | result, err = golzmq.ZmqRequest(zmqSock, "push", "default", "myname", "anon") 28 | fmt.Println(*request_port01) 29 | expected = "" 30 | golassert.AssertEqual(expected, result) 31 | golassert.AssertEqual(err, nil) 32 | 33 | result, err = golzmq.ZmqRequest(zmqSock, "read", "default", "myname") 34 | expected = "myname,anon" 35 | golassert.AssertEqual(expected, result) 36 | golassert.AssertEqual(err, nil) 37 | 38 | result, err = golzmq.ZmqRequest(zmqSock, "read", "default-csv", "myname") 39 | expected = "myname,anon" 40 | golassert.AssertEqual(expected, result) 41 | golassert.AssertEqual(err, nil) 42 | 43 | result, err = golzmq.ZmqRequest(zmqSock, "read", "default-json", "[\"myname\"]") 44 | expected = "{\"myname\":\"anon\"}" 45 | golassert.AssertEqual(expected, result) 46 | golassert.AssertEqual(err, nil) 47 | 48 | result, err = golzmq.ZmqRequest(zmqSock, "push", "default", "myname", "anonymous") 49 | expected = "" 50 | golassert.AssertEqual(expected, result) 51 | golassert.AssertEqual(err, nil) 52 | 53 | result, err = golzmq.ZmqRequest(zmqSock, "read", "default", "myname") 54 | expected = "myname,anonymous" 55 | golassert.AssertEqual(expected, result) 56 | golassert.AssertEqual(err, nil) 57 | 58 | result, err = golzmq.ZmqRequest(zmqSock, "delete", "default", "myname") 59 | expected = "" 60 | golassert.AssertEqual(expected, result) 61 | golassert.AssertEqual(err, nil) 62 | 63 | result, err = golzmq.ZmqRequest(zmqSock, "delete", "default", "myname") 64 | expected = "" 65 | golassert.AssertEqual(expected, result) 66 | golassert.AssertEqual(err, nil) 67 | 68 | result, err = golzmq.ZmqRequest(zmqSock, "read", "default", "myname") 69 | expected = "Error for request sent: read default myname" 70 | golassert.AssertEqual(expected, result) 71 | golassert.AssertEqual(err, nil) 72 | } 73 | 74 | // for key-type ns 75 | func TestNSKeyType() { 76 | result, err = golzmq.ZmqRequest(zmqSock, "push", "ns", "myname:last:first", "anon") 77 | expected = "" 78 | golassert.AssertEqual(expected, result) 79 | golassert.AssertEqual(err, nil) 80 | 81 | result, err = golzmq.ZmqRequest(zmqSock, "read", "ns", "myname") 82 | expected = "myname:last:first,anon" 83 | golassert.AssertEqual(expected, result) 84 | golassert.AssertEqual(err, nil) 85 | 86 | result, err = golzmq.ZmqRequest(zmqSock, "push", "ns", "myname:last", "ymous") 87 | expected = "" 88 | golassert.AssertEqual(expected, result) 89 | golassert.AssertEqual(err, nil) 90 | 91 | result, err = golzmq.ZmqRequest(zmqSock, "push", "ns", "myname", "anonymous") 92 | expected = "" 93 | golassert.AssertEqual(expected, result) 94 | golassert.AssertEqual(err, nil) 95 | 96 | result, err = golzmq.ZmqRequest(zmqSock, "read", "ns", "myname") 97 | expected_arr := []string{"myname,anonymous", "myname:last,ymous", "myname:last:first,anon"} 98 | result_arr := strings.Split(result, "\n") 99 | golassert.AssertEqualStringArray(expected_arr, result_arr) 100 | golassert.AssertEqual(err, nil) 101 | 102 | result, err = golzmq.ZmqRequest(zmqSock, "delete", "ns", "myname") 103 | expected = "" 104 | golassert.AssertEqual(expected, result) 105 | golassert.AssertEqual(err, nil) 106 | 107 | result, err = golzmq.ZmqRequest(zmqSock, "read", "ns", "myname") 108 | expected = "Error for request sent: read ns myname" 109 | golassert.AssertEqual(expected, result) 110 | golassert.AssertEqual(err, nil) 111 | } 112 | 113 | // for key-type tsds 114 | func TestTSDSKeyType() { 115 | result, err = golzmq.ZmqRequest(zmqSock, "push", "tsds", "2014", "2", "10", "9", "8", "7", "myname:last:first", "anon") 116 | expected = "" 117 | golassert.AssertEqual(expected, result) 118 | golassert.AssertEqual(err, nil) 119 | 120 | result, err = golzmq.ZmqRequest(zmqSock, "read", "ns", "myname") 121 | expected = "myname:last:first:2014:2:10:9:8:7:0,anon" 122 | golassert.AssertEqual(expected, result) 123 | golassert.AssertEqual(err, nil) 124 | 125 | result, err = golzmq.ZmqRequest(zmqSock, "read", "tsds", "myname") 126 | expected = "myname:last:first:2014:2:10:9:8:7:0,anon" 127 | golassert.AssertEqual(expected, result) 128 | golassert.AssertEqual(err, nil) 129 | 130 | result, err = golzmq.ZmqRequest(zmqSock, "read", "tsds", "myname:last:first") 131 | expected = "myname:last:first:2014:2:10:9:8:7:0,anon" 132 | golassert.AssertEqual(expected, result) 133 | golassert.AssertEqual(err, nil) 134 | 135 | result, err = golzmq.ZmqRequest(zmqSock, "push", "tsds", "2014", "2", "10", "9", "18", "37", "myname", "anonymous") 136 | expected = "" 137 | golassert.AssertEqual(expected, result) 138 | golassert.AssertEqual(err, nil) 139 | 140 | result, err = golzmq.ZmqRequest(zmqSock, "read", "tsds", "myname") 141 | result_arr := strings.Split(result, "\n") 142 | expected_arr := []string{"myname:last:first:2014:2:10:9:8:7:0,anon", 143 | "myname:2014:2:10:9:18:37:0,anonymous"} 144 | golassert.AssertEqualStringArray(expected_arr, result_arr) 145 | golassert.AssertEqual(err, nil) 146 | 147 | result, err = golzmq.ZmqRequest(zmqSock, "push", "tsds-csv", "2014", "2", "10", "9", "18", "37", "myname,bob\nmyemail,bob@b.com") 148 | expected = "" 149 | golassert.AssertEqual(expected, result) 150 | golassert.AssertEqual(err, nil) 151 | 152 | result, err = golzmq.ZmqRequest(zmqSock, "push", "tsds-csv", "2014", "2", "10", "9", "18", "37.0", "myname,bob\nmyemail,bob@b.com") 153 | expected = "" 154 | golassert.AssertEqual(expected, result) 155 | golassert.AssertEqual(err, nil) 156 | 157 | result, err = golzmq.ZmqRequest(zmqSock, "read", "tsds", "myname") 158 | result_arr = strings.Split(result, "\n") 159 | expected_arr = []string{"myname:last:first:2014:2:10:9:8:7:0,anon", 160 | "myname:2014:2:10:9:18:37:0,bob"} 161 | golassert.AssertEqualStringArray(expected_arr, result_arr) 162 | golassert.AssertEqual(err, nil) 163 | 164 | result, err = golzmq.ZmqRequest(zmqSock, "push", "tsds-csv", "2014", "2", "10", "9", "18", "37", "myname,alice\nmytxt,\"my email, bob@b.com\"") 165 | expected = "" 166 | golassert.AssertEqual(expected, result) 167 | golassert.AssertEqual(err, nil) 168 | 169 | result, err = golzmq.ZmqRequest(zmqSock, "read", "tsds", "myemail") 170 | expected = "myemail:2014:2:10:9:18:37:0,bob@b.com" 171 | golassert.AssertEqual(expected, result) 172 | golassert.AssertEqual(err, nil) 173 | 174 | result, err = golzmq.ZmqRequest(zmqSock, "read", "tsds", "mytxt") 175 | expected = "mytxt:2014:2:10:9:18:37:0,\"my email, bob@b.com\"" 176 | golassert.AssertEqual(expected, result) 177 | golassert.AssertEqual(err, nil) 178 | 179 | result, err = golzmq.ZmqRequest(zmqSock, "read", "tsds", "myname:2014:2:10") 180 | expected = "myname:2014:2:10:9:18:37:0,alice" 181 | golassert.AssertEqual(expected, result) 182 | golassert.AssertEqual(err, nil) 183 | 184 | result, err = golzmq.ZmqRequest(zmqSock, "delete", "ns", "myname") 185 | expected = "" 186 | golassert.AssertEqual(expected, result) 187 | golassert.AssertEqual(err, nil) 188 | 189 | result, err = golzmq.ZmqRequest(zmqSock, "delete", "ns", "myemail") 190 | expected = "" 191 | golassert.AssertEqual(expected, result) 192 | golassert.AssertEqual(err, nil) 193 | 194 | result, err = golzmq.ZmqRequest(zmqSock, "delete", "ns", "mytxt") 195 | expected = "" 196 | golassert.AssertEqual(expected, result) 197 | golassert.AssertEqual(err, nil) 198 | 199 | result, err = golzmq.ZmqRequest(zmqSock, "read", "tsds", "myname") 200 | expected = "Error for request sent: read tsds myname" 201 | golassert.AssertEqual(expected, result) 202 | golassert.AssertEqual(err, nil) 203 | } 204 | 205 | // for key-type now 206 | func TestNowKeyType() { 207 | result, err = golzmq.ZmqRequest(zmqSock, "push", "now", "myname:last:first", "anon") 208 | expected = "" 209 | golassert.AssertEqual(expected, result) 210 | golassert.AssertEqual(err, nil) 211 | 212 | result, err = golzmq.ZmqRequest(zmqSock, "read", "tsds", "myname") 213 | result_length := len(golhashmap.CSVToHashMap(result)) 214 | expected_length := 1 215 | golassert.AssertEqual(result_length, expected_length) 216 | golassert.AssertEqual(err, nil) 217 | 218 | time.Sleep(1) 219 | 220 | result, err = golzmq.ZmqRequest(zmqSock, "push", "now", "myname:last", "ymous") 221 | expected = "" 222 | golassert.AssertEqual(expected, result) 223 | golassert.AssertEqual(err, nil) 224 | 225 | result, err = golzmq.ZmqRequest(zmqSock, "read", "tsds", "myname") 226 | result_length = len(golhashmap.CSVToHashMap(result)) 227 | expected_length = 2 228 | golassert.AssertEqual(result_length, expected_length) 229 | golassert.AssertEqual(err, nil) 230 | 231 | result, err = golzmq.ZmqRequest(zmqSock, "delete", "ns", "myname") 232 | expected = "" 233 | golassert.AssertEqual(expected, result) 234 | golassert.AssertEqual(err, nil) 235 | 236 | result, err = golzmq.ZmqRequest(zmqSock, "read", "ns", "myname") 237 | expected = "Error for request sent: read ns myname" 238 | golassert.AssertEqual(expected, result) 239 | golassert.AssertEqual(err, nil) 240 | } 241 | 242 | /* for parentNS for key-type */ 243 | func TestParentNSValType() { 244 | result, err = golzmq.ZmqRequest(zmqSock, "push", "ns-default-parent", "people", "myname", "anonymous") 245 | expected = "" 246 | golassert.AssertEqual(expected, result) 247 | golassert.AssertEqual(err, nil) 248 | 249 | result, err = golzmq.ZmqRequest(zmqSock, "read", "ns", "people:myname") 250 | expected = "people:myname,anonymous" 251 | golassert.AssertEqual(expected, result) 252 | golassert.AssertEqual(err, nil) 253 | 254 | result, err = golzmq.ZmqRequest(zmqSock, "read", "ns-default-parent", "people", "myname") 255 | expected = "people:myname,anonymous" 256 | golassert.AssertEqual(expected, result) 257 | golassert.AssertEqual(err, nil) 258 | 259 | result, err = golzmq.ZmqRequest(zmqSock, "push", "tsds-csv-parent", "2014", "2", "10", "9", "18", "37", "people", "myname,bob") 260 | expected = "" 261 | golassert.AssertEqual(expected, result) 262 | golassert.AssertEqual(err, nil) 263 | 264 | result, err = golzmq.ZmqRequest(zmqSock, "read", "tsds-default-parent", "people", "myname") 265 | expected = "people:myname,anonymous\npeople:myname:2014:2:10:9:18:37:0,bob" 266 | golassert.AssertEqual(expected, result) 267 | golassert.AssertEqual(err, nil) 268 | 269 | result, err = golzmq.ZmqRequest(zmqSock, "delete", "ns-default-parent", "people", "myname") 270 | expected = "" 271 | golassert.AssertEqual(expected, result) 272 | golassert.AssertEqual(err, nil) 273 | 274 | result, err = golzmq.ZmqRequest(zmqSock, "read", "tsds-default-parent", "people", "myname") 275 | expected = "Error for request sent: read tsds-default-parent people myname" 276 | golassert.AssertEqual(expected, result) 277 | golassert.AssertEqual(err, nil) 278 | } 279 | 280 | func TestRequestorZeromq() { 281 | _packet := goshare.Packet{} 282 | _packet.KeyType = "default" 283 | 284 | _packet.DBAction = "push" 285 | _packet.HashMap = make(golhashmap.HashMap) 286 | _packet.HashMap["an0n"] = "ymous" 287 | strPacket := string(goshare_requestor.RequestPacketBytes(&_packet)) 288 | result, err = golzmq.ZmqRequest(zmqSock, strPacket) 289 | expected = "" 290 | golassert.AssertEqual(expected, result) 291 | golassert.AssertEqual(err, nil) 292 | 293 | _packet.DBAction = "read" 294 | _packet.KeyList = []string{"an0n"} 295 | strPacket = string(goshare_requestor.RequestPacketBytes(&_packet)) 296 | result, err = golzmq.ZmqRequest(zmqSock, strPacket) 297 | expected = "an0n,ymous" 298 | golassert.AssertEqual(expected, result) 299 | golassert.AssertEqual(err, nil) 300 | 301 | _packet.DBAction = "delete" 302 | strPacket = string(goshare_requestor.RequestPacketBytes(&_packet)) 303 | result, err = golzmq.ZmqRequest(zmqSock, strPacket) 304 | expected = "" 305 | golassert.AssertEqual(expected, result) 306 | golassert.AssertEqual(err, nil) 307 | 308 | _packet.DBAction = "read" 309 | strPacket = string(goshare_requestor.RequestPacketBytes(&_packet)) 310 | result, err = golzmq.ZmqRequest(zmqSock, strPacket) 311 | expected = "Error for request sent: read default-default an0n" 312 | golassert.AssertEqual(expected, result) 313 | golassert.AssertEqual(err, nil) 314 | } 315 | 316 | func main() { 317 | flag.Parse() 318 | fmt.Printf("client ZeroMQ REP/REQ... at %d, %d\n", *request_port01, *request_port02) 319 | 320 | fmt.Println("Checking out levigo based storage...") 321 | TestDefaultKeyType() 322 | 323 | fmt.Println("Checking out levigoNS based storage...") 324 | TestNSKeyType() 325 | 326 | fmt.Println("Checking out levigoTSDS based storage...") 327 | TestTSDSKeyType() 328 | TestNowKeyType() 329 | 330 | TestParentNSValType() 331 | } 332 | -------------------------------------------------------------------------------- /tests/gohttp_client.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "io/ioutil" 7 | "net/http" 8 | "strings" 9 | 10 | golassert "github.com/abhishekkr/gol/golassert" 11 | golhashmap "github.com/abhishekkr/gol/golhashmap" 12 | ) 13 | 14 | var ( 15 | httphost = flag.String("host", "127.0.0.1", "what Host to run at") 16 | httpport = flag.Int("port", 9999, "what Socket PORT to connect") 17 | ) 18 | 19 | // return Get URL for task_type, key 20 | func GetURL(host string, port int, key_type, key string) string { 21 | return fmt.Sprintf("http://%s:%d/get?type=%s&key=%s", host, port, key_type, key) 22 | } 23 | 24 | // return Push URL for task_type, key, val 25 | func PutURL(host string, port int, key_type, key, val string) string { 26 | return fmt.Sprintf("http://%s:%d/put?type=%s&key=%s&val=%s", 27 | host, port, key_type, key, val) 28 | } 29 | 30 | // return Delete URL for task_type, key 31 | func DelURL(host string, port int, key_type, key string) string { 32 | return fmt.Sprintf("http://%s:%d/del?type=%s&key=%s", host, port, key_type, key) 33 | } 34 | 35 | // return Push URL for TSDS type key, val, time-elements 36 | func TSDSPutURL(host string, port int, key, val, year, month, day, hr, min, sec string) string { 37 | return fmt.Sprintf("http://%s:%d/put?key=%s&val=%s&type=tsds&year=%s&month=%s&day=%s&hour=%s&min=%s&sec=%s", 38 | host, port, key, val, year, month, day, hr, min, sec) 39 | } 40 | 41 | // return Push URL for multi-val-type on task-type and multi-val 42 | func MultiValPutURL(host string, port int, task_type, multi_value string) string { 43 | return fmt.Sprintf("http://%s:%d/put?dbdata=%s&type=%s", host, port, multi_value, task_type) 44 | } 45 | 46 | // return Push URL for TSDS multi-val-type on task-type, val-type and multi-val 47 | func MultiTSDSPutURL(host string, port int, val_type, multi_value, year, month, day, hr, min, sec string) string { 48 | return fmt.Sprintf("http://%s:%d/put?dbdata=%s&type=tsds-%s&year=%s&month=%s&day=%s&hour=%s&min=%s&sec=%s", 49 | host, port, multi_value, val_type, year, month, day, hr, min, sec) 50 | } 51 | 52 | // append url values for parentNS and return URL 53 | func URLAppendParentNS(url string, parentNS string) string { 54 | return fmt.Sprintf("%s&parentNS=%s", url, parentNS) 55 | } 56 | 57 | // makes HTTP call for given URL and returns response body 58 | func HttpGet(url string) (int, string) { 59 | resp, err := http.Get(url) 60 | if err != nil { 61 | return 404, "Error: " + url + " failed for HTTP GET" 62 | } 63 | defer resp.Body.Close() 64 | 65 | body, _ := ioutil.ReadAll(resp.Body) 66 | 67 | //fmt.Printf("Url: %s; with\n result:\n%s\n", url, string(body)) 68 | return resp.StatusCode, string(body) 69 | } 70 | 71 | // for default key-type 72 | func testDefaultKeyType() { 73 | _, body := HttpGet(PutURL(*httphost, *httpport, "default", "myname", "anon")) 74 | golassert.AssertEqual(body, "Success") 75 | 76 | _, body = HttpGet(GetURL(*httphost, *httpport, "default", "myname")) 77 | golassert.AssertEqual(body, "myname,anon") 78 | 79 | _, body = HttpGet(PutURL(*httphost, *httpport, "default", "myname", "anonymous")) 80 | golassert.AssertEqual(body, "Success") 81 | 82 | _, body = HttpGet(GetURL(*httphost, *httpport, "default", "myname")) 83 | golassert.AssertEqual(body, "myname,anonymous") 84 | 85 | _, body = HttpGet(DelURL(*httphost, *httpport, "default", "myname")) 86 | golassert.AssertEqual(body, "Success") 87 | 88 | code, _ := HttpGet(GetURL(*httphost, *httpport, "default", "myname")) 89 | golassert.AssertEqual(code, 500) 90 | 91 | fmt.Println("No panic for 'default' key type.") 92 | } 93 | 94 | // for ns key-type 95 | func testNamespaceKeyType() { 96 | _, body := HttpGet(PutURL(*httphost, *httpport, "ns", "myname:last:first", "anon")) 97 | golassert.AssertEqual(body, "Success") 98 | 99 | _, body = HttpGet(GetURL(*httphost, *httpport, "ns", "myname:last:first")) 100 | golassert.AssertEqual(body, "myname:last:first,anon") 101 | 102 | _, body = HttpGet(PutURL(*httphost, *httpport, "ns", "myname:last", "ymous")) 103 | golassert.AssertEqual(body, "Success") 104 | 105 | _, body = HttpGet(PutURL(*httphost, *httpport, "ns", "myname", "anonymous")) 106 | golassert.AssertEqual(body, "Success") 107 | 108 | _, body = HttpGet(GetURL(*httphost, *httpport, "ns", "myname:last")) 109 | strings2 := strings.Split(body, "\n") 110 | strings2_expected := []string{"myname:last,ymous", 111 | "myname:last:first,anon"} 112 | golassert.AssertEqualStringArray(strings2, strings2_expected) 113 | 114 | _, body = HttpGet(DelURL(*httphost, *httpport, "ns", "myname")) 115 | golassert.AssertEqual(body, "Success") 116 | 117 | code, _ := HttpGet(GetURL(*httphost, *httpport, "ns", "myname:last")) 118 | golassert.AssertEqual(code, 500) 119 | 120 | fmt.Println("No panic for 'namespace' key type.") 121 | } 122 | 123 | // for tsds key-type 124 | func testTSDSKeyType() { 125 | _, body := HttpGet(TSDSPutURL(*httphost, *httpport, "myname:last", "ymous", "2014", "2", "10", "1", "1", "1")) 126 | golassert.AssertEqual(body, "Success") 127 | 128 | _, body = HttpGet(TSDSPutURL(*httphost, *httpport, "myname", "anonymous", "2014", "2", "10", "9", "8", "7")) 129 | golassert.AssertEqual(body, "Success") 130 | 131 | _, body = HttpGet(TSDSPutURL(*httphost, *httpport, "myname", "untitled", "2014", "2", "10", "0", "1", "7")) 132 | golassert.AssertEqual(body, "Success") 133 | 134 | _, body = HttpGet(GetURL(*httphost, *httpport, "tsds", "myname:last")) 135 | golassert.AssertEqual(body, "myname:last:2014:2:10:1:1:1:0,ymous") 136 | 137 | _, body = HttpGet(GetURL(*httphost, *httpport, "tsds", "myname")) 138 | strings3 := strings.Split(body, "\n") 139 | strings3_expected := []string{"myname:last:2014:2:10:1:1:1:0,ymous", 140 | "myname:2014:2:10:9:8:7:0,anonymous", 141 | "myname:2014:2:10:0:1:7:0,untitled"} 142 | golassert.AssertEqualStringArray(strings3, strings3_expected) 143 | 144 | _, body = HttpGet(GetURL(*httphost, *httpport, "tsds", "myname:2014:2:10")) 145 | strings2 := strings.Split(body, "\n") 146 | strings2_expected := []string{"myname:2014:2:10:9:8:7:0,anonymous", 147 | "myname:2014:2:10:0:1:7:0,untitled"} 148 | golassert.AssertEqualStringArray(strings2, strings2_expected) 149 | 150 | _, body = HttpGet(DelURL(*httphost, *httpport, "tsds", "myname")) 151 | golassert.AssertEqual(body, "Success") 152 | 153 | code, _ := HttpGet(GetURL(*httphost, *httpport, "tsds", "myname")) 154 | golassert.AssertEqual(code, 500) 155 | 156 | fmt.Println("No panic for 'timeseries' key type.") 157 | } 158 | 159 | // for now key-type 160 | func testNowKeyType() { 161 | csvmap := golhashmap.GetHashMapEngine("csv") 162 | 163 | _, body := HttpGet(DelURL(*httphost, *httpport, "tsds", "yname")) 164 | golassert.AssertEqual(body, "Success") 165 | 166 | _, body = HttpGet(PutURL(*httphost, *httpport, "now", "yname:last:first", "zodiac")) 167 | golassert.AssertEqual(body, "Success") 168 | 169 | _, body = HttpGet(GetURL(*httphost, *httpport, "tsds", "yname:last:first")) 170 | golassert.AssertEqual(len(csvmap.ToHashMap(body)), 1) 171 | 172 | _, body = HttpGet(DelURL(*httphost, *httpport, "tsds", "yname")) 173 | golassert.AssertEqual(body, "Success") 174 | 175 | code, body := HttpGet(GetURL(*httphost, *httpport, "tsds", "yname")) 176 | golassert.AssertEqual(code, 500) 177 | 178 | _, body = HttpGet(DelURL(*httphost, *httpport, "tsds", "myname")) 179 | golassert.AssertEqual(body, "Success") 180 | 181 | _, body = HttpGet(PutURL(*httphost, *httpport, "now", "myname:last:first", "ripper")) 182 | golassert.AssertEqual(body, "Success") 183 | 184 | _, body = HttpGet(GetURL(*httphost, *httpport, "tsds", "myname:last:first")) 185 | golassert.AssertEqual(len(csvmap.ToHashMap(body)), 1) 186 | 187 | _, body = HttpGet(DelURL(*httphost, *httpport, "tsds", "myname")) 188 | golassert.AssertEqual(body, "Success") 189 | 190 | code, _ = HttpGet(GetURL(*httphost, *httpport, "tsds", "myname")) 191 | golassert.AssertEqual(code, 500) 192 | 193 | fmt.Println("No panic for 'timeseries now' key type.") 194 | } 195 | 196 | // for csv val-type 197 | func testForCSV() { 198 | _, body := HttpGet(MultiTSDSPutURL(*httphost, *httpport, "csv", "yourname:last:first,trudy", "2014", "2", "10", "1", "1", "1")) 199 | golassert.AssertEqual(body, "Success") 200 | 201 | _, body = HttpGet(GetURL(*httphost, *httpport, "tsds", "yourname:last:first:2014:2")) 202 | golassert.AssertEqual(body, "yourname:last:first:2014:2:10:1:1:1:0,trudy") 203 | 204 | _, body = HttpGet(DelURL(*httphost, *httpport, "tsds", "yourname")) 205 | golassert.AssertEqual(body, "Success") 206 | 207 | _, body = HttpGet(MultiValPutURL(*httphost, *httpport, "ns-csv", "yname:frend:first,monica")) 208 | golassert.AssertEqual(body, "Success") 209 | 210 | _, body = HttpGet(MultiValPutURL(*httphost, *httpport, "ns-csv", "yname:frend:second,lolita")) 211 | golassert.AssertEqual(body, "Success") 212 | 213 | _, body = HttpGet(GetURL(*httphost, *httpport, "tsds", "yname:frend")) 214 | strings2 := strings.Split(body, "\n") 215 | strings2_expected := []string{"yname:frend:first,monica", 216 | "yname:frend:second,lolita"} 217 | golassert.AssertEqualStringArray(strings2, strings2_expected) 218 | 219 | _, body = HttpGet(DelURL(*httphost, *httpport, "tsds", "yname")) 220 | golassert.AssertEqual(body, "Success") 221 | 222 | _, body = HttpGet(MultiValPutURL(*httphost, *httpport, "ns-csv", "yname:frend:first,monica%0D%0Ayname:frend:second,lolita%0D%0Auname:frend:second,juno%0D%0A")) 223 | golassert.AssertEqual(body, "Success") 224 | 225 | _, body = HttpGet(GetURL(*httphost, *httpport, "tsds", "yname:frend")) 226 | strings2 = strings.Split(body, "\n") 227 | strings2_expected = []string{"yname:frend:first,monica", 228 | "yname:frend:second,lolita"} 229 | golassert.AssertEqualStringArray(strings2, strings2_expected) 230 | 231 | _, body = HttpGet(GetURL(*httphost, *httpport, "tsds", "uname:frend")) 232 | golassert.AssertEqual(body, "uname:frend:second,juno") 233 | 234 | _, body = HttpGet(DelURL(*httphost, *httpport, "tsds", "yname")) 235 | golassert.AssertEqual(body, "Success") 236 | 237 | _, body = HttpGet(DelURL(*httphost, *httpport, "tsds", "uname")) 238 | golassert.AssertEqual(body, "Success") 239 | 240 | fmt.Println("No panic for 'csv' key type.") 241 | } 242 | 243 | // for JSON val-type 244 | func testForJSON() { 245 | _, body := HttpGet(MultiValPutURL(*httphost, *httpport, "ns-json", "{\"power:first\":\"yay\",\"power:second\":\"way\",\"rower:second\":\"kay\"}")) 246 | golassert.AssertEqual(body, "Success") 247 | 248 | _, body = HttpGet(GetURL(*httphost, *httpport, "tsds", "power")) 249 | strings2 := strings.Split(body, "\n") 250 | strings2_expected := []string{"power:first,yay", "power:second,way"} 251 | golassert.AssertEqualStringArray(strings2, strings2_expected) 252 | 253 | _, body = HttpGet(GetURL(*httphost, *httpport, "tsds", "rower")) 254 | golassert.AssertEqual(body, "rower:second,kay") 255 | 256 | _, body = HttpGet(DelURL(*httphost, *httpport, "tsds", "power")) 257 | golassert.AssertEqual(body, "Success") 258 | 259 | _, body = HttpGet(DelURL(*httphost, *httpport, "tsds", "rower")) 260 | golassert.AssertEqual(body, "Success") 261 | 262 | fmt.Println("No panic for 'json' key type.") 263 | } 264 | 265 | // for &parentNS=parent:namespace 266 | func testWithParentNS() { 267 | var url string 268 | 269 | url = MultiValPutURL(*httphost, *httpport, "ns-csv-parent", "yname:frend:first,monica") 270 | _, body := HttpGet(URLAppendParentNS(url, "animal:people")) 271 | golassert.AssertEqual(body, "Success") 272 | 273 | url = MultiValPutURL(*httphost, *httpport, "ns-csv-parent", "yname:frend:second,lolita") 274 | _, body = HttpGet(URLAppendParentNS(url, "animal:people")) 275 | golassert.AssertEqual(body, "Success") 276 | 277 | url = URLAppendParentNS(GetURL(*httphost, *httpport, "tsds-csv-parent", "yname:frend"), "animal:people") 278 | _, body = HttpGet(url) 279 | strings2 := strings.Split(body, "\n") 280 | strings2_expected := []string{"animal:people:yname:frend:first,monica", 281 | "animal:people:yname:frend:second,lolita"} 282 | golassert.AssertEqualStringArray(strings2, strings2_expected) 283 | 284 | url = URLAppendParentNS(GetURL(*httphost, *httpport, "tsds-csv-parent", "people:yname:frend"), "animal") 285 | _, body = HttpGet(url) 286 | strings2 = strings.Split(body, "\n") 287 | golassert.AssertEqualStringArray(strings2, strings2_expected) 288 | 289 | url = GetURL(*httphost, *httpport, "tsds-csv-parent", "animal:people:yname:frend") 290 | _, body = HttpGet(url) 291 | strings2 = strings.Split(body, "\n") 292 | strings2_expected = []string{"animal:people:yname:frend:first,monica", 293 | "animal:people:yname:frend:second,lolita"} 294 | golassert.AssertEqualStringArray(strings2, strings2_expected) 295 | 296 | url = URLAppendParentNS(DelURL(*httphost, *httpport, "tsds-csv-parent", "yname"), "animal:people") 297 | _, body = HttpGet(url) 298 | golassert.AssertEqual(body, "Success") 299 | 300 | fmt.Println("No panic for 'Parent NS' key type.") 301 | } 302 | 303 | func main() { 304 | flag.Parse() 305 | 306 | testDefaultKeyType() 307 | testNamespaceKeyType() 308 | testTSDSKeyType() 309 | testNowKeyType() 310 | testForCSV() 311 | testForJSON() 312 | testWithParentNS() 313 | fmt.Println("passed not panic") 314 | } 315 | -------------------------------------------------------------------------------- /tests/million_go0mq.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "io/ioutil" 7 | "net/http" 8 | "time" 9 | 10 | golzmq "github.com/abhishekkr/gol/golzmq" 11 | ) 12 | 13 | var ( 14 | httphost = flag.String("host", "127.0.0.1", "what Host to run at") 15 | httpport = flag.Int("port", 9999, "what Socket PORT to connect") 16 | request_port01 = flag.Int("req-port01", 9797, "what Socket PORT to run at") 17 | request_port02 = flag.Int("req-port02", 9898, "what Socket PORT to run at") 18 | protocol = flag.String("protocol", "zmq", "zmq||http") 19 | dbaction = flag.String("axn", "push", "push||delete||read") 20 | zmqSock = golzmq.ZmqRequestSocket("127.0.0.1", []int{*request_port01, *request_port02}) 21 | expected, result string 22 | err error 23 | daycount = 0 24 | days = []int{31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31} 25 | ) 26 | 27 | // return Push URL for TSDS type key, val, time-elements 28 | func TSDSPutURL(host string, port int, key, val, year, month, day, hr, min, sec string) string { 29 | return fmt.Sprintf("http://%s:%d/put?key=%s&val=%s&type=tsds&year=%s&month=%s&day=%s&hour=%s&min=%s&sec=%s", 30 | host, port, key, val, year, month, day, hr, min, sec) 31 | } 32 | 33 | // return Get URL for task_type, key 34 | func TSDSGetURL(host string, port int, key, year, month, day, hr, min, sec string) string { 35 | return fmt.Sprintf("http://%s:%d/get?key=%s&type=tsds&year=%s&month=%s&day=%s&hour=%s&min=%s&sec=%s", 36 | host, port, key, year, month, day, hr, min, sec) 37 | } 38 | 39 | // return Delete URL for task_type, key 40 | func TSDSDelURL(host string, port int, key, year, month, day, hr, min, sec string) string { 41 | return fmt.Sprintf("http://%s:%d/del?key=%s&type=tsds&year=%s&month=%s&day=%s&hour=%s&min=%s&sec=%s", 42 | host, port, key, year, month, day, hr, min, sec) 43 | } 44 | 45 | // makes HTTP call for given URL and returns response body 46 | func HttpGet(url string) (int, string) { 47 | resp, err := http.Get(url) 48 | if err != nil { 49 | return 404, "Error: " + url + " failed for HTTP GET" 50 | } 51 | defer resp.Body.Close() 52 | 53 | body, _ := ioutil.ReadAll(resp.Body) 54 | 55 | //fmt.Printf("Url: %s; with\n result:\n%s\n", url, string(body)) 56 | return resp.StatusCode, string(body) 57 | } 58 | 59 | func zmqTasks(yy, mm, dd, hr, min, sec int, key, val string) { 60 | var result string 61 | var err error 62 | switch *dbaction { 63 | case "push": 64 | result, err = golzmq.ZmqRequest(zmqSock, "push", "tsds", string(yy), string(mm), string(dd), string(hr), string(min), string(sec), key, val) 65 | case "read": 66 | result, err = golzmq.ZmqRequest(zmqSock, "read", "tsds", string(yy), string(mm), string(dd), string(hr), string(min), string(sec), key) 67 | case "delete": 68 | result, err = golzmq.ZmqRequest(zmqSock, "delete", "tsds", string(yy), string(mm), string(dd), string(hr), string(min), string(sec), key) 69 | default: 70 | panic("Unhandled dbaction") 71 | } 72 | fmt.Println(result, err) 73 | } 74 | 75 | func httpTasks(yy, mm, dd, hr, min, sec int, key, val string) { 76 | var body string 77 | switch *dbaction { 78 | case "push": 79 | _, body = HttpGet(TSDSPutURL(*httphost, *httpport, key, val, string(yy), string(mm), string(dd), string(hr), string(min), string(sec))) 80 | case "read": 81 | _, body = HttpGet(TSDSGetURL(*httphost, *httpport, key, string(yy), string(mm), string(dd), string(hr), string(min), string(sec))) 82 | case "delete": 83 | _, body = HttpGet(TSDSDelURL(*httphost, *httpport, key, string(yy), string(mm), string(dd), string(hr), string(min), string(sec))) 84 | default: 85 | panic("Unhandled dbaction") 86 | } 87 | fmt.Println(body) 88 | } 89 | 90 | func everysecond(yy, mm, dd int) { 91 | for hr := 1; hr <= 24; hr++ { 92 | for min := 1; min <= 60; min++ { 93 | for sec := 1; sec <= 60; sec++ { 94 | if *protocol == "zmq" { 95 | zmqTasks(yy, mm, dd, hr, min, sec, "daystate", string(daycount)) 96 | } else if *protocol == "http" { 97 | httpTasks(yy, mm, dd, hr, min, sec, "daystate", string(daycount)) 98 | } 99 | daycount++ 100 | } 101 | } 102 | } 103 | } 104 | 105 | func everyday(fromYear, toYear int) { 106 | for yy := fromYear; yy <= toYear; yy++ { 107 | for mm := 1; mm <= 12; mm++ { 108 | for dd := 1; dd <= days[mm]; dd++ { 109 | everysecond(yy, mm, dd) 110 | } 111 | } 112 | } 113 | } 114 | 115 | func main() { 116 | flag.Parse() 117 | fmt.Printf("client ZeroMQ REP/REQ... at %d, %d\n", *request_port01, *request_port02) 118 | fmt.Println(time.Now()) 119 | //everyday(2001, 2001) 120 | everysecond(2015, 6, 21) 121 | fmt.Println(time.Now()) 122 | fmt.Println(daycount) 123 | /* 124 | //for i := 0; i < 1000000; i++ { 125 | for i := 0; i < 10000; i++ { 126 | _i := fmt.Sprintf("%d", i) 127 | result, err = golzmq.ZmqRequest(zmqSock, "push", "tsds", "2014", "2", "10", "9", "18", "37", "people"+_i, "bob"+_i) 128 | //result, err = golzmq.ZmqRequest(zmqSock, "push", "tsds-csv", "2014", "2", "10", "9", "18", "37", "people"+_i+",bob"+_i) 129 | //result, err = golzmq.ZmqRequest(zmqSock, "read", "tsds-default", "people"+_i) 130 | //result, err = golzmq.ZmqRequest(zmqSock, "delete", "ns-default", "people"+_i) 131 | 132 | //_i := fmt.Sprintf("people:group%d", i) 133 | //result, err = golzmq.ZmqRequest(zmqSock, "push", "tsds-csv-parent", "2014", "2", "10", "9", "18", "37", _i, "myname,bob") 134 | //result, err = golzmq.ZmqRequest(zmqSock, "read", "tsds-default-parent", _i, "myname") 135 | //result, err = golzmq.ZmqRequest(zmqSock, "delete", "ns-default-parent", _i, "myname") 136 | 137 | //ZmqKeyVal("push", "default", "k_"+_i, _i) 138 | //ZmqKey("read", "default", "k_"+_i) 139 | //ZmqKey("delete", "default", "k_"+_i) 140 | } 141 | */ 142 | } 143 | -------------------------------------------------------------------------------- /zxtra/goshare_daemon.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/abhishekkr/gol/golservice" 5 | "github.com/abhishekkr/goshare" 6 | ) 7 | 8 | func main() { 9 | var goshare_service golservice.Funk 10 | goshare_service = func() { goshare.GoShare() } 11 | golservice.Daemon(goshare_service) 12 | } 13 | -------------------------------------------------------------------------------- /zxtra/goshare_server.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/abhishekkr/goshare" 4 | 5 | func main(){ 6 | goshare.GoShare() 7 | } 8 | --------------------------------------------------------------------------------