├── .travis.yml ├── doc.go ├── .gitignore ├── stadiserver └── stadiserver.go ├── examples └── basichttp │ └── basichttp.go ├── LICENSE ├── config.go ├── proxy.go ├── api_server.go ├── api_client.go ├── README.md ├── stadis_test.go ├── conn.go └── topology.go /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.2.1 -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | stadis provide distributed system layer on top of localhost on a single machine 3 | 4 | 5 | 6 | 7 | */ 8 | package stadis 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | -------------------------------------------------------------------------------- /stadiserver/stadiserver.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "github.com/coocood/stadis" 6 | "net/http" 7 | ) 8 | 9 | var port = flag.String("port", "", "the port to listen on") 10 | 11 | func main() { 12 | flag.Parse() 13 | if *port != "" { 14 | stadis.Cli.ApiAddr = "localhost:" + *port 15 | } 16 | http.ListenAndServe(stadis.Cli.ApiAddr, stadis.NewApiServer()) 17 | } 18 | -------------------------------------------------------------------------------- /examples/basichttp/basichttp.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/coocood/stadis" 6 | "io" 7 | "log" 8 | "net/http" 9 | "os" 10 | "time" 11 | ) 12 | 13 | func main() { 14 | //Start a api server. 15 | go http.ListenAndServe(stadis.Cli.ApiAddr, stadis.NewApiServer()) 16 | //Setup the http client dialer. 17 | http.DefaultTransport.(*http.Transport).Dial = stadis.NewDialFunc("matter.metal.gold", 5*time.Second) 18 | 19 | eagleListener, err := stadis.Listen("tcp", "localhost:8585", "animal.air.eagle") 20 | if err != nil { 21 | log.Fatal(err) 22 | } 23 | 24 | http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request){ 25 | w.Write([]byte("hello\n")) 26 | }) 27 | 28 | go http.Serve(eagleListener, nil) 29 | 30 | time.Sleep(time.Millisecond) 31 | before := time.Now() 32 | resp, err := http.Get("http://localhost:8585/") 33 | if err != nil { 34 | log.Fatal(err) 35 | } 36 | io.Copy(os.Stdout, resp.Body) 37 | resp.Body.Close() 38 | //the latency should be a little more than 888ms which is define at stadis.DefaultConfig. 39 | fmt.Println("latency:", time.Now().Sub(before)) 40 | } 41 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) 2014 Ewan Chou. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | package stadis 2 | 3 | import "time" 4 | 5 | var DefaultConfig = []byte(` 6 | { 7 | "DcDefault":{"Latency":100000000}, 8 | "RackDefault":{"Latency":10000000}, 9 | "HostDefault":{"Latency":1000000}, 10 | "DataCenters":[ 11 | { 12 | "Name":"animal", 13 | "Racks":[ 14 | { 15 | "Name":"land", 16 | "Hosts":[{"Name":"tiger"},{"Name":"lion"},{"Name":"wolf"}] 17 | }, 18 | { 19 | "Name":"sea", 20 | "Hosts":[{"Name":"shark"},{"Name":"whale"},{"Name":"cod"}] 21 | }, 22 | { 23 | "Name":"air", 24 | "Hosts":[{"Name":"eagle"},{"Name":"crow"},{"Name":"owl"}] 25 | } 26 | ] 27 | }, 28 | { 29 | "Name":"plant", 30 | "Racks":[ 31 | { 32 | "Name":"fruit", 33 | "Hosts":[{"Name":"apple"},{"Name":"pear"},{"Name":"grape"}] 34 | }, 35 | { 36 | "Name":"crop", 37 | "Hosts":[{"Name":"corn"},{"Name":"rice"},{"Name":"wheat"}] 38 | }, 39 | { 40 | "Name":"flower", 41 | "Hosts":[{"Name":"rose"},{"Name":"lily"},{"Name":"lotus"}] 42 | } 43 | ] 44 | }, 45 | { 46 | "Name":"matter", 47 | "Racks":[ 48 | { 49 | "Name":"metal", 50 | "Hosts":[{"Name":"gold"},{"Name":"silver"},{"Name":"iron"}] 51 | }, 52 | { 53 | "Name":"gem", 54 | "Hosts":[{"Name":"ruby"},{"Name":"ivory"},{"Name":"pearl"}] 55 | }, 56 | { 57 | "Name":"liquid", 58 | "Hosts":[{"Name":"water"},{"Name":"oil"},{"Name":"wine"}] 59 | } 60 | ] 61 | } 62 | ] 63 | } 64 | `) 65 | 66 | type ConnState struct { 67 | Latency time.Duration //the sleep time before write data to the connection. 68 | OK bool //If not ok, local process should not write data to the connection. 69 | } 70 | 71 | type NodeState struct { 72 | Latency time.Duration 73 | InternalDown bool 74 | ExternalDown bool 75 | } 76 | 77 | type Config struct { 78 | DcDefault *NodeState 79 | RackDefault *NodeState 80 | HostDefault *NodeState 81 | DataCenters []*DataCenter 82 | } 83 | 84 | type DataCenter struct { 85 | RackDefault *NodeState 86 | HostDefault *NodeState 87 | Name string 88 | Racks []*Rack 89 | *NodeState 90 | } 91 | 92 | type Rack struct { 93 | HostDefault *NodeState 94 | Name string 95 | Hosts []*Host 96 | *NodeState 97 | } 98 | 99 | type Host struct { 100 | Name string 101 | Ports []int 102 | *NodeState 103 | } 104 | -------------------------------------------------------------------------------- /proxy.go: -------------------------------------------------------------------------------- 1 | package stadis 2 | 3 | import ( 4 | "io" 5 | "log" 6 | "net" 7 | "sync" 8 | "time" 9 | ) 10 | 11 | type proxyServer struct { 12 | mu sync.RWMutex 13 | clientName string 14 | proxyName string 15 | proxyPort string 16 | originAddr string 17 | listener net.Listener 18 | } 19 | 20 | func newProxyServer(clientName, proxyName, proxyPort, originAddr string) (ps *proxyServer, err error) { 21 | ps = new(proxyServer) 22 | ps.clientName = clientName 23 | ps.proxyName = proxyName 24 | ps.proxyPort = proxyPort 25 | ps.originAddr = originAddr 26 | ps.listener, err = net.Listen("tcp", "localhost:"+ps.proxyPort) 27 | if err != nil { 28 | log.Println("failed to listen proxy", err) 29 | return 30 | } 31 | err = Cli.ServerStarted(ps.proxyName, ps.proxyPort) 32 | if err != nil { 33 | log.Println("at newProxyServer", err) 34 | return 35 | } 36 | return 37 | } 38 | 39 | func (ps *proxyServer) serve() { 40 | for { 41 | downstream, err := ps.listener.Accept() 42 | if err != nil { 43 | return 44 | } 45 | 46 | originConn, err := net.DialTimeout("tcp", ps.originAddr, time.Second) 47 | if err != nil { 48 | log.Println(err) 49 | downstream.Close() 50 | return 51 | } 52 | clientPort := remotePort(downstream) 53 | err = Cli.ClientConnected(ps.getClientName(), clientPort) 54 | if err != nil { 55 | log.Printf("%v, clinetPort:%s\n", err, clientPort) 56 | return 57 | } 58 | 59 | //the delay and failing happens on this upstream conn. 60 | upstream, err := newConnection(originConn, clientPort, ps.proxyPort) 61 | if err != nil { 62 | log.Println(err) 63 | downstream.Close() 64 | originConn.Close() 65 | return 66 | } 67 | go handleCopy(downstream, upstream) 68 | } 69 | } 70 | 71 | func (ps *proxyServer) close() (err error) { 72 | ps.listener.Close() 73 | err = Cli.ServerStopped(ps.proxyName, ps.proxyPort) 74 | return 75 | } 76 | 77 | func (ps *proxyServer) setClientName(clientName string) { 78 | ps.mu.Lock() 79 | ps.clientName = clientName 80 | ps.mu.Unlock() 81 | } 82 | 83 | func (ps *proxyServer) getClientName() (clientName string) { 84 | ps.mu.RLock() 85 | clientName = ps.clientName 86 | ps.mu.RUnlock() 87 | return 88 | } 89 | 90 | func handleCopy(downstream, upstream net.Conn) { 91 | done := make(chan bool) 92 | go func() { 93 | n, err := io.Copy(downstream, upstream) 94 | if err != nil { 95 | log.Println(n, err) 96 | } 97 | done <- true 98 | }() 99 | n, err := io.Copy(upstream, downstream) 100 | if err != nil { 101 | log.Println(n, err) 102 | } 103 | <-done 104 | upstream.Close() 105 | downstream.Close() 106 | err = Cli.ClientDisconnected(remotePort(downstream)) 107 | if err != nil { 108 | log.Println(err) 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /api_server.go: -------------------------------------------------------------------------------- 1 | package stadis 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "log" 8 | "net/http" 9 | "strconv" 10 | "sync" 11 | "time" 12 | ) 13 | 14 | //The API server holds the state of topology and proxy servers, serve requests from API client. 15 | type ApiServer struct { 16 | mu sync.RWMutex 17 | topo *topology 18 | proxies map[string]*proxyServer 19 | } 20 | 21 | func NewApiServer() (ms *ApiServer) { 22 | ms = new(ApiServer) 23 | ms.proxies = make(map[string]*proxyServer) 24 | ms.topo, _ = newTopology(bytes.NewReader(DefaultConfig)) 25 | return 26 | } 27 | 28 | func (s *ApiServer) connState(w http.ResponseWriter, r *http.Request) { 29 | clientPort := intFormValue(r, "clientPort") 30 | if clientPort == 0 { 31 | http.Error(w, "'clientPort' required", 400) 32 | return 33 | } 34 | serverPort := intFormValue(r, "serverPort") 35 | if serverPort == 0 { 36 | http.Error(w, "'serverPort' required", 400) 37 | return 38 | } 39 | s.mu.Lock() 40 | topo := s.topo 41 | s.mu.Unlock() 42 | updateCh := topo.getUpdateChannel() 43 | connState, err := topo.connState(clientPort, serverPort) 44 | if err != nil { 45 | log.Println(err) 46 | http.Error(w, err.Error(), 400) 47 | return 48 | } 49 | oldStateStr := r.Header.Get("If-None-Match") 50 | if oldStateStr == "" { 51 | connStateBytes, _ := json.Marshal(connState) 52 | w.Write(connStateBytes) 53 | return 54 | } 55 | 56 | var oldState ConnState 57 | err = json.Unmarshal([]byte(oldStateStr), &oldState) 58 | if err != nil { 59 | log.Println(err) 60 | http.Error(w, "invalid If-None-Match header", 400) 61 | return 62 | } 63 | if oldState == connState { 64 | //long-poling 65 | select { 66 | case <-time.After(time.Second * 3): 67 | case <-updateCh: 68 | connState, _ = topo.connState(clientPort, serverPort) 69 | } 70 | } 71 | if oldState == connState { 72 | w.WriteHeader(304) 73 | } else { 74 | connStateBytes, _ := json.Marshal(connState) 75 | w.Write(connStateBytes) 76 | } 77 | } 78 | 79 | func (s *ApiServer) dialState(w http.ResponseWriter, r *http.Request) { 80 | clientName := r.FormValue("clientName") 81 | if clientName == "" { 82 | http.Error(w, "'clientName' required", 400) 83 | return 84 | } 85 | serverPort := intFormValue(r, "serverPort") 86 | if serverPort == 0 { 87 | http.Error(w, "'serverPort' required", 400) 88 | return 89 | } 90 | connState, err := s.topo.dialState(clientName, serverPort) 91 | if err != nil { 92 | http.Error(w, err.Error(), 400) 93 | return 94 | } 95 | jsonBytes, _ := json.Marshal(connState) 96 | w.Write(jsonBytes) 97 | } 98 | 99 | func (s *ApiServer) serverPort(w http.ResponseWriter, r *http.Request) { 100 | port := intFormValue(r, "port") 101 | if port == 0 { 102 | http.Error(w, "'port' required", 400) 103 | return 104 | } 105 | name := r.FormValue("name") 106 | if name == "" { 107 | http.Error(w, "'name' required", 400) 108 | return 109 | } 110 | var err error 111 | switch r.Method { 112 | case "POST": 113 | err = s.topo.addServerPort(name, port) 114 | case "DELETE": 115 | err = s.topo.removeServerPort(name, port) 116 | } 117 | if err != nil { 118 | http.Error(w, err.Error(), 400) 119 | } 120 | } 121 | 122 | func (s *ApiServer) clientPort(w http.ResponseWriter, r *http.Request) { 123 | port := intFormValue(r, "port") 124 | if port == 0 { 125 | http.Error(w, "'port' required", 400) 126 | return 127 | } 128 | var err error 129 | switch r.Method { 130 | case "POST": 131 | name := r.FormValue("name") 132 | if name == "" { 133 | http.Error(w, "'name' required", 400) 134 | return 135 | } 136 | err = s.topo.addClientPort(name, port) 137 | case "DELETE": 138 | err = s.topo.removeClientPort(port) 139 | } 140 | if err != nil { 141 | http.Error(w, err.Error(), 400) 142 | } 143 | } 144 | 145 | func (s *ApiServer) nodeState(w http.ResponseWriter, r *http.Request) { 146 | name := r.FormValue("name") 147 | nodeState, err := s.topo.nodeState(name) 148 | if err != nil { 149 | http.Error(w, err.Error(), 400) 150 | return 151 | } 152 | if r.Method == "GET" { 153 | data, _ := json.Marshal(nodeState) 154 | w.Write(data) 155 | } else if r.Method == "POST" { 156 | var newState NodeState 157 | decoder := json.NewDecoder(r.Body) 158 | err = decoder.Decode(&newState) 159 | if err != nil { 160 | http.Error(w, err.Error(), 400) 161 | return 162 | } 163 | s.topo.setNodeState(name, newState) 164 | } 165 | } 166 | 167 | func (s *ApiServer) postConfig(w http.ResponseWriter, r *http.Request) { 168 | topo, err := newTopology(r.Body) 169 | if err != nil { 170 | http.Error(w, err.Error(), 400) 171 | return 172 | } 173 | s.mu.Lock() 174 | if s.topo != nil { 175 | close(s.topo.getUpdateChannel()) 176 | } 177 | s.topo = topo 178 | s.mu.Unlock() 179 | } 180 | 181 | func (s *ApiServer) proxy(w http.ResponseWriter, r *http.Request) { 182 | 183 | proxyPort := r.FormValue("proxyPort") 184 | if proxyPort == "" { 185 | http.Error(w, "'proxyPort' required", 400) 186 | return 187 | } 188 | ps := s.getProxy(proxyPort) 189 | var err error 190 | switch r.Method { 191 | case "POST": 192 | if ps != nil { 193 | errStr := "proxy port is taken" 194 | log.Println(errStr) 195 | http.Error(w, errStr, 400) 196 | return 197 | } 198 | proxyName := r.FormValue("proxyName") 199 | if proxyName == "" { 200 | http.Error(w, "'proxyName' required", 400) 201 | return 202 | } 203 | originAddr := r.FormValue("originAddr") 204 | if originAddr == "" { 205 | http.Error(w, "'originAddr' required", 400) 206 | return 207 | } 208 | clientName := r.FormValue("clientName") 209 | if clientName == "" { 210 | http.Error(w, "'clientName' required", 400) 211 | return 212 | } 213 | ps, err = newProxyServer(clientName, proxyName, proxyPort, originAddr) 214 | if err != nil { 215 | log.Println(err) 216 | http.Error(w, err.Error(), 400) 217 | return 218 | } 219 | go ps.serve() 220 | s.setProxy(proxyPort, ps) 221 | case "PUT": 222 | if ps == nil { 223 | errStr := "proxy server not found" 224 | log.Println(errStr) 225 | http.Error(w, errStr, 404) 226 | return 227 | } 228 | clientName := r.FormValue("clientName") 229 | if clientName == "" { 230 | http.Error(w, "'clientName' required", 400) 231 | return 232 | } 233 | ps.clientName = clientName 234 | case "DELETE": 235 | if ps == nil { 236 | errStr := "proxy server not found" 237 | log.Println(errStr) 238 | http.Error(w, errStr, 404) 239 | return 240 | } 241 | err = ps.close() 242 | if err != nil { 243 | log.Println(err) 244 | return 245 | } 246 | s.setProxy(proxyPort, nil) 247 | } 248 | } 249 | 250 | func (s *ApiServer) getProxy(port string) (ps *proxyServer) { 251 | s.mu.RLock() 252 | ps = s.proxies[port] 253 | s.mu.RUnlock() 254 | return 255 | } 256 | 257 | func (s *ApiServer) setProxy(port string, ps *proxyServer) { 258 | s.mu.Lock() 259 | s.proxies[port] = ps 260 | s.mu.Unlock() 261 | } 262 | 263 | //dump the node state, if 'name' is empty, the whole topology will be dumped. 264 | func (s *ApiServer) DumpNode(name string) { 265 | s.mu.RLock() 266 | defer s.mu.RUnlock() 267 | if name == "" { 268 | fmt.Println(s.topo) 269 | return 270 | } 271 | node, err := s.topo.lookup(name) 272 | if err != nil { 273 | log.Println(err) 274 | return 275 | } 276 | fmt.Println(node) 277 | } 278 | 279 | func (s *ApiServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { 280 | if r.URL.Path == "/config" && r.Method == "POST" { 281 | s.postConfig(w, r) 282 | return 283 | } 284 | s.mu.Lock() 285 | topo := s.topo 286 | s.mu.Unlock() 287 | if topo == nil { 288 | http.Error(w, "server uninitialized.", 400) 289 | return 290 | } 291 | switch r.URL.Path { 292 | case "/connState": 293 | s.connState(w, r) 294 | case "/nodeState": 295 | s.nodeState(w, r) 296 | case "/serverPort": 297 | s.serverPort(w, r) 298 | case "/clientPort": 299 | s.clientPort(w, r) 300 | case "/dialState": 301 | s.dialState(w, r) 302 | case "/proxy": 303 | s.proxy(w, r) 304 | default: 305 | w.WriteHeader(404) 306 | } 307 | } 308 | 309 | //just return 0 if form value is empty string. 310 | func intFormValue(r *http.Request, key string) (intVal int) { 311 | val := r.FormValue(key) 312 | intVal, _ = strconv.Atoi(val) 313 | return 314 | } 315 | -------------------------------------------------------------------------------- /api_client.go: -------------------------------------------------------------------------------- 1 | package stadis 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "io/ioutil" 10 | "log" 11 | "net/http" 12 | ) 13 | 14 | //Default API client 15 | var Cli = &ApiClient{ApiAddr: "localhost:8989"} 16 | 17 | var httpClient = &http.Client{Transport: &http.Transport{}} 18 | 19 | type ApiClient struct { 20 | ApiAddr string 21 | } 22 | 23 | //Register the server port on API server. 24 | //Should be called after open a listener. 25 | func (client *ApiClient) ServerStarted(name, port string) error { 26 | return client.serverPort("POST", name, port) 27 | } 28 | 29 | //Unregister the server port on API server. 30 | //Should be called after closed a listener. 31 | func (client *ApiClient) ServerStopped(name, port string) error { 32 | return client.serverPort("DELETE", name, port) 33 | } 34 | 35 | func (client *ApiClient) serverPort(method, name, port string) (err error) { 36 | url := fmt.Sprintf("http://%v/serverPort?name=%v&port=%v", client.ApiAddr, name, port) 37 | req, _ := http.NewRequest(method, url, nil) 38 | resp, err := httpClient.Do(req) 39 | if err != nil { 40 | log.Println(err) 41 | return 42 | } 43 | defer resp.Body.Close() 44 | if resp.StatusCode != 200 { 45 | err = errorFromResponse(resp) 46 | log.Println(err) 47 | return 48 | } 49 | return 50 | } 51 | 52 | 53 | //Register client port at API server. 54 | //Should be called after client created a connection. 55 | func (client *ApiClient) ClientConnected(name, port string) error { 56 | return client.clientPort("POST", name, port) 57 | } 58 | 59 | //Unregister client port at API server. 60 | //Should be called after client closed a connection. 61 | func (client *ApiClient) ClientDisconnected(port string) error { 62 | return client.clientPort("DELETE", "", port) 63 | } 64 | 65 | func (client *ApiClient) clientPort(method, name, port string) (err error) { 66 | url := fmt.Sprintf("http://%v/clientPort?name=%v&port=%v", 67 | client.ApiAddr, name, port) 68 | req, _ := http.NewRequest(method, url, nil) 69 | resp, err := httpClient.Do(req) 70 | if err != nil { 71 | log.Println(err) 72 | return 73 | } 74 | defer resp.Body.Close() 75 | if resp.StatusCode != 200 { 76 | err = errorFromResponse(resp) 77 | log.Println(err) 78 | return 79 | } 80 | return 81 | } 82 | 83 | //Get the dial state which can be used to simulate network latency or failure before actually dial the server. 84 | func (client *ApiClient) DialState(clientName, serverPort string) (state ConnState, err error) { 85 | url := fmt.Sprintf("http://%v/dialState?clientName=%v&serverPort=%v", client.ApiAddr, clientName, serverPort) 86 | resp, err := httpClient.Get(url) 87 | if err != nil { 88 | log.Println(err) 89 | return 90 | } 91 | defer resp.Body.Close() 92 | if resp.StatusCode != 200 { 93 | err = errorFromResponse(resp) 94 | log.Println(err) 95 | return 96 | } 97 | data, err := ioutil.ReadAll(resp.Body) 98 | if err != nil { 99 | log.Println(err) 100 | return 101 | } 102 | err = json.Unmarshal(data, &state) 103 | if err != nil { 104 | log.Println(err) 105 | return 106 | } 107 | return 108 | } 109 | 110 | //Get the current connection state for the connection between 'clientPort' and 'serverPort'. 111 | //If 'oldState' is provided, this request will do long-polling, blocking for a few seconds 112 | //before get response if there is no new state updated. 113 | func (client *ApiClient) ConnState(clientPort, serverPort string, oldState *ConnState) (state *ConnState, err error) { 114 | url := fmt.Sprintf("http://%v/connState?clientPort=%v&serverPort=%v", client.ApiAddr, clientPort, serverPort) 115 | req, _ := http.NewRequest("GET", url, nil) 116 | if oldState != nil { 117 | jsonBytes, _ := json.Marshal(oldState) 118 | req.Header.Add("If-None-Match", string(jsonBytes)) 119 | } 120 | resp, err := httpClient.Do(req) 121 | if err != nil { 122 | log.Println(err) 123 | return 124 | } 125 | if resp.StatusCode == 304 { 126 | state = oldState 127 | return 128 | } 129 | if resp.StatusCode != 200 { 130 | err = errorFromResponse(resp) 131 | log.Println(err) 132 | return 133 | } 134 | data, err := ioutil.ReadAll(resp.Body) 135 | if err != nil { 136 | log.Println(err) 137 | return 138 | } 139 | state = new(ConnState) 140 | err = json.Unmarshal(data, state) 141 | if err != nil { 142 | log.Println(err) 143 | return 144 | } 145 | return 146 | } 147 | 148 | //Get the node state by node name 149 | func (client *ApiClient) NodeState(name string) (nodeState NodeState, err error) { 150 | url := fmt.Sprintf("http://%v/nodeState?name=%v", client.ApiAddr, name) 151 | resp, err := httpClient.Get(url) 152 | if err != nil { 153 | log.Println(err) 154 | return 155 | } 156 | defer resp.Body.Close() 157 | if resp.StatusCode != 200 { 158 | err = errorFromResponse(resp) 159 | log.Println(err) 160 | return 161 | } 162 | data, err := ioutil.ReadAll(resp.Body) 163 | if err != nil { 164 | log.Println(err) 165 | return 166 | } 167 | err = json.Unmarshal(data, &nodeState) 168 | if err != nil { 169 | log.Println(err) 170 | return 171 | } 172 | return 173 | } 174 | 175 | //Set the node's state, the name can be dc, rack or host depends on the number of dot in the name. 176 | //If latency of the nodeState is zero, the target node's latency will stay unchanged. 177 | func (client *ApiClient) UpdateNodeState(name string, nodeState NodeState) (err error) { 178 | url := fmt.Sprintf("http://%v/nodeState?name=%v", client.ApiAddr, name) 179 | jsonData, _ := json.Marshal(nodeState) 180 | resp, err := httpClient.Post(url, "application/json", bytes.NewReader(jsonData)) 181 | if err != nil { 182 | log.Println(err) 183 | return 184 | } 185 | defer resp.Body.Close() 186 | if resp.StatusCode != 200 { 187 | err = errorFromResponse(resp) 188 | log.Println(err) 189 | return 190 | } 191 | return 192 | } 193 | 194 | //Update the API server config, the topology on API server will be rebuild. 195 | //You can use the 'DefaultConfig' as a base config, then do some modification to meet your requirement. 196 | func (client *ApiClient) UpdateConfig(reader io.Reader) (err error) { 197 | url := fmt.Sprintf("http://%v/config", client.ApiAddr) 198 | resp, err := httpClient.Post(url, "application/json", reader) 199 | if err != nil { 200 | log.Println(err) 201 | return 202 | } 203 | defer resp.Body.Close() 204 | if resp.StatusCode != 200 { 205 | err = errorFromResponse(resp) 206 | log.Println(err) 207 | return 208 | } 209 | return 210 | } 211 | 212 | //Start a proxy server in API server process. 213 | //The 'clientName' going to be used to register a client port at API server when the proxy server accepts a new connection, 214 | //For example, if you pass 'matter.metal.gold' as clientName, every client connected to the proxy server will be considered 215 | //located at 'matter.metal.gold'. 216 | //This can be updated after the proxy is open, but only one 'clientName' can be used by a proxy server at a time. 217 | func (client *ApiClient) StartProxy(clientName, proxyName, proxyPort, originAddr string) (err error) { 218 | return client.proxy("POST", clientName, proxyName, proxyPort, originAddr) 219 | } 220 | 221 | //Update the 'clientName' for a proxy server, so future connection will be registered with the new name. 222 | //This will not affect connections that have been registered already. 223 | func (client *ApiClient) UpdateProxy(clientName, proxyPort string) (err error) { 224 | return client.proxy("PUT", clientName, "", proxyPort, "") 225 | } 226 | 227 | //Stop a proxy server 228 | func (client *ApiClient) StopProxy(proxyPort string) (err error) { 229 | return client.proxy("DELETE", "", "", proxyPort, "") 230 | } 231 | 232 | func (client *ApiClient) proxy(method, clientName, proxyName, proxyPort, originAddr string) (err error) { 233 | url := fmt.Sprintf("http://%v/proxy?clientName=%s&proxyName=%s&proxyPort=%s&originAddr=%s", 234 | client.ApiAddr, clientName, proxyName, proxyPort, originAddr) 235 | req, _ := http.NewRequest(method, url, nil) 236 | resp, err := httpClient.Do(req) 237 | if err != nil { 238 | log.Println(err) 239 | return 240 | } 241 | defer resp.Body.Close() 242 | if resp.StatusCode != 200 { 243 | err = errorFromResponse(resp) 244 | log.Println(err) 245 | return 246 | } 247 | return 248 | } 249 | 250 | func errorFromResponse(resp *http.Response) error { 251 | bodyBytes := make([]byte, resp.ContentLength) 252 | io.ReadFull(resp.Body, bodyBytes) 253 | return errors.New(string(bodyBytes)) 254 | } 255 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ##Stadis - Stand-alone Distributed System 2 | 3 | The easiest way to learn, develop and test a distributed system. 4 | 5 | [![GoDoc](https://godoc.org/github.com/coocood/stadis?status.png)](https://godoc.org/github.com/coocood/stadis) 6 | [![Build Status](https://travis-ci.org/coocood/stadis.png?branch=master)](https://travis-ci.org/coocood/stadis) 7 | 8 | 9 | ##Why should I use it? 10 | 11 | Testing distributed system is very expensive. 12 | 13 | It takes a large amount of resources, takes a lot of time to deploy and config. 14 | 15 | More importantly, as your application is running on multiple machine, 16 | it's nearly impossible to coordinate the network state with your application state. 17 | as a result, there will be lots of error handling code leave untested, 18 | which can cause serious problem once that actually happen. 19 | 20 | With stadis, you can run a multi-data-center distributed system on a single machine. 21 | 22 | You can change the topology and node state at any time by a single API call. 23 | Everything happens exactly the way you want. 24 | Every network error handling code can be easily covered. 25 | 26 | ##Get started 27 | 28 | Require Go 1.2+ installed. 29 | 30 | Stadis can be used as a library in Go application. 31 | 32 | For applications written in other programming language, stadis can be used as a proxy server. 33 | 34 | ###Use stadis as a library in Go program. 35 | 36 | Hello world example 37 | 38 | package main 39 | 40 | import ( 41 | "fmt" 42 | "github.com/coocood/stadis" 43 | "io" 44 | "log" 45 | "net/http" 46 | "os" 47 | "time" 48 | ) 49 | 50 | func main() { 51 | //Start a api server. 52 | go http.ListenAndServe(stadis.Cli.ApiAddr, stadis.NewApiServer()) 53 | //Setup the http client dialer. 54 | http.DefaultTransport.(*http.Transport).Dial = stadis.NewDialFunc("matter.metal.gold", 5*time.Second) 55 | 56 | eagleListener, err := stadis.Listen("tcp", "localhost:8585", "animal.air.eagle") 57 | if err != nil { 58 | log.Fatal(err) 59 | } 60 | 61 | http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request){ 62 | w.Write([]byte("hello\n")) 63 | }) 64 | 65 | go http.Serve(eagleListener, nil) 66 | 67 | time.Sleep(time.Millisecond) 68 | before := time.Now() 69 | resp, err := http.Get("http://localhost:8585/") 70 | if err != nil { 71 | log.Fatal(err) 72 | } 73 | io.Copy(os.Stdout, resp.Body) 74 | resp.Body.Close() 75 | //the latency should be a little more than 888ms which is define at stadis.DefaultConfig. 76 | fmt.Println("latency:", time.Now().Sub(before)) 77 | } 78 | 79 | ###Run stadis as a API/proxy server. 80 | 81 | Build and Run 82 | 83 | go get github.com/coocood/stadis 84 | go build github.com/coocood/stadis/stadiserver 85 | ./stadiserver 86 | 87 | Now, the stadis API server is started at port `8989`. 88 | 89 | Then start a trivial http server as the origin server on port `12345` by run 90 | 91 | go run $GOROOT/src/pkg/net/http/triv.go 92 | 93 | Then call the REST API to open a proxy 94 | 95 | curl -X POST 'http://localhost:8989/proxy?clientName=matter.metal.gold&proxyName=animal.air.eagle&proxyPort=8586&originAddr=localhost:12345' 96 | 97 | We can ask the API server for the dial state: 98 | 99 | curl 'http://localhost:8989/dialState?clientName=matter.metal.gold&serverPort=8586' 100 | 101 | You will get dial state like `{"Latency":444000000,"OK":true}`, the latency unit is nano second. 102 | Normally you should sleep for that amount of time in your program before actually dial the proxy server. 103 | If 'OK' is 'false' you should return error or throw an exception. 104 | 105 | Then request the proxy server and record the time. 106 | 107 | time curl -i 'http://localhost:8586/counter' 108 | 109 | The real time should be a little more than 444ms, which is the roundtrip time from 'matter.metal.gold' to 'animal.air.eagle'. 110 | 111 | ##Configuration 112 | Stadis server does not use any config file, it starts with a default configuration, and you can update it by REST API call. 113 | 114 | Stadis API server maintains a virtual topology which has three level:'DataCenter', 'Rack' and 'Host'. 115 | All of them has a 'NodeState' of attributes 'Name', 'Latency', 'InternalDown' and 'ExternalDown' . 116 | The topology contains multiple 'DataCenter', which contains multiple 'Rack', which in turn contains 117 | multiple 'Host'. 118 | 119 | We all know naming is hard, so I did the hard work for you. 120 | After spent many hours, I managed to come up with 39 node names. 121 | The topology has three data centers, each data center has three racks, each rack has three hosts. 122 | 123 | { 124 | "DcDefault":{"Latency":100000000}, 125 | "RackDefault":{"Latency":10000000}, 126 | "HostDefault":{"Latency":1000000}, 127 | "DataCenters":[ 128 | { 129 | "Name":"animal", 130 | "Racks":[ 131 | { 132 | "Name":"land", 133 | "Hosts":[{"Name":"tiger"},{"Name":"lion"},{"Name":"wolf"}] 134 | }, 135 | { 136 | "Name":"sea", 137 | "Hosts":[{"Name":"shark"},{"Name":"whale"},{"Name":"cod"}] 138 | }, 139 | { 140 | "Name":"air", 141 | "Hosts":[{"Name":"eagle"},{"Name":"crow"},{"Name":"owl"}] 142 | } 143 | ] 144 | }, 145 | { 146 | "Name":"plant", 147 | "Racks":[ 148 | { 149 | "Name":"fruit", 150 | "Hosts":[{"Name":"apple"},{"Name":"pear"},{"Name":"grape"}] 151 | }, 152 | { 153 | "Name":"crop", 154 | "Hosts":[{"Name":"corn"},{"Name":"rice"},{"Name":"wheat"}] 155 | }, 156 | { 157 | "Name":"flower", 158 | "Hosts":[{"Name":"rose"},{"Name":"lily"},{"Name":"lotus"}] 159 | } 160 | ] 161 | }, 162 | { 163 | "Name":"matter", 164 | "Racks":[ 165 | { 166 | "Name":"metal", 167 | "Hosts":[{"Name":"gold"},{"Name":"silver"},{"Name":"iron"}] 168 | }, 169 | { 170 | "Name":"gem", 171 | "Hosts":[{"Name":"ruby"},{"Name":"ivory"},{"Name":"pearl"}] 172 | }, 173 | { 174 | "Name":"liquid", 175 | "Hosts":[{"Name":"water"},{"Name":"oil"},{"Name":"wine"}] 176 | } 177 | ] 178 | } 179 | ] 180 | } 181 | 182 | In default configuration, each data center has 100ms latency, each rack has 10ms latency, each host has 1ms latency. 183 | 184 | So the latency from 'matter.metal.gold' to 'animal.air.eagle' should be "1ms+10ms+100ms+100ms+10ms+1ms = 222ms" 185 | 186 | The round trip time should be 444ms, so the total time to create a connection 187 | and then make a http request from 'gold' to 'eagle' should be a little more than 888ms. 188 | 189 | ##REST API 190 | 191 | - Update configuration with json payload like the default config shown above. 192 | 193 | POST /config 194 | 195 | 196 | - Update node state with json body like `{"Latency":10000000,"InternalDown":false,"ExternalDown":true}` 197 | 198 | POST /nodeState?name=%s 199 | 200 | The 'name' parameter is the node name in the topology, e.g. "animal.air.eagle". 201 | 202 | 203 | - Get node state: 204 | 205 | GET /nodeState?name=%s 206 | 207 | 208 | - Start a proxy: 209 | 210 | POST /proxy?clientName=%s&proxyName=%s&proxyPort=%s&originAddr=%s 211 | 212 | 'clientName' defines where the client process is located in the topology. 213 | 'proxyName' defines where the proxy is located in the topology. 214 | 'proxyPort' will be opened and accepts incoming connection. 215 | 'originAddr' is the origin server address for the proxy, it can be an address in another machine, 216 | eg. `192.168.1.100:12345`, so if you can not run the origin server on your localhost, you can proxy it. 217 | 218 | 219 | - Stop a proxy: 220 | 221 | DELETE /proxy?proxyPort=%s 222 | 223 | 224 | - Get dial state from a client to server. 225 | 226 | GET /dialState?clientName={clientName}&serverPort={serverPort} 227 | 228 | The response is a json object like `{"Latency":10000000,"OK":true}` 229 | 230 | 231 | - Get the current connection state after that connection has been created. 232 | 233 | GET /connState?clientPort={clientPort}&serverPort={serverPort} 234 | 235 | ##Performance 236 | 237 | Stadis adds an extra layer on top of tcp connection, the throughput is greatly decreased. 238 | On my laptop, a proxy connection with 1ms latency gets about 30MB/s throughput. 239 | I think it is sufficient for most of applications for testing purpose. 240 | Higher latency gets lower throughput which is pretty much the way raw connections work. 241 | 242 | ##LICENSE 243 | 244 | The MIT License 245 | -------------------------------------------------------------------------------- /stadis_test.go: -------------------------------------------------------------------------------- 1 | package stadis 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "io/ioutil" 7 | "log" 8 | "net" 9 | "net/http" 10 | "testing" 11 | "time" 12 | ) 13 | 14 | const ( 15 | appleHostName = "plant.fruit.apple" 16 | lionHostName = "animal.land.lion" 17 | tigerHostName = "animal.land.tiger" 18 | ) 19 | 20 | var defaultServer = NewApiServer() 21 | 22 | func init() { 23 | log.SetFlags(log.Lshortfile) 24 | go http.ListenAndServe(Cli.ApiAddr, defaultServer) 25 | time.Sleep(time.Millisecond) 26 | } 27 | 28 | func resetDefaultServer() (err error) { 29 | err = Cli.UpdateConfig(bytes.NewReader(DefaultConfig)) 30 | if err != nil { 31 | log.Println(err) 32 | return 33 | } 34 | return 35 | } 36 | 37 | func TestApi(t *testing.T) { 38 | err := resetDefaultServer() 39 | if err != nil { 40 | t.Fatal(err) 41 | } 42 | 43 | lionPort := "30001" 44 | Cli.UpdateNodeState("animal.land", NodeState{ExternalDown: true}) 45 | Cli.ServerStarted(lionHostName, lionPort) 46 | 47 | appleDialLion, _ := Cli.DialState(appleHostName, lionPort) 48 | expectedState := ConnState{ 49 | Latency: 3 * time.Minute, 50 | } 51 | if expectedState != appleDialLion { 52 | t.Fatal("wrong state for apple dial lion, expected", expectedState, "actual", appleDialLion) 53 | } 54 | 55 | applePort := "30003" 56 | Cli.ServerStarted(appleHostName, applePort) 57 | 58 | appleDialApple, _ := Cli.DialState(appleHostName, applePort) 59 | 60 | expectedState = ConnState{ 61 | OK: true, 62 | Latency: 0, 63 | } 64 | if expectedState != appleDialApple { 65 | t.Fatal("latency for connection between processes on the same host should be 0, expected", 66 | expectedState, "actual ", appleDialApple) 67 | } 68 | 69 | tigerPort := "30011" 70 | Cli.ServerStarted(tigerHostName, tigerPort) 71 | Cli.UpdateNodeState("animal.land", NodeState{ExternalDown: false}) 72 | 73 | appleDialTiger, _ := Cli.DialState(appleHostName, tigerPort) 74 | 75 | expectedState = ConnState{ 76 | OK: true, 77 | Latency: 2 * (1 + 10 + 100 + 100 + 10 + 1) * time.Millisecond, 78 | } 79 | if expectedState != appleDialTiger { 80 | defaultServer.DumpNode("") 81 | t.Fatal("latency should be two times the sum of latency of every node along the path, expected", 82 | expectedState, "actual", appleDialTiger) 83 | } 84 | 85 | Cli.UpdateNodeState(appleHostName, NodeState{Latency: 30 * time.Millisecond}) 86 | appleDialTiger, _ = Cli.DialState(appleHostName, tigerPort) 87 | 88 | expectedState = ConnState{ 89 | OK: true, 90 | Latency: 2 * (30 + 10 + 100 + 100 + 10 + 1) * time.Millisecond, 91 | } 92 | if expectedState != appleDialTiger { 93 | t.Fatal("wrong state for apple dial tiger, expected", expectedState, "actual", appleDialTiger) 94 | } 95 | 96 | Cli.UpdateNodeState("plant.fruit", NodeState{ExternalDown: true}) 97 | appleDialTiger, _ = Cli.DialState(appleHostName, tigerPort) 98 | 99 | expectedState = ConnState{ 100 | OK: false, 101 | Latency: 3 * time.Minute, 102 | } 103 | if appleDialTiger != expectedState { 104 | defaultServer.DumpNode("") 105 | t.Fatal("wrong state for apple dial tiger, expected", expectedState, "actual", appleDialTiger) 106 | } 107 | 108 | //set NodeState to internal down, the expected state will stay the same. 109 | Cli.UpdateNodeState("plant.fruit", NodeState{InternalDown: true}) 110 | 111 | appleDialTiger, _ = Cli.DialState(appleHostName, tigerPort) 112 | if appleDialTiger != expectedState { 113 | t.Fatal("wrong state for apple dial tiger, expected", expectedState, "actual", appleDialTiger) 114 | } 115 | 116 | Cli.UpdateNodeState("plant.fruit", NodeState{}) 117 | appleDialTiger, _ = Cli.DialState(appleHostName, tigerPort) 118 | if appleDialTiger.OK != true { 119 | t.Fatal("apple dial tiger should be ok.") 120 | } 121 | 122 | Cli.UpdateNodeState("plant", NodeState{InternalDown: true}) 123 | 124 | appleDialTiger, _ = Cli.DialState(appleHostName, tigerPort) 125 | if appleDialTiger.OK != false { 126 | t.Fatal("apple dial tiger should not be ok when plant internal is down.") 127 | } 128 | 129 | Cli.UpdateNodeState("plant", NodeState{ExternalDown: true}) 130 | appleDialTiger, _ = Cli.DialState(appleHostName, tigerPort) 131 | if appleDialTiger.OK != false { 132 | t.Fatal("apple dial tiger should not be ok when plant external is down.") 133 | } 134 | 135 | Cli.UpdateNodeState("plant", NodeState{}) 136 | Cli.UpdateNodeState("animal", NodeState{ExternalDown: true}) 137 | appleDialTiger, _ = Cli.DialState(appleHostName, tigerPort) 138 | if appleDialTiger.OK != false { 139 | t.Fatal("apple dial tiger should not be ok when animal external is down.") 140 | } 141 | 142 | Cli.UpdateNodeState("animal", NodeState{InternalDown: true}) 143 | appleDialTiger, _ = Cli.DialState(appleHostName, tigerPort) 144 | if appleDialTiger.OK != false { 145 | t.Fatal("apple dial tiger should not be ok when animal internal is down.") 146 | } 147 | 148 | Cli.UpdateNodeState("animal", NodeState{}) 149 | Cli.UpdateNodeState("animal.land", NodeState{ExternalDown: true}) 150 | appleDialTiger, _ = Cli.DialState(appleHostName, tigerPort) 151 | if appleDialTiger.OK != false { 152 | t.Fatal("apple dial tiger should not be ok when animal.land external is down.") 153 | } 154 | 155 | Cli.UpdateNodeState("animal.land", NodeState{InternalDown: true}) 156 | appleDialTiger, _ = Cli.DialState(appleHostName, tigerPort) 157 | if appleDialTiger.OK != false { 158 | t.Fatal("apple dial tiger should not be ok when animal.land internal is down.") 159 | } 160 | 161 | Cli.UpdateNodeState("animal.land", NodeState{}) 162 | Cli.UpdateNodeState(tigerHostName, NodeState{ExternalDown: true}) 163 | appleDialTiger, _ = Cli.DialState(appleHostName, tigerPort) 164 | if appleDialTiger.OK != false { 165 | t.Fatal("apple dial tiger should not be ok when tiger external is down.") 166 | } 167 | 168 | Cli.UpdateNodeState(tigerHostName, NodeState{InternalDown: true}) 169 | appleDialTiger, _ = Cli.DialState(appleHostName, tigerPort) 170 | if appleDialTiger.OK != false { 171 | t.Fatal("apple dial tiger should not be ok when tiger internal is down.") 172 | } 173 | if appleDialTiger.Latency != 3*time.Minute { 174 | t.Fatal("apple dial tiger's latency should be 15 minutes.") 175 | } 176 | 177 | Cli.UpdateNodeState(tigerHostName, NodeState{}) 178 | appleDialTiger, _ = Cli.DialState(appleHostName, tigerPort) 179 | if appleDialTiger.OK != true { 180 | defaultServer.DumpNode("") 181 | t.Fatal("apple dial tiger should be ok.") 182 | } 183 | 184 | err = Cli.ServerStopped(tigerHostName, tigerPort) 185 | if err != nil { 186 | t.Fatal(err) 187 | } 188 | appleDialTiger, _ = Cli.DialState(appleHostName, tigerPort) 189 | expectedState = ConnState{ 190 | OK: false, 191 | Latency: 2 * (30 + 10 + 100 + 100 + 10 + 1) * time.Millisecond, 192 | } 193 | if appleDialTiger != expectedState { 194 | defaultServer.DumpNode("") 195 | t.Fatal("apple dial tiger latency should be ", expectedState, "actual", appleDialTiger) 196 | } 197 | } 198 | 199 | func TestLatency(t *testing.T) { 200 | err := resetDefaultServer() 201 | if err != nil { 202 | t.Fatal(err) 203 | } 204 | appleHostName := "plant.fruit.apple" 205 | applePort := "30003" 206 | appleListener, err := net.Listen("tcp", "localhost:"+applePort) 207 | if err != nil { 208 | t.Fatal(err) 209 | } 210 | 211 | mListener, err := NewListener(appleListener, appleHostName) 212 | if err != nil { 213 | t.Fatal(err) 214 | } 215 | go echoServe(mListener) 216 | 217 | tigerHostName := "animal.land.tiger" 218 | tigerDialFunc := NewDialFunc(tigerHostName, 0) 219 | tigerConn, err := tigerDialFunc("tcp", "localhost:30003") 220 | if err != nil { 221 | t.Fatal(err) 222 | } 223 | defer tigerConn.Close() 224 | tigerToAppleState, _ := Cli.ConnState(localPort(tigerConn), applePort, nil) 225 | buf := make([]byte, 4096) 226 | before := time.Now() 227 | tigerConn.Write(buf) 228 | tigerConn.Write(buf) 229 | _, err = tigerConn.Read(buf) 230 | if err != nil { 231 | t.Fatal(err) 232 | } 233 | 234 | tigerConn.Read(buf) 235 | latency := time.Now().Sub(before) 236 | if latency < tigerToAppleState.Latency*2 || latency > tigerToAppleState.Latency*3 { 237 | defaultServer.DumpNode("") 238 | t.Fatal("expected latency", tigerToAppleState.Latency*2, "actual", latency) 239 | } 240 | } 241 | 242 | func TestProxy(t *testing.T) { 243 | originAddr := "localhost:6546" 244 | originListener, err := net.Listen("tcp", originAddr) 245 | if err != nil { 246 | t.Fatal(err) 247 | } 248 | go echoServe(originListener) 249 | resetDefaultServer() 250 | 251 | proxyName := "matter.metal.gold" 252 | proxyPort := "6577" 253 | localName := "animal.air.eagle" 254 | err = Cli.StartProxy(localName, proxyName, proxyPort, originAddr) 255 | if err != nil { 256 | t.Fatal(err) 257 | } 258 | defer Cli.StopProxy(proxyPort) 259 | testConn, err := net.DialTimeout("tcp", "localhost:"+proxyPort, time.Second) 260 | if err != nil { 261 | t.Fatal(err) 262 | } 263 | before := time.Now() 264 | writeData := bytes.Repeat([]byte("abcdefghi"), 1000) 265 | testConn.Write(writeData) 266 | readData, err := ioutil.ReadAll(io.LimitReader(testConn, int64(len(writeData)))) 267 | if err != nil { 268 | t.Fatal(err) 269 | } 270 | duration := time.Now().Sub(before) 271 | t.Log(duration) 272 | if duration < time.Millisecond*444 || duration > time.Millisecond*555 { 273 | t.Error("wrong latency, expected", 444*time.Millisecond, "actual", duration) 274 | defaultServer.DumpNode("") 275 | } 276 | 277 | if !bytes.Equal(readData, writeData) { 278 | t.Error("read data should be equal to written data") 279 | } 280 | err = testConn.Close() 281 | if err != nil { 282 | t.Fatal(err) 283 | } 284 | 285 | // change proxy's client name. 286 | newLocalName := "matter.metal.gold" 287 | err = Cli.UpdateProxy(newLocalName, proxyPort) 288 | if err != nil { 289 | t.Fatal(err) 290 | } 291 | 292 | newConn, err := net.Dial("tcp", "localhost:"+proxyPort) 293 | if err != nil { 294 | t.Fatal(err) 295 | } 296 | defer newConn.Close() 297 | before = time.Now() 298 | newConn.Write(writeData) 299 | _, err = io.ReadFull(newConn, readData) 300 | if err != nil { 301 | t.Fatal(err) 302 | } 303 | duration = time.Now().Sub(before) 304 | t.Log(duration) 305 | if duration > time.Millisecond*10 { 306 | t.Error("wrong latency, expected less than", 10*time.Millisecond, "actual", duration) 307 | defaultServer.DumpNode("") 308 | } 309 | 310 | if !bytes.Equal(readData, writeData) { 311 | t.Error("read data should be equal to wrttien data") 312 | } 313 | } 314 | 315 | func echoServe(listener net.Listener) { 316 | for { 317 | conn, err := listener.Accept() 318 | if err != nil { 319 | return 320 | } 321 | go func() { 322 | n, err := io.Copy(conn, conn) 323 | if err != nil { 324 | log.Println(n, err) 325 | } 326 | conn.Close() 327 | }() 328 | } 329 | } 330 | -------------------------------------------------------------------------------- /conn.go: -------------------------------------------------------------------------------- 1 | package stadis 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "log" 7 | "net" 8 | "strings" 9 | "sync" 10 | "time" 11 | ) 12 | 13 | //Each packet is 4KB, so buffer size is 4KB*NumOfPackets which by default is 512KB, 14 | //Larger buffer size gets more throughput, consume more memory. 15 | var NumOfPackets = 128 16 | 17 | const packetSize = 4 * 1024 18 | 19 | type connection struct { 20 | conn net.Conn 21 | connState *ConnState 22 | readDeadline time.Time 23 | writeDeadline time.Time 24 | mutex sync.RWMutex 25 | writePacketCh chan *packet 26 | readPacketCh chan *packet 27 | readBuffer bytes.Buffer //handle the case when packet length is longer than read buf length 28 | readErr error 29 | writeErrCh chan error 30 | pool packetPool 31 | closeCh chan struct{} 32 | updateCh chan struct{} 33 | updateErrCh chan error 34 | oldState *ConnState 35 | clientPort string 36 | serverPort string 37 | } 38 | 39 | type packet struct { 40 | data []byte 41 | length int 42 | sentTime int64 43 | err error 44 | } 45 | 46 | type packetPool chan *packet 47 | 48 | func (pp packetPool) get() (pack *packet) { 49 | select { 50 | case pack = <-pp: 51 | default: 52 | pack = new(packet) 53 | pack.data = make([]byte, packetSize) 54 | } 55 | return 56 | } 57 | 58 | func (pp packetPool) put(p *packet) { 59 | select { 60 | case pp <- p: 61 | default: 62 | } 63 | } 64 | 65 | func (c *connection) updateLoop() { 66 | for { 67 | select { 68 | case <-c.closeCh: 69 | return 70 | default: 71 | oldState := c.getState() 72 | newState, err := Cli.ConnState(c.clientPort, c.serverPort, oldState) 73 | if err != nil { 74 | log.Println(err) 75 | c.updateErrCh <- err 76 | return 77 | } 78 | if newState == oldState { 79 | continue 80 | } 81 | c.mutex.Lock() 82 | close(c.updateCh) 83 | c.updateCh = make(chan struct{}) 84 | c.connState = newState 85 | c.mutex.Unlock() 86 | } 87 | 88 | } 89 | } 90 | 91 | func (c *connection) readLoop() { 92 | for { 93 | packet := c.pool.get() 94 | n, err := c.conn.Read(packet.data) 95 | packet.err = err 96 | packet.length = n 97 | packet.sentTime = time.Now().UnixNano() 98 | select { 99 | case <-c.closeCh: 100 | return 101 | case c.readPacketCh <- packet: 102 | } 103 | } 104 | } 105 | 106 | func (c *connection) readPacket(packet *packet, b []byte) (n int, err error) { 107 | for { 108 | now := time.Now().UnixNano() 109 | elapsed := now - packet.sentTime 110 | state := c.getState() 111 | remainedDelay := state.Latency - time.Duration(elapsed) 112 | select { 113 | case <-c.closeCh: 114 | err = errors.New("connection closed") 115 | return 116 | case <-time.After(remainedDelay): 117 | n = copy(b, packet.data[:packet.length]) 118 | if packet.length <= len(b) { 119 | err = packet.err 120 | return 121 | } else { 122 | c.mutex.Lock() 123 | c.readBuffer.Write(packet.data[packet.length:]) 124 | c.readErr = packet.err 125 | c.mutex.Unlock() 126 | } 127 | c.pool.put(packet) 128 | return 129 | case <-c.updateCh: 130 | } 131 | } 132 | } 133 | 134 | func (mc *connection) Read(b []byte) (n int, err error) { 135 | mc.mutex.Lock() 136 | n, _ = mc.readBuffer.Read(b) 137 | if n == 0 { 138 | err = mc.readErr 139 | mc.readErr = nil 140 | } 141 | mc.mutex.Unlock() 142 | if n > 0 { 143 | return 144 | } else if err != nil { 145 | return 146 | } 147 | var deadlineTimer <-chan time.Time 148 | mc.mutex.Lock() 149 | if !mc.readDeadline.IsZero() { 150 | deadlineTimer = time.After(mc.readDeadline.Sub(time.Now())) 151 | } 152 | mc.mutex.Unlock() 153 | select { 154 | case packet := <-mc.readPacketCh: 155 | n, err = mc.readPacket(packet, b) 156 | case err = <-mc.updateErrCh: 157 | log.Println(err) 158 | case <-mc.closeCh: 159 | err = errors.New("connection closed") 160 | case <-deadlineTimer: 161 | err = errors.New("read timeout") 162 | } 163 | return 164 | } 165 | 166 | func (c *connection) writeLoop() { 167 | for { 168 | select { 169 | case <-c.closeCh: 170 | return 171 | case packet := <-c.writePacketCh: 172 | c.writePacket(packet) 173 | } 174 | } 175 | } 176 | func (c *connection) writePacket(packet *packet) { 177 | for { 178 | now := time.Now().UnixNano() 179 | elapsed := now - packet.sentTime 180 | remainedLatency := c.connState.Latency - time.Duration(elapsed) 181 | select { 182 | case <-c.closeCh: 183 | return 184 | case <-c.updateCh: 185 | //latency has been updated by manager, it may be less than remained time to wait. 186 | //so we should recalculate it. 187 | continue 188 | case <-time.After(remainedLatency): 189 | } 190 | var err error 191 | if c.connState.OK { 192 | _, err = c.conn.Write(packet.data[:packet.length]) 193 | } else { 194 | err = errors.New("connection error") 195 | } 196 | c.pool.put(packet) 197 | if err != nil { 198 | select { 199 | case c.writeErrCh <- err: 200 | default: 201 | } 202 | } 203 | return 204 | } 205 | } 206 | 207 | func (mc *connection) Write(b []byte) (n int, err error) { 208 | for n < len(b) { 209 | now := time.Now() 210 | packet := mc.pool.get() 211 | length := copy(packet.data, b[n:]) 212 | packet.length = length 213 | packet.sentTime = now.UnixNano() 214 | var deadlineTimer <-chan time.Time 215 | mc.mutex.Lock() 216 | if !mc.writeDeadline.IsZero() { 217 | deadlineTimer = time.After(mc.writeDeadline.Sub(now)) 218 | } 219 | mc.mutex.Unlock() 220 | select { 221 | case err = <-mc.updateErrCh: 222 | log.Println(err) 223 | case err = <-mc.writeErrCh: 224 | case <-deadlineTimer: 225 | err = errors.New("write timeout") 226 | case mc.writePacketCh <- packet: 227 | n += length 228 | case <-mc.closeCh: 229 | err = errors.New("connection closed") 230 | } 231 | if err != nil { 232 | return 233 | } 234 | } 235 | return 236 | } 237 | 238 | func (mc *connection) Close() error { 239 | close(mc.closeCh) 240 | return mc.conn.Close() 241 | } 242 | 243 | func (mc *connection) LocalAddr() net.Addr { 244 | return mc.conn.LocalAddr() 245 | } 246 | func (mc *connection) RemoteAddr() net.Addr { 247 | return mc.conn.RemoteAddr() 248 | } 249 | func (mc *connection) SetDeadline(t time.Time) (err error) { 250 | mc.mutex.Lock() 251 | mc.writeDeadline = t 252 | mc.readDeadline = t 253 | err = mc.conn.SetDeadline(t) 254 | mc.mutex.Unlock() 255 | return nil 256 | } 257 | func (mc *connection) SetReadDeadline(t time.Time) (err error) { 258 | mc.mutex.Lock() 259 | mc.readDeadline = t 260 | err = mc.conn.SetReadDeadline(t) 261 | mc.mutex.Unlock() 262 | return nil 263 | } 264 | 265 | func (mc *connection) SetWriteDeadline(t time.Time) (err error) { 266 | mc.mutex.Lock() 267 | mc.writeDeadline = t 268 | err = mc.conn.SetWriteDeadline(t) 269 | mc.mutex.Unlock() 270 | return nil 271 | } 272 | 273 | func (mc *connection) getState() (state *ConnState) { 274 | mc.mutex.RLock() 275 | state = mc.connState 276 | mc.mutex.RUnlock() 277 | return 278 | } 279 | 280 | func newConnection(conn net.Conn, clientPort, serverPort string) (mConn *connection, err error) { 281 | mConn = new(connection) 282 | mConn.conn = conn 283 | mConn.clientPort = clientPort 284 | mConn.serverPort = serverPort 285 | mConn.pool = make(packetPool, 10) 286 | mConn.writeErrCh = make(chan error, 1) 287 | mConn.writePacketCh = make(chan *packet, NumOfPackets) 288 | mConn.readPacketCh = make(chan *packet, NumOfPackets) 289 | mConn.updateCh = make(chan struct{}) 290 | mConn.updateErrCh = make(chan error, 1) 291 | mConn.closeCh = make(chan struct{}) 292 | 293 | connState, err := Cli.ConnState(clientPort, serverPort, nil) 294 | if err != nil { 295 | log.Println(err) 296 | return 297 | } 298 | mConn.connState = connState 299 | go mConn.writeLoop() 300 | go mConn.readLoop() 301 | go mConn.updateLoop() 302 | return 303 | } 304 | 305 | type dialer struct { 306 | clientName string 307 | timeout time.Duration 308 | } 309 | 310 | func (d *dialer) dial(network, serverAddr string) (conn net.Conn, err error) { 311 | serverPort := serverAddr[strings.LastIndex(serverAddr, ":")+1:] 312 | state, err := Cli.DialState(d.clientName, serverPort) 313 | if err != nil { 314 | log.Println(err) 315 | return 316 | } 317 | select { 318 | case <-time.After(time.Duration(state.Latency)): 319 | if state.OK { 320 | var realConn net.Conn 321 | realConn, err = net.Dial(network, serverAddr) 322 | if err != nil { 323 | return 324 | } 325 | err = Cli.ClientConnected(d.clientName, localPort(realConn)) 326 | if err != nil { 327 | log.Println(err) 328 | return 329 | } 330 | conn, err = newConnection(realConn, localPort(realConn), remotePort(realConn)) 331 | if err != nil { 332 | log.Println(err) 333 | return 334 | } 335 | } else { 336 | err = errors.New("connection error") 337 | } 338 | case <-time.After(d.timeout): 339 | err = errors.New("dial timeout") 340 | } 341 | return 342 | } 343 | 344 | func NewDialFunc(clientName string, timeout time.Duration) func(network, addr string) (conn net.Conn, err error) { 345 | d := new(dialer) 346 | d.clientName = clientName 347 | if timeout == 0 { 348 | timeout = time.Minute * 3 349 | } 350 | d.timeout = timeout 351 | return d.dial 352 | } 353 | 354 | type listener struct { 355 | ol net.Listener 356 | name string 357 | } 358 | 359 | func (l *listener) Accept() (mConn net.Conn, err error) { 360 | return l.ol.Accept() 361 | } 362 | 363 | func (l *listener) Close() (err error) { 364 | l.ol.Close() 365 | err = Cli.ServerStopped(l.name, listenerPort(l.ol)) 366 | if err != nil { 367 | log.Println(err) 368 | return 369 | } 370 | return 371 | } 372 | 373 | func (l *listener) Addr() net.Addr { 374 | return l.ol.Addr() 375 | } 376 | 377 | func NewListener(ol net.Listener, name string) (l net.Listener, err error) { 378 | err = Cli.ServerStarted(name, listenerPort(ol)) 379 | if err != nil { 380 | log.Println(err) 381 | return 382 | } 383 | l = &listener{ol, name} 384 | return 385 | } 386 | 387 | func Listen(network, addr, name string) (l net.Listener, err error) { 388 | originListener, err := net.Listen(network, addr) 389 | if err != nil { 390 | log.Println(err) 391 | return 392 | } 393 | l, err = NewListener(originListener, name) 394 | return 395 | } 396 | 397 | func listenerPort(l net.Listener) string { 398 | addr := l.Addr().String() 399 | return addr[strings.LastIndex(addr, ":")+1:] 400 | } 401 | 402 | func localPort(conn net.Conn) (port string) { 403 | addr := conn.LocalAddr().String() 404 | port = addr[strings.LastIndex(addr, ":")+1:] 405 | return 406 | } 407 | 408 | func remotePort(conn net.Conn) (port string) { 409 | addr := conn.RemoteAddr().String() 410 | port = addr[strings.LastIndex(addr, ":")+1:] 411 | return 412 | } 413 | -------------------------------------------------------------------------------- /topology.go: -------------------------------------------------------------------------------- 1 | package stadis 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "log" 9 | "strconv" 10 | "strings" 11 | "sync" 12 | "time" 13 | ) 14 | 15 | const ( 16 | tcpTimeOut = 15 * time.Minute 17 | dialTimeOut = 3 * time.Minute 18 | serverPortType = true 19 | clientPortType = false 20 | ) 21 | 22 | type node interface { 23 | state() NodeState 24 | setState(state NodeState) 25 | String() string 26 | } 27 | 28 | type dataCenter struct { 29 | name string 30 | rackMap map[string]*rack 31 | topo *topology 32 | NodeState 33 | } 34 | 35 | func (dc *dataCenter) state() NodeState { 36 | return dc.NodeState 37 | } 38 | 39 | func (dc *dataCenter) setState(state NodeState) { 40 | if state.Latency == 0 { 41 | state.Latency = dc.NodeState.Latency 42 | } 43 | dc.NodeState = state 44 | } 45 | 46 | func (dc *dataCenter) String() string { 47 | var racks []*rack 48 | for _, rack := range dc.rackMap { 49 | racks = append(racks, rack) 50 | } 51 | return fmt.Sprintf("\nname:%v internalDown:%v externalDown:%v latency:%v racks:%v", 52 | dc.name, dc.InternalDown, dc.ExternalDown, dc.Latency, racks) 53 | } 54 | 55 | func newDc(confDC *DataCenter, topo *topology) (dc *dataCenter) { 56 | dc = new(dataCenter) 57 | dc.topo = topo 58 | dc.name = confDC.Name 59 | dc.rackMap = make(map[string]*rack) 60 | if confDC.NodeState != nil { 61 | dc.NodeState = *(confDC.NodeState) 62 | } 63 | for _, confRack := range confDC.Racks { 64 | if confRack.HostDefault == nil { 65 | confRack.HostDefault = confDC.HostDefault 66 | } 67 | if confRack.NodeState == nil { 68 | confRack.NodeState = confDC.RackDefault 69 | } 70 | rack := newRack(confRack, dc) 71 | dc.rackMap[rack.name] = rack 72 | } 73 | return 74 | } 75 | 76 | type rack struct { 77 | name string 78 | hostMap map[string]*host 79 | dataCenter *dataCenter 80 | NodeState 81 | } 82 | 83 | func (r *rack) state() NodeState { 84 | return r.NodeState 85 | } 86 | func (r *rack) setState(state NodeState) { 87 | if state.Latency == 0 { 88 | state.Latency = r.NodeState.Latency 89 | } 90 | r.NodeState = state 91 | } 92 | 93 | func (rack *rack) String() string { 94 | var hosts []*host 95 | for _, host := range rack.hostMap { 96 | hosts = append(hosts, host) 97 | } 98 | return fmt.Sprintf("\n\tname:%v internalDown:%v externalDown:%v latency:%v hosts:%v", 99 | rack.name, rack.InternalDown, rack.ExternalDown, rack.Latency, hosts) 100 | } 101 | 102 | func newRack(confRack *Rack, dc *dataCenter) (r *rack) { 103 | r = new(rack) 104 | r.dataCenter = dc 105 | r.name = confRack.Name 106 | r.hostMap = make(map[string]*host) 107 | if confRack.NodeState != nil { 108 | r.NodeState = *(confRack.NodeState) 109 | } 110 | for _, confHost := range confRack.Hosts { 111 | if confHost.NodeState == nil { 112 | confHost.NodeState = confRack.HostDefault 113 | } 114 | host := newHost(confHost, r) 115 | r.hostMap[host.name] = host 116 | } 117 | return 118 | } 119 | 120 | type host struct { 121 | name string 122 | portMap map[int]bool //value is true for server port, false for client port. 123 | rack *rack 124 | NodeState 125 | } 126 | 127 | func (host *host) state() NodeState { 128 | return host.NodeState 129 | } 130 | 131 | func (host *host) setState(state NodeState) { 132 | if state.Latency == 0 { 133 | state.Latency = host.NodeState.Latency 134 | } 135 | host.NodeState = state 136 | } 137 | 138 | func (host *host) String() string { 139 | var ports []int 140 | for port := range host.portMap { 141 | ports = append(ports, port) 142 | } 143 | return fmt.Sprintf("\n\t\tname:%v internalDown:%v externalDown:%v latency:%v ports:\n\t\t\t%v", 144 | host.name, host.InternalDown, host.ExternalDown, host.Latency, ports) 145 | } 146 | 147 | func newHost(confHost *Host, rack *rack) (h *host) { 148 | h = new(host) 149 | h.rack = rack 150 | h.name = confHost.Name 151 | h.portMap = make(map[int]bool) 152 | for _, port := range confHost.Ports { 153 | h.portMap[port] = serverPortType 154 | rack.dataCenter.topo.ports[port] = h 155 | } 156 | if confHost.NodeState != nil { 157 | h.NodeState = *(confHost.NodeState) 158 | } 159 | return 160 | } 161 | 162 | type topology struct { 163 | HostLatency time.Duration 164 | RackLatency time.Duration 165 | DcLatency time.Duration 166 | ports map[int]*host //ports to host map 167 | dataCenterMap map[string]*dataCenter 168 | mutex sync.RWMutex 169 | updateCh chan struct{} 170 | } 171 | 172 | func (topo *topology) String() (s string) { 173 | var dcs []*dataCenter 174 | for _, dc := range topo.dataCenterMap { 175 | dcs = append(dcs, dc) 176 | } 177 | s = fmt.Sprint(dcs) 178 | return 179 | } 180 | 181 | //When a server port is added, the topology need to close update channel, and make a new one. 182 | //So all the blocking request will get their new states. 183 | //It's not necessary when adding client port, because adding a client port won't affect any other connections. 184 | func (topo *topology) addServerPort(name string, port int) (err error) { 185 | topo.mutex.Lock() 186 | defer topo.mutex.Unlock() 187 | host, err := topo.lookupHost(name) 188 | if err != nil { 189 | log.Println(err) 190 | return err 191 | } 192 | if _, ok := host.portMap[port]; ok { 193 | err = errors.New("port has been taken already.") 194 | log.Println(err) 195 | return 196 | } 197 | host.portMap[port] = serverPortType 198 | topo.ports[port] = host 199 | 200 | close(topo.updateCh) 201 | topo.updateCh = make(chan struct{}) 202 | return nil 203 | } 204 | 205 | func (topo *topology) removeServerPort(name string, port int) (err error) { 206 | topo.mutex.Lock() 207 | defer topo.mutex.Unlock() 208 | host, err := topo.lookupHost(name) 209 | if err != nil { 210 | log.Println(err) 211 | return 212 | } 213 | portType, ok := host.portMap[port] 214 | if !ok { 215 | err = errors.New("port hasn't been registered") 216 | log.Println(err) 217 | return 218 | } 219 | if portType != serverPortType { 220 | err = errors.New("port isn't registered as server") 221 | log.Println(err) 222 | return 223 | } 224 | 225 | delete(host.portMap, port) 226 | close(topo.updateCh) 227 | topo.updateCh = make(chan struct{}) 228 | return nil 229 | } 230 | 231 | func (topo *topology) addClientPort(name string, port int) (err error) { 232 | topo.mutex.Lock() 233 | defer topo.mutex.Unlock() 234 | host, err := topo.lookupHost(name) 235 | if err != nil { 236 | log.Println(err) 237 | return 238 | } 239 | if _, ok := host.portMap[port]; ok { 240 | err = errors.New("port has been taken already.") 241 | log.Println(err) 242 | return 243 | } 244 | host.portMap[port] = clientPortType 245 | topo.ports[port] = host 246 | return 247 | } 248 | 249 | func (topo *topology) removeClientPort(port int) (err error) { 250 | topo.mutex.Lock() 251 | defer topo.mutex.Unlock() 252 | host := topo.ports[port] 253 | if host == nil { 254 | err = errors.New("unknown port") 255 | log.Println(err) 256 | return 257 | } 258 | portType, ok := host.portMap[port] 259 | if !ok { 260 | err = errors.New("port hasn't been registered") 261 | log.Println(err) 262 | return 263 | } 264 | if portType != clientPortType { 265 | err = errors.New("port isn't registered as client port") 266 | log.Println(err) 267 | return 268 | } 269 | delete(host.portMap, port) 270 | delete(topo.ports, port) 271 | return 272 | } 273 | 274 | func (topo *topology) setNodeState(nodeName string, newState NodeState) (err error) { 275 | topo.mutex.Lock() 276 | defer topo.mutex.Unlock() 277 | node, err := topo.lookup(nodeName) 278 | if err != nil { 279 | log.Println(err) 280 | return 281 | } 282 | node.setState(newState) 283 | close(topo.updateCh) 284 | topo.updateCh = make(chan struct{}) 285 | return 286 | } 287 | 288 | func (topo *topology) nodeState(nodeName string) (nodeState NodeState, err error) { 289 | topo.mutex.RLock() 290 | defer topo.mutex.RUnlock() 291 | 292 | node, err := topo.lookup(nodeName) 293 | if err != nil { 294 | log.Println(err) 295 | return 296 | } 297 | nodeState = node.state() 298 | return 299 | } 300 | 301 | func (topo *topology) dialState(clientName string, serverPort int) (connState ConnState, err error) { 302 | topo.mutex.RLock() 303 | defer topo.mutex.RUnlock() 304 | clientHost, err := topo.lookupHost(clientName) 305 | if err != nil { 306 | log.Println(err) 307 | return 308 | } 309 | 310 | serverHost := topo.ports[serverPort] 311 | if serverHost == nil { 312 | err = errors.New("host undefined for address " + strconv.Itoa(serverPort)) 313 | log.Println(err) 314 | return 315 | } 316 | 317 | networkOk, latency := computeNetworkState(clientHost, serverHost) 318 | 319 | if networkOk { 320 | _, connState.OK = serverHost.portMap[serverPort] 321 | connState.Latency = latency * 2 //dial has double latency. 322 | } else { 323 | connState.OK = false 324 | connState.Latency = dialTimeOut 325 | } 326 | return 327 | } 328 | 329 | 330 | //compute the state of the connection based on entire network state. 331 | func (topo *topology) connState(clientPort, serverPort int) (connState ConnState, err error) { 332 | topo.mutex.RLock() 333 | defer topo.mutex.RUnlock() 334 | clientHost := topo.ports[clientPort] 335 | if clientHost == nil { 336 | err = fmt.Errorf("connState:unknown client port %d", clientPort) 337 | return 338 | } 339 | serverHost := topo.ports[serverPort] 340 | if serverHost == nil { 341 | err = fmt.Errorf("connState:unknown server port %d", serverPort) 342 | return 343 | } 344 | 345 | _, ok := clientHost.portMap[clientPort] 346 | if !ok { 347 | return 348 | } 349 | 350 | networkOk, latency := computeNetworkState(clientHost, serverHost) 351 | 352 | if networkOk { 353 | _, connState.OK = serverHost.portMap[serverPort] 354 | connState.Latency = latency 355 | } else { 356 | connState.OK = false 357 | connState.Latency = tcpTimeOut 358 | } 359 | return 360 | } 361 | 362 | //compute network connection state between client and server host, no ports involved. 363 | func computeNetworkState(clientHost, serverHost *host) (ok bool, latency time.Duration) { 364 | clientSideDown := clientHost.InternalDown 365 | serverSideDown := serverHost.InternalDown 366 | if clientHost != serverHost { 367 | clientSideDown = clientSideDown || clientHost.ExternalDown || clientHost.rack.InternalDown 368 | serverSideDown = serverSideDown || serverHost.ExternalDown || serverHost.rack.InternalDown 369 | latency += clientHost.Latency + serverHost.Latency 370 | if clientHost.rack != serverHost.rack { 371 | clientSideDown = clientSideDown || clientHost.rack.ExternalDown || clientHost.rack.dataCenter.InternalDown 372 | serverSideDown = serverSideDown || serverHost.rack.ExternalDown || serverHost.rack.dataCenter.InternalDown 373 | latency += clientHost.rack.Latency + serverHost.rack.Latency 374 | if clientHost.rack.dataCenter != serverHost.rack.dataCenter { 375 | clientSideDown = clientSideDown || clientHost.rack.dataCenter.ExternalDown 376 | serverSideDown = serverSideDown || serverHost.rack.dataCenter.ExternalDown 377 | latency += clientHost.rack.dataCenter.Latency + serverHost.rack.dataCenter.Latency 378 | } 379 | } 380 | } 381 | down := clientSideDown || serverSideDown 382 | ok = !down 383 | return 384 | } 385 | 386 | func newTopology(configReader io.Reader) (topo *topology, err error) { 387 | decoder := json.NewDecoder(configReader) 388 | config := new(Config) 389 | err = decoder.Decode(config) 390 | if err != nil { 391 | log.Println(err) 392 | return 393 | } 394 | 395 | topo = new(topology) 396 | topo.ports = make(map[int]*host) 397 | topo.dataCenterMap = make(map[string]*dataCenter) 398 | topo.updateCh = make(chan struct{}) 399 | for _, confDC := range config.DataCenters { 400 | if confDC.RackDefault == nil { 401 | confDC.RackDefault = config.RackDefault 402 | } 403 | if confDC.HostDefault == nil { 404 | confDC.HostDefault = config.HostDefault 405 | } 406 | if confDC.NodeState == nil { 407 | confDC.NodeState = config.DcDefault 408 | } 409 | dc := newDc(confDC, topo) 410 | topo.dataCenterMap[dc.name] = dc 411 | } 412 | return 413 | } 414 | 415 | func (topo *topology) lookup(fullName string) (nod node, err error) { 416 | parts := strings.Split(fullName, ".") 417 | dc := topo.dataCenterMap[parts[0]] 418 | if dc == nil { 419 | err = errors.New("undefined dataCenter name," + parts[0]) 420 | log.Println(err) 421 | return 422 | } 423 | if len(parts) < 2 { 424 | nod = dc 425 | return 426 | } 427 | rack := dc.rackMap[parts[1]] 428 | if rack == nil { 429 | err = errors.New("undefined rack name," + parts[1]) 430 | log.Println(err) 431 | return 432 | } 433 | if len(parts) < 3 { 434 | nod = rack 435 | return 436 | } 437 | host := rack.hostMap[parts[2]] 438 | if host == nil { 439 | err = errors.New("undefined host name," + parts[2]) 440 | log.Println(err) 441 | return 442 | } 443 | nod = host 444 | return 445 | } 446 | 447 | func (topo *topology) lookupHost(hostName string) (ho *host, err error) { 448 | node, err := topo.lookup(hostName) 449 | if err != nil { 450 | log.Println(err) 451 | return 452 | } 453 | ho, ok := node.(*host) 454 | if !ok { 455 | err = errors.New("invalid host name") 456 | log.Println(err) 457 | } 458 | return 459 | } 460 | 461 | func (topo *topology) getUpdateChannel() (updateCh chan struct{}) { 462 | topo.mutex.RLock() 463 | updateCh = topo.updateCh 464 | topo.mutex.RUnlock() 465 | return 466 | } 467 | 468 | func parsePort(port string) (portNum int, passive bool, err error) { 469 | portNum, err = strconv.Atoi(port) 470 | if err != nil { 471 | log.Println(err) 472 | return 473 | } 474 | if portNum < 0 { 475 | portNum = -portNum 476 | passive = true 477 | } 478 | return 479 | } 480 | --------------------------------------------------------------------------------