├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── compose-logstash-elasticsearch.yml ├── config ├── con.go └── config.go ├── example-config.json ├── main.go ├── proxy ├── loadbalancer.go ├── proxy.go └── rule.go ├── router └── router.go └── services └── service.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | *~ 26 | gremlinproxy 27 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.1 2 | 3 | ADD example-config.json /opt/gremlinproxy/ 4 | ADD gremlinproxy /opt/gremlinproxy/ 5 | CMD ["/opt/gremlinproxy/gremlinproxy", "-c", "/opt/gremlinproxy/example-config.json"] 6 | 7 | # Expose control port. 8 | EXPOSE 9876 9 | 10 | ## IMPORTANT: expose all proxy ports that you want gremlinproxy to listen on for your application services (from the proxy block in config file) 11 | EXPOSE 7777 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: gremlinproxy gremlinexampleapp 2 | gremlinproxy: 3 | docker run --rm -v "$PWD":/usr/src/gremlinproxy -w /usr/src/gremlinproxy golang:alpine go build -v 4 | gremlinexampleapp: gremlinproxy 5 | docker build -t gremlinexampleapp . 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## A service proxy with failure injection API 2 | 3 | This is a reference implementation of a client-side service proxy. It is 4 | meant to be used with Gremlin, a systematic resiliency testing framework. 5 | Every microservice instance making outbound API calls needs to have an 6 | associated gremlin proxy. Typically, it runs in the same VM or container 7 | alongside the calling process, and communicates over the loopback interface 8 | with the caller. 9 | 10 | Remote services and their instances have to be statically configured in the 11 | configuration file. The service proxy acts as a HTTP/HTTPS request router 12 | to route requests that arrive at _localhost:port_ to the 13 | _remotehost:port_. It has built in support for load balancing requests 14 | across remote service instances in a round robin manner. There is no 15 | support for sticky-sessions nor client-side TLS. Note that while the proxy 16 | can connect to HTTPS endpoints, the caller must connect to the proxy at the 17 | localhost via HTTP only. See the [example-config.json](example-config.json) 18 | for an example of how to support HTTPS upstream endpoints, while connecting 19 | to the proxy via http://localhost:port. 20 | 21 | ### Failure injection 22 | 23 | Requests that carry a pre-defined HTTP header, are subjected to 24 | various forms of fault injection. Requests can be aborted (caller 25 | gets back a HTTP 404, HTTP 503, etc.), delayed, or rewritten. The 26 | proxy can be controlled remotely using a REST API. Rules for various 27 | fault injection actions can be installed through this API. The 28 | [Gremlin resliency testing framework] 29 | (https://github.com/ResilienceTesting/gremlinsdk) provides a Python-based 30 | control plane library, to write high-level recipes, that will be 31 | automatically broken down into low-level fault injection commands to 32 | be executed by the gremlin proxy. 33 | 34 | ### Usage 35 | 36 | #### Configuration 37 | 38 | The _services_ section of the config file describes a list remote 39 | services that need to be proxied. Each element in the list is a JSON 40 | dictionary object, describing a single service. 41 | 42 | The _proxy_ block under each service specifies the local port at which 43 | requests for the remote service will be received, the IP address to bind to 44 | (defaults to localhost), and the proxy protocol. The valid values are 45 | "http" or "tcp". While the proxy can work with HTTP/HTTPS and generic TCP 46 | endpoints, fault injection support for TCP endpoints is limited to 47 | aborting/delaying connections at the beginning of a TCP session. 48 | 49 | The _loadbalancer_ section configures the set of hosts that provide 50 | the remote service as well as the load balancing method (currently 51 | roundrobin and random load balancing modes are supported). When the proxy 52 | protocol is set to "http", you can specify hosts with or without a 53 | scheme prefix (i.e., http/https). When the scheme prefix is absent, 54 | "http" will be added to the host entry. For example, if a host entry 55 | is of the form _192.168.0.1:9080_, request URLs will be of the form 56 | _http://192.168.0.1:9080_. If you would like to proxy requests to 57 | HTTPS endpoints, host entries in the _loadbalancer_ section must be 58 | prefixed with "https://" (e.g., _https://myacc.cloudant.com_). 59 | 60 | The _router_ block configures the REST interface of the gremlin 61 | proxy. The port 9876 is the default port at which the service proxy 62 | exposes the REST API. The _gremlinheader_ parameter specifies the HTTP 63 | header that triggers the fault injection actions. Requests that do not 64 | contain this header are left untouched. The _name_ parameter indicates 65 | the name of the microservice for which this service proxy is being 66 | used. 67 | 68 | Fields _loglevel_, _logjson_, and _logstash_ configure the logging 69 | aspects of the service proxy. All logs from the service proxy can be 70 | directly sent to a logstash server, and then subsequently piped to 71 | Elasticsearch. The Gremlin framework's assertion engine can directly 72 | interface with Elasticsearch to execute assertions over the logs 73 | generated by the gremlinproxy. 74 | 75 | An example configuration file is provided in 76 | [example-config.json](example-config.json). It configures a proxy for 77 | a _Client_ microservice (as indicated by the _name_ parameter in the 78 | _router_ block). The proxy listens for requests to the _Server_ 79 | microservice at _0.0.0.0:7777_ and forwards them to either 80 | _54.175.222.246:80_ or _https://httpbin.org_. All requests from the 81 | _Client_ microservice, containing the HTTP header _X-Gremlin-ID_ will 82 | be subjected to fault injection. 83 | 84 | #### Building and running the proxy 85 | - Before you run the proxy, you need to run logstash server and elasticsearch. Run ``docker-compose -f compose-logstash-elasticsearch.yml up -d`` 86 | - Setup your go environment and GOPATH variable 87 | - Clone the repository to ``$GOPATH/go/src/github.com/gremlin`` folder. 88 | - Build: ``go get && go build`` 89 | - Run ``./gremlinproxy -c yourconfig.json`` 90 | 91 | ### Proxy REST API 92 | ```GET /gremlin/v1```: simple hello world test 93 | 94 | ```POST /gremlin/v1/rules/add```: add a Rule. Rule must be posted as a JSON. Format is as follows 95 | 96 | ```javascript 97 | { 98 | source: , 99 | dest: , 100 | messagetype: 101 | headerpattern: 102 | bodypattern: 103 | delayprobability: 104 | delaydistribution: probability distribution function 105 | 106 | mangleprobability: 107 | mangledistribution: probability distribution function 108 | 109 | abortprobability: 110 | abortdistribution: probability distribution function 111 | 112 | delaytime: latency to inject into requests 113 | errorcode: HTTP error code or -1 to reset TCP connection 114 | searchstring: string to replace when Mangle is enabled 115 | replacestring: string to replace with for Mangle fault 116 | } 117 | ``` 118 | 119 | ```POST /gremlin/v1/rules/remove``` : remove the rule specified in the message body (see rule format above) 120 | 121 | ```GET /gremlin/v1/rules/list```: list all installed rules 122 | 123 | ```DELETE /gremlin/v1/rules```: clear all rules 124 | 125 | ```GET /gremlin/v1/proxy/:service/instances```: get list of instances for for ```:service``` 126 | 127 | ```PUT /gremlin/v1/proxy/:service/:instances```: set list of instances for ```:service```. ```:instances``` is a comma separated list. 128 | 129 | ```DELETE /gremlin/v1/proxy/:service/instances```: clear list of instances under ```:service``` 130 | 131 | ```PUT /gremlin/v1/test/:id```: set new test ```:id```, that will be logged along with request/response logs 132 | 133 | ```DELETE /gremlin/v1/test/:id```: remove the currently set test ```:id``` 134 | -------------------------------------------------------------------------------- /compose-logstash-elasticsearch.yml: -------------------------------------------------------------------------------- 1 | es: 2 | image: elasticsearch:1.7 3 | command: elasticsearch -Des.index.analysis.analyzer.default.type=keyword 4 | ports: 5 | - "29200:9200" 6 | - "29300:9300" 7 | logstash: 8 | image: logstash 9 | command: logstash -e " input {udp {codec=>json port=>8092}} output {elasticsearch {hosts=>es index=>gremlin}} " 10 | ports: 11 | - "8092:8092/udp" 12 | links: 13 | - es 14 | 15 | -------------------------------------------------------------------------------- /config/con.go: -------------------------------------------------------------------------------- 1 | // Some constants and globally accessible vars that remain constats once configured 2 | package config 3 | 4 | import ( 5 | log "github.com/Sirupsen/logrus" 6 | "os" 7 | ) 8 | 9 | const OK = "OK" 10 | const ERROR = "ERROR" 11 | const NAME = "gremlinproxy" 12 | 13 | var TrackingHeader string 14 | var ProxyFor string 15 | 16 | var GlobalLogger = &log.Logger{ 17 | Out: os.Stderr, 18 | Formatter: new(log.TextFormatter), 19 | Hooks: make(log.LevelHooks), 20 | Level: log.WarnLevel, 21 | } 22 | var ProxyLogger = &log.Logger{ 23 | Out: os.Stderr, 24 | Formatter: new(log.JSONFormatter), 25 | Hooks: make(log.LevelHooks), 26 | Level: log.InfoLevel, 27 | } 28 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "encoding/json" 5 | "io/ioutil" 6 | ) 7 | 8 | // Config stores global routing configuration 9 | type Config struct { 10 | Services []ServiceConfig `json:"services"` 11 | Router RouterConfig `json:"router"` 12 | LogLevel string `json:"loglevel"` 13 | LogJSON bool `json:"logjson"` 14 | LogstashHost string `json:"logstash"` 15 | } 16 | 17 | // ServiceConfig stores configuration for a single remote service 18 | type ServiceConfig struct { 19 | Name string `json:"name"` 20 | Proxyconf ProxyConfig `json:"proxy"` 21 | LBConfig LoadBalancerConfig `json:"loadbalancer"` 22 | } 23 | 24 | // ProxyConfig stores the proxy options. Protocol refers to proxying mode: tcp or http 25 | type ProxyConfig struct { 26 | Port uint16 `json:"port"` 27 | BindHost string `json:"bindhost"` 28 | Protocol string `json:"protocol"` 29 | } 30 | 31 | // LoadBalancerConfig configures each loadbalancer. 32 | type LoadBalancerConfig struct { 33 | Hosts []string `json:"hosts"` 34 | BalanceMode string `json:"balancemode"` 35 | } 36 | 37 | // RouterConfig stores options specific to routers REST interface, and other 38 | // global config options, such as which headers we track in HTTP requests 39 | type RouterConfig struct { 40 | Port uint16 `json:"port"` 41 | TrackingHeader string `json:"trackingheader"` 42 | Name string `json:"name"` 43 | } 44 | 45 | // RuleConfig represents different rules 46 | type RuleConfig struct { 47 | Source string `json:"source"` 48 | Dest string `json:"dest"` 49 | MType string `json:"messagetype"` 50 | 51 | BodyPattern string `json:"bodypattern"` 52 | HeaderPattern string `json:"headerpattern"` 53 | // TestID string `json:"testid"` 54 | 55 | DelayProbability float64 `json:"delayprobability"` 56 | DelayDistribution string `json:"delaydistribution"` 57 | MangleProbability float64 `json:"mangleprobability"` 58 | MangleDistribution string `json:"mangledistribution"` 59 | AbortProbability float64 `json:"abortprobability"` 60 | AbortDistribution string `json:"abortdistribution"` 61 | 62 | DelayTime string `json:"delaytime"` 63 | // Method string `json:"method"` 64 | ErrorCode int `json:"errorcode"` 65 | SearchString string `json:"searchstring"` 66 | ReplaceString string `json:"replacestring"` 67 | } 68 | 69 | // ReadConfig reads a config from disk 70 | func ReadConfig(path string) Config { 71 | bytes, err := ioutil.ReadFile(path) 72 | if err != nil { 73 | panic(err.Error()) 74 | } 75 | var c Config 76 | err = json.Unmarshal(bytes, &c) 77 | if err != nil { 78 | panic(err.Error()) 79 | } 80 | return c 81 | } 82 | -------------------------------------------------------------------------------- /example-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "services": [ 3 | { 4 | "name": "Server", 5 | "proxy": { 6 | "bindhost" : "0.0.0.0", 7 | "port": 7777, 8 | "protocol": "http" 9 | }, 10 | "loadbalancer": { 11 | "hosts": [ 12 | "54.175.222.246", 13 | "https://httpbin.org" 14 | ], 15 | "mode": "roundrobin" 16 | } 17 | } 18 | ], 19 | "router": { 20 | "name": "Client", 21 | "port": 9876, 22 | "trackingheader": "X-Gremlin-ID" 23 | }, 24 | "loglevel": "debug", 25 | "logjson": true, 26 | "logstash": "" 27 | } 28 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "github.com/ResilienceTesting/gremlinproxy/config" 7 | "github.com/ResilienceTesting/gremlinproxy/router" 8 | "net" 9 | "os" 10 | 11 | "github.com/Sirupsen/logrus" 12 | ) 13 | 14 | func main() { 15 | // Read config 16 | cpath := flag.String("c", "", "Path to the config file") 17 | flag.Parse() 18 | if *cpath == "" { 19 | fmt.Println("No config file specified.\nusage: gremlinproxy -c configfile") 20 | os.Exit(1) 21 | } 22 | conf := config.ReadConfig(*cpath) 23 | fmt.Println("Config read successful") 24 | 25 | var log = config.GlobalLogger 26 | // Log as JSON instead of the default ASCII formatter. 27 | if conf.LogJSON { 28 | log.Formatter = new(logrus.JSONFormatter) 29 | } 30 | 31 | if conf.LogstashHost != "" { 32 | conn, err := net.Dial("udp", conf.LogstashHost) 33 | if err == nil { 34 | config.ProxyLogger.Out = conn 35 | } else { 36 | config.ProxyLogger.Out = os.Stderr 37 | config.ProxyLogger.Warn("Could not establish connection to logstash, logging to stderr") 38 | } 39 | } else { //else console 40 | config.ProxyLogger.Out = os.Stderr 41 | } 42 | // parse and set our log level 43 | if conf.LogLevel != "" { 44 | lvl, err := logrus.ParseLevel(conf.LogLevel) 45 | if err != nil { 46 | // default is info, if something went wrong 47 | log.Level = logrus.InfoLevel 48 | log.Error("Error parsing log level, defaulting to info") 49 | } else { 50 | log.Level = lvl 51 | } 52 | } else { 53 | log.Level = logrus.InfoLevel 54 | } 55 | config.TrackingHeader = conf.Router.TrackingHeader 56 | log.WithField("trackingHeader", config.TrackingHeader).Debug("Config value") 57 | if config.TrackingHeader == "" { 58 | panic("No trackingheader provided") 59 | } 60 | 61 | // Start the router 62 | r := router.NewRouter(conf) 63 | r.Run() //this blocks 64 | } 65 | -------------------------------------------------------------------------------- /proxy/loadbalancer.go: -------------------------------------------------------------------------------- 1 | package proxy 2 | 3 | import ( 4 | "github.com/ResilienceTesting/gremlinproxy/config" 5 | "github.com/Sirupsen/logrus" 6 | "math/rand" 7 | str "strings" 8 | "sync" 9 | ) 10 | 11 | //var globallog = config.GlobalLogger 12 | 13 | // LoadBalancer is in charge of switching up which host the request goes to 14 | type LoadBalancer struct { 15 | mode string 16 | hosts []string 17 | hostLock *sync.RWMutex 18 | index uint 19 | } 20 | 21 | // NewLoadBalancer creates a new load balancer 22 | func NewLoadBalancer(c config.LoadBalancerConfig) *LoadBalancer { 23 | var lb LoadBalancer 24 | if c.Hosts != nil && len(c.Hosts) > 0 { 25 | lb.hosts = make([]string, len(c.Hosts)) 26 | for i, server := range c.Hosts { 27 | lb.hosts[i] = server 28 | globallog.WithFields(logrus.Fields{ 29 | "host": server, 30 | "index": i, 31 | }).Debug("adding lb host") 32 | } 33 | } else { 34 | lb.hosts = make([]string, 10) 35 | } 36 | lb.mode = c.BalanceMode 37 | lb.hostLock = new(sync.RWMutex) 38 | return &lb 39 | } 40 | 41 | // GetHost returns a single host that the client should connect based on the loadbalance mode 42 | func (l *LoadBalancer) GetHost() string { 43 | l.hostLock.RLock() 44 | defer l.hostLock.RUnlock() 45 | var i uint 46 | switch str.ToLower(l.mode) { 47 | case "roundrobin": 48 | default: 49 | i = l.index % uint(len(l.hosts)) 50 | l.index++ 51 | break 52 | case "random": 53 | i = uint(rand.Int31n(int32(len(l.hosts)))) 54 | break 55 | } 56 | return l.hosts[i] 57 | } 58 | 59 | // GetInstances retrieves the available instances for the service 60 | func (l *LoadBalancer) GetInstances() []string { 61 | l.hostLock.Lock() 62 | defer l.hostLock.Unlock() 63 | retVal := make([]string, len(l.hosts)) 64 | copy(retVal, l.hosts) 65 | return retVal 66 | } 67 | 68 | // SetInstances updates internally stored available instances for the service 69 | func (l *LoadBalancer) SetInstances(hosts []string) { 70 | l.hostLock.Lock() 71 | defer l.hostLock.Unlock() 72 | l.hosts = hosts[:] 73 | } 74 | -------------------------------------------------------------------------------- /proxy/proxy.go: -------------------------------------------------------------------------------- 1 | package proxy 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "compress/zlib" 7 | "encoding/json" 8 | "github.com/ResilienceTesting/gremlinproxy/config" 9 | "io" 10 | "io/ioutil" 11 | "math/rand" 12 | "net" 13 | "net/http" 14 | "os" 15 | "regexp" 16 | "strconv" 17 | str "strings" 18 | "sync" 19 | "time" 20 | "fmt" 21 | "github.com/Sirupsen/logrus" 22 | ) 23 | 24 | var proxylog = config.ProxyLogger 25 | var globallog = config.GlobalLogger 26 | 27 | // Proxy implements the proxying logic between a pair of services. 28 | // A single router can have multiple proxies, one for each service that the local service needs to talk to 29 | type Proxy struct { 30 | name string 31 | testid string 32 | port uint16 33 | bindhost string 34 | Protocol string 35 | rules map[MessageType][]Rule 36 | ruleLock *sync.RWMutex 37 | /** 38 | expects map[string]chan int 39 | expectLock *sync.RWMutex 40 | **/ 41 | httpclient http.Client 42 | lb *LoadBalancer 43 | httpregexp *regexp.Regexp 44 | } 45 | 46 | // NewProxy returns a new proxy instance. 47 | func NewProxy(serviceName string, conf config.ProxyConfig, 48 | lbconf config.LoadBalancerConfig) *Proxy { 49 | var p Proxy 50 | p.name = serviceName 51 | if lbconf.Hosts == nil || len(lbconf.Hosts) < 1 { 52 | fmt.Println("Missing backend instances for service "+serviceName) 53 | os.Exit(1) 54 | } 55 | p.lb = NewLoadBalancer(lbconf) 56 | p.port = conf.Port 57 | p.httpclient = http.Client{} 58 | p.bindhost = conf.BindHost 59 | if (conf.BindHost == "") { 60 | p.bindhost = "localhost" 61 | } 62 | 63 | p.Protocol = conf.Protocol 64 | p.rules = map[MessageType][]Rule{Request: {}, Response: {}} 65 | p.ruleLock = new(sync.RWMutex) 66 | /** 67 | p.expects = map[string]chan int{} 68 | p.expectLock = new(sync.RWMutex) 69 | **/ 70 | p.httpregexp = regexp.MustCompile("^https?://") 71 | return &p 72 | } 73 | 74 | // getRule returns first rule matched to the given request. If no stored rules match, 75 | // a special NOPRule is returned. 76 | func (p *Proxy) getRule(r MessageType, reqID string, data []byte) Rule { 77 | p.ruleLock.RLock() 78 | defer p.ruleLock.RUnlock() 79 | // globallog.Debug("In getRule") 80 | for counter, rule := range p.rules[r] { 81 | globallog.WithField("ruleCounter", counter).Debug("Rule counter") 82 | // If request ID is empty, do not match unless wildcard rule 83 | if reqID == "" { 84 | if (rule.HeaderPattern == "*" || rule.BodyPattern == "*") { 85 | return rule 86 | } 87 | continue 88 | } 89 | 90 | // if requestID is a wildcard, pick up the first rule and return 91 | if reqID == "*" { 92 | return rule 93 | } 94 | 95 | if (rule.HeaderPattern == "*" && rule.BodyPattern == "*") { 96 | return rule 97 | } 98 | 99 | if rule.HeaderPattern != "*" { 100 | b, err := regexp.Match(rule.HeaderPattern, []byte(reqID)) 101 | if err != nil { 102 | globallog.WithFields(logrus.Fields{ 103 | "reqID": reqID, 104 | "errmsg": err.Error(), 105 | "headerpattern": rule.HeaderPattern, 106 | }).Error("Rule request ID matching error") 107 | continue 108 | } 109 | if !b { 110 | globallog.Debug("Id regex no match") 111 | continue 112 | } 113 | //globallog.WithField("ruleCounter", rule.ToConfig()).Debug("Id regex match") 114 | } 115 | 116 | if data == nil { 117 | // No match if body pattern is empty, but match if rule pattern is empty or this is a special pattern 118 | if rule.BodyPattern != "*" { 119 | continue 120 | } 121 | } else { 122 | if rule.BodyPattern != "*" { 123 | globallog.WithField("ruleCounter", counter).Debug("Body pattern !*") 124 | b, err := regexp.Match(rule.BodyPattern, data) 125 | if err != nil { 126 | globallog.WithFields(logrus.Fields{ 127 | "reqID": reqID, 128 | "errmsg": err.Error(), 129 | "bodypattern": rule.BodyPattern, 130 | }).Error("Rule body matching error") 131 | continue 132 | } 133 | if !b { 134 | globallog.Debug("Body regex no match") 135 | continue 136 | } 137 | } 138 | } 139 | //globallog.WithField("returning rule ", rule.ToConfig()).Debug("Id regex match") 140 | return rule 141 | } 142 | return NopRule 143 | } 144 | 145 | /** 146 | // expectCheck matches data against much any data the proxy should be seeing (i.e. expecting) on the wire 147 | func (p *Proxy) expectCheck(data []byte) { 148 | p.expectLock.RLock() 149 | defer p.expectLock.RUnlock() 150 | if len(p.expects) == 0 { 151 | return 152 | } 153 | for k, v := range p.expects { 154 | go func(k string, v chan int, data []byte) { 155 | b, err := regexp.Match(k, data) 156 | if err != nil { 157 | globallog.Error("Rule matching error") 158 | return 159 | } 160 | if b { 161 | v <- 1 162 | } 163 | }(k, v, data) 164 | } 165 | } 166 | **/ 167 | 168 | func glueHostAndPort(host string, port uint16) string { 169 | return host + ":" + strconv.Itoa(int(port)) 170 | } 171 | 172 | // Run starts up a proxy in the desired mode: tcp or http. This is a blocking call 173 | func (p *Proxy) Run() { 174 | globallog.WithFields(logrus.Fields{ 175 | "service": p.name, 176 | "bindhost" : p.bindhost, 177 | "port": p.port, 178 | "protocol": p.Protocol}).Info("Starting up proxy") 179 | switch str.ToLower(p.Protocol) { 180 | case "tcp": 181 | localhost, err := net.ResolveTCPAddr("tcp", glueHostAndPort(p.bindhost, p.port)) 182 | if err != nil { 183 | globallog.Error(err.Error()) 184 | break 185 | } 186 | listener, err := net.ListenTCP("tcp", localhost) 187 | if err != nil { 188 | globallog.Error(err.Error()) 189 | break 190 | } 191 | // Standard accept connection loop 192 | for { 193 | conn, err := listener.AcceptTCP() 194 | if err != nil { 195 | globallog.Error(err.Error()) 196 | continue 197 | } 198 | // go and handle the connection in separate thread 199 | go p.proxyTCP(conn) 200 | } 201 | break 202 | case "http": 203 | err := http.ListenAndServe(glueHostAndPort(p.bindhost, p.port), p) 204 | if err != nil { 205 | globallog.Error(err.Error()) 206 | } 207 | break 208 | default: 209 | panic(p.Protocol + " not supported") 210 | } 211 | } 212 | 213 | // tcpReadWrite handles low-level details of the proxying between two TCP connections 214 | // FIXME: update this to the new RULE format 215 | // func (p *Proxy) tcpReadWrite(src, dst *net.TCPConn, rtype MessageType, wg *sync.WaitGroup) { 216 | // // Copy the data from one connection to the other 217 | // data := make([]byte, 65536) //FIXME: This is bad. 218 | // defer wg.Done() 219 | // for { 220 | // n, err := src.Read(data) 221 | // if err != nil { 222 | // dst.Close() 223 | // return 224 | // } 225 | 226 | // var i int = 0 227 | // var n2 int = 0 228 | 229 | // for (n2 < n) { 230 | // i, err = dst.Write(data[n2:n]) 231 | // if err != nil { 232 | // src.Close() 233 | // return 234 | // } 235 | // //we have to write n-i more bytes 236 | // n2 = n2 + i 237 | // } 238 | // } 239 | // } 240 | 241 | func copyBytes(dest, src *net.TCPConn, wg *sync.WaitGroup) { 242 | defer wg.Done() 243 | io.Copy(dest, src) 244 | dest.CloseWrite() 245 | src.CloseRead() 246 | } 247 | 248 | //TODO: Need to add connection termination in the middle of a connection & bandwidth throttling. 249 | // Delay implementation is half-baked (only adds initial delay). 250 | 251 | // proxyTCP is responsible for handling a new TCP connection. 252 | func (p *Proxy) proxyTCP(conn *net.TCPConn) { 253 | 254 | //We can abort the connection immediately, in case of an Abort action. 255 | //FIXME: Need to have a way to abort in the middle of a connection too. 256 | rule := p.getRule(Request, "", nil) 257 | t := time.Now() 258 | 259 | //FIXME: Add proper delay support for TCP channels. 260 | if ((rule.DelayProbability > 0.0) && 261 | drawAndDecide(rule.DelayDistribution, rule.DelayProbability)) { 262 | proxylog.WithFields(logrus.Fields{ 263 | "dest": p.name, 264 | "source": config.ProxyFor, 265 | "protocol" : "tcp", 266 | "action" : "delay", 267 | "rule": rule.ToConfig(), 268 | "testid": p.getmyID(), 269 | "ts" : t.Format("2006-01-02T15:04:05.999999"), 270 | }).Info("Stream") 271 | time.Sleep(rule.DelayTime) 272 | } 273 | 274 | if ((rule.AbortProbability > 0.0) && 275 | drawAndDecide(rule.AbortDistribution, rule.AbortProbability)) { 276 | proxylog.WithFields(logrus.Fields{ 277 | "dest": p.name, 278 | "source": config.ProxyFor, 279 | "protocol" : "tcp", 280 | "action" : "abort", 281 | "rule": rule.ToConfig(), 282 | "testid": p.getmyID(), 283 | "ts" : t.Format("2006-01-02T15:04:05.999999"), 284 | }).Info("Stream") 285 | conn.SetLinger(0) 286 | conn.Close() 287 | return 288 | } 289 | 290 | remotehost := p.lb.GetHost() 291 | rAddr, err := net.ResolveTCPAddr("tcp", remotehost) 292 | if err != nil { 293 | globallog.Error("Could not resolve remote address: " + err.Error()) 294 | conn.Close() 295 | return 296 | } 297 | rConn, err := net.DialTCP("tcp", nil, rAddr) 298 | if err != nil { 299 | globallog.WithField("errmsg", err.Error()).Error("Could not connect to remote destination") 300 | conn.Close() 301 | return 302 | } 303 | // Make sure to copy data both directions, do it in separate threads 304 | var wg sync.WaitGroup 305 | wg.Add(2) 306 | // go p.tcpReadWrite(conn, rConn, Request, &wg) 307 | // go p.tcpReadWrite(rConn, conn, Response, &wg) 308 | //from proxier.go code in Kubernetes 309 | go copyBytes(conn, rConn, &wg) 310 | go copyBytes(rConn, conn, &wg) 311 | wg.Wait() 312 | conn.Close() 313 | rConn.Close() 314 | } 315 | 316 | //TODO: Need to add drip rule for HTTP (receiver taking in data byte by byte or sender sending data byte by byte, in low bandwidth situations). 317 | //TODO: In the request path, a slow receiver will cause buffer bloat at sender and ultimately lead to memory pressure -- VALIDATE 318 | //TODO: In the response path, emulating a slow response will keep caller connection alive but ultimately delay full req processing, sending HTTP header first, then byte by byte 319 | // -- VALIDATE if this is useful for common frameworks in languages like Java, Python, Node, Ruby, etc. 320 | ///If its not true, there is no need to emulate drip at all. 321 | // ServeHTTP: code that handles proxying of all HTTP requests 322 | 323 | /* FIXME: BUG This method reads requests/replies into memory. 324 | * DO NOT use this on very large size requests. 325 | */ 326 | func (p *Proxy) ServeHTTP(w http.ResponseWriter, req *http.Request) { 327 | reqID := req.Header.Get(config.TrackingHeader) 328 | var rule Rule 329 | var decodedData []byte 330 | var cont bool 331 | data, err := readBody(req.Body) 332 | if (reqID != "") { 333 | // Process the request, see if any rules match it. 334 | decodedData, err := decodeBody(data, req.Header.Get("content-type"), 335 | req.Header.Get("content-encoding")) 336 | if err != nil { 337 | globallog.WithFields(logrus.Fields{ 338 | "service": p.name, 339 | "reqID": reqID, 340 | "errmsg": err.Error()}).Error("Error reading HTTP request") 341 | rule = NopRule 342 | } else { 343 | // Check if we were expecting it on the wire: 344 | //p.expectCheck(decodedData) 345 | 346 | // Get the rule 347 | rule = p.getRule(Request, reqID, decodedData) 348 | } 349 | cont := p.executeRequestRule(reqID, rule, req, decodedData, w) 350 | if !cont { 351 | return 352 | } 353 | } 354 | 355 | var host = p.lb.GetHost() 356 | globallog.WithFields(logrus.Fields{ 357 | "service": p.name, 358 | "reqID": reqID, 359 | "host": host}).Debug("Sending to") 360 | 361 | // If scheme (http/https is not explicitly specified, construct a http request to the requested service 362 | if (!p.httpregexp.MatchString(host)) { 363 | host = "http://"+host 364 | } 365 | newreq, err := http.NewRequest(req.Method, host+req.RequestURI, bytes.NewReader(data)) 366 | if err != nil { 367 | status := http.StatusBadRequest 368 | http.Error(w, http.StatusText(status), status) 369 | globallog.WithFields(logrus.Fields{ 370 | "service": p.name, 371 | "reqID": reqID, 372 | "errmsg" : err.Error()}).Error("Could not construct proxy request") 373 | return 374 | } 375 | 376 | // Copy over the headers 377 | for k, v := range req.Header { 378 | if k != "Host" { 379 | for _, vv := range v { 380 | newreq.Header.Set(k, vv) 381 | } 382 | } else { 383 | newreq.Header.Set(k, host) 384 | } 385 | } 386 | 387 | // Make a connection 388 | starttime := time.Now() 389 | resp, err := p.httpclient.Do(newreq) 390 | respTime := time.Since(starttime) 391 | if err != nil { 392 | status := http.StatusInternalServerError 393 | http.Error(w, http.StatusText(status), status) 394 | globallog.WithFields( 395 | logrus.Fields{ 396 | "service": p.name, 397 | "duration": respTime.String(), 398 | "status": -1, 399 | "errmsg": err.Error(), 400 | }).Info("Request proxying failed") 401 | return 402 | } 403 | 404 | // Read the response and see if it matches any rules 405 | rule = NopRule 406 | data, err = readBody(resp.Body) 407 | resp.Body.Close() 408 | if (reqID != "") { 409 | decodedData, err = decodeBody(data, resp.Header.Get("content-type"), 410 | resp.Header.Get("content-encoding")) 411 | 412 | if err != nil { 413 | globallog.WithFields(logrus.Fields{ 414 | "service": p.name, 415 | "reqID": reqID, 416 | "errmsg": err.Error()}).Error("Error reading HTTP reply") 417 | rule = NopRule 418 | } else { 419 | // Check if we were expecting this 420 | //p.expectCheck(decodedData) 421 | 422 | // Execute rules, if any 423 | rule = p.getRule(Response, reqID, decodedData) 424 | } 425 | 426 | cont = p.executeResponseRule(reqID, rule, resp, decodedData, respTime, w) 427 | if !cont { 428 | return 429 | } 430 | } 431 | 432 | //return resp to caller 433 | for k, v := range resp.Header { 434 | for _, vv := range v { 435 | w.Header().Set(k, vv) 436 | } 437 | } 438 | w.WriteHeader(resp.StatusCode) 439 | _, err = w.Write(data) 440 | if err != nil { 441 | globallog.WithFields(logrus.Fields{ 442 | "service": p.name, 443 | "errmsg": err.Error()}).Error("HTTP Proxy write error") 444 | } 445 | } 446 | 447 | // Executes the rule on the request path or response path. ResponseWriter corresponds to the caller's connection 448 | // Returns a bool, indicating whether we should continue request processing further or not 449 | func (p *Proxy) doHTTPAborts(reqID string, rule Rule, w http.ResponseWriter) bool { 450 | 451 | if (rule.ErrorCode < 0) { 452 | hj, ok := w.(http.Hijacker) 453 | if !ok { 454 | // Revert to 500 455 | status := http.StatusInternalServerError 456 | http.Error(w, http.StatusText(status), status) 457 | globallog.WithFields(logrus.Fields{ 458 | "service": p.name, 459 | "reqID": reqID, 460 | "abortmethod" : "reset", 461 | "errmsg" : "Hijacking not supported", 462 | }).Error("Hijacking not supported") 463 | return false 464 | } 465 | 466 | conn, _, err := hj.Hijack() 467 | if err != nil { 468 | // Revert to 500 469 | status := http.StatusInternalServerError 470 | http.Error(w, http.StatusText(status), status) 471 | globallog.WithFields(logrus.Fields{ 472 | "service": p.name, 473 | "reqID": reqID, 474 | "abortmethod" : "reset", 475 | "errmsg" : err.Error(), 476 | }).Error("Hijacking Failed") 477 | return false 478 | } 479 | 480 | // Close the connection, discarding any unacked data 481 | tcpConn, ok := conn.(*net.TCPConn) 482 | if (ok) { 483 | tcpConn.SetLinger(0) 484 | tcpConn.Close() 485 | } else { 486 | //we couldn't type cast net.Conn to net.TCPConn successfully. 487 | //This shouldn't occur unless the underlying transport is not TCP. 488 | conn.Close() 489 | } 490 | } else { 491 | status := rule.ErrorCode 492 | http.Error(w, http.StatusText(status), status) 493 | } 494 | 495 | return true 496 | } 497 | 498 | 499 | // Fault injection happens here. 500 | // Log every request with valid reqID irrespective of fault injection 501 | func (p *Proxy) executeRequestRule(reqID string, rule Rule, req *http.Request, body []byte, w http.ResponseWriter) bool { 502 | 503 | var actions []string 504 | delay, errorCode, retVal := time.Duration(0), -2, true 505 | t := time.Now() 506 | 507 | if rule.Enabled { 508 | globallog.WithField("rule", rule.ToConfig()).Debug("execRequestRule") 509 | 510 | if ((rule.DelayProbability > 0.0) && 511 | drawAndDecide(rule.DelayDistribution, rule.DelayProbability)) { 512 | // In future, this could be dynamically computed -- variable delays 513 | delay = rule.DelayTime 514 | actions = append(actions, "delay") 515 | time.Sleep(rule.DelayTime) 516 | } 517 | 518 | if ((rule.AbortProbability > 0.0) && 519 | drawAndDecide(rule.AbortDistribution, rule.AbortProbability) && 520 | p.doHTTPAborts(reqID, rule, w)) { 521 | actions = append(actions, "abort") 522 | errorCode = rule.ErrorCode 523 | retVal = false 524 | } 525 | } 526 | 527 | proxylog.WithFields(logrus.Fields{ 528 | "dest": p.name, 529 | "source": config.ProxyFor, 530 | "protocol" : "http", 531 | "trackingheader": config.TrackingHeader, 532 | "reqID": reqID, 533 | "testid": p.getmyID(), 534 | "actions" : "["+str.Join(actions, ",")+"]", 535 | "delaytime": delay.Nanoseconds()/(1000*1000), //actual time req was delayed in milliseconds 536 | "errorcode": errorCode, //actual error injected or -2 537 | "uri": req.RequestURI, 538 | "ts" : t.Format("2006-01-02T15:04:05.999999"), 539 | "rule": rule.ToConfig(), 540 | }).Info("Request") 541 | 542 | return retVal 543 | } 544 | 545 | // Wrapper function around executeRule for the Response path 546 | //TODO: decide if we want to log body and header 547 | func (p *Proxy) executeResponseRule(reqID string, rule Rule, resp *http.Response, body []byte, after time.Duration, w http.ResponseWriter) bool { 548 | 549 | var actions []string 550 | delay, errorCode, retVal := time.Duration(0), -2, true 551 | t := time.Now() 552 | 553 | if rule.Enabled { 554 | if ((rule.DelayProbability > 0.0) && 555 | drawAndDecide(rule.DelayDistribution, rule.DelayProbability)) { 556 | // In future, this could be dynamically computed -- variable delays 557 | delay = rule.DelayTime 558 | actions = append(actions, "delay") 559 | time.Sleep(rule.DelayTime) 560 | } 561 | 562 | if ((rule.AbortProbability > 0.0) && 563 | drawAndDecide(rule.AbortDistribution, rule.AbortProbability) && 564 | p.doHTTPAborts(reqID, rule, w)) { 565 | actions = append(actions, "abort") 566 | errorCode = rule.ErrorCode 567 | retVal = false 568 | } 569 | } 570 | 571 | proxylog.WithFields(logrus.Fields{ 572 | "dest": p.name, 573 | "source": config.ProxyFor, 574 | "protocol" : "http", 575 | "trackingheader": config.TrackingHeader, 576 | "reqID": reqID, 577 | "testid": p.getmyID(), 578 | "actions" : "["+str.Join(actions, ",")+"]", 579 | "delaytime": delay.Nanoseconds()/(1000*1000), //actual time resp was delayed in milliseconds 580 | "errorcode": errorCode, //actual error injected or -2 581 | "status": resp.StatusCode, 582 | "duration": after.String(), 583 | "ts" : t.Format("2006-01-02T15:04:05.999999"), 584 | //log header/body? 585 | "rule": rule.ToConfig(), 586 | }).Info("Response") 587 | 588 | return retVal 589 | } 590 | 591 | // AddRule adds a new rule to the proxy. All requests/replies carrying the trackingheader will be checked 592 | // against all rules, if something matches, the first matched rule will be executed 593 | func (p *Proxy) AddRule(r Rule) { 594 | //TODO: check validity of regexes before installing a rule! 595 | p.ruleLock.Lock() 596 | p.rules[r.MType] = append(p.rules[r.MType], r) 597 | p.ruleLock.Unlock() 598 | } 599 | 600 | // RemoveRule removes a rule from this proxy 601 | func (p *Proxy) RemoveRule(r Rule) bool { 602 | p.ruleLock.RLock() 603 | n := len(p.rules[r.MType]) 604 | b := p.rules[r.MType][:0] 605 | for _, x := range p.rules[r.MType] { 606 | if x != r { 607 | b = append(b, x) 608 | } 609 | } 610 | p.ruleLock.RUnlock() 611 | p.ruleLock.Lock() 612 | p.rules[r.MType] = b 613 | p.ruleLock.Unlock() 614 | return len(p.rules[r.MType]) != n 615 | } 616 | 617 | // GetRules returns all rules currently active at this proxy 618 | func (p *Proxy) GetRules() []Rule { 619 | globallog.Debug("REST get rules") 620 | p.ruleLock.RLock() 621 | defer p.ruleLock.RUnlock() 622 | return append(p.rules[Request], p.rules[Response]...) 623 | } 624 | 625 | // GetInstances returns the service instances available in the loadbalancer for a given service 626 | func (p *Proxy) GetInstances() []string { 627 | return p.lb.GetInstances() 628 | } 629 | 630 | // SetInstances sets the service instances available in the loadbalancer for a given service 631 | func (p *Proxy) SetInstances(hosts []string) { 632 | p.lb.SetInstances(hosts) 633 | } 634 | 635 | // Reset clears proxy state. Removes all stored rules and expects. However loadbalancer hosts remain. 636 | func (p *Proxy) Reset() { 637 | // lock rules, clear, unlock 638 | p.ruleLock.Lock() 639 | p.rules = map[MessageType][]Rule{Request: {}, 640 | Response: {}} 641 | p.ruleLock.Unlock() 642 | /** 643 | // lock expects, clear, unlock 644 | p.expectLock.Lock() 645 | p.expects = map[string]chan int{} 646 | p.expectLock.Unlock() 647 | **/ 648 | } 649 | 650 | /** 651 | // Expect waits for a pattern to be encountered on the wire, up until timeout, if timeout > 0 652 | // Returns value < 0 if we timeout'd OR >0 if we saw the expected pattern on the wire 653 | func (p *Proxy) Expect(pattern string, timeout time.Duration) int { 654 | // Add a new expect to the list 655 | p.expectLock.Lock() 656 | c := make(chan int) 657 | p.expects[pattern] = c 658 | p.expectLock.Unlock() 659 | // If we have a timeout, set it up. 660 | if timeout > 0 { 661 | time.AfterFunc(timeout, func() { 662 | // after timeout write a value on the channel 663 | c <- -1 664 | }) 665 | } 666 | // Wait for the value, which means we've seen something or timeout'd. 667 | val := <-c 668 | // Remove the expect 669 | p.expectLock.Lock() 670 | delete(p.expects, pattern) 671 | p.expectLock.Unlock() 672 | return val 673 | } 674 | **/ 675 | 676 | func (p *Proxy) SetTestID(testID string) { 677 | // p.expectLock.Lock() 678 | // defer p.expectLock.Unlock() 679 | p.testid = testID 680 | t := time.Now() 681 | proxylog.WithFields(logrus.Fields{ 682 | "source": config.ProxyFor, 683 | "dest": p.name, 684 | "testid": testID, 685 | "ts" : t.Format("2006-01-02T15:04:05.999999"), 686 | }).Info("Test start") 687 | } 688 | 689 | func (p *Proxy) getmyID() string { 690 | // p.expectLock.RLock() 691 | // defer p.expectLock.RUnlock() 692 | return p.testid 693 | } 694 | 695 | func (p *Proxy) StopTest(testID string) bool { 696 | t := time.Now() 697 | // p.expectLock.Lock() 698 | // defer p.expectLock.Unlock() 699 | if testID == p.testid { 700 | p.testid = "" 701 | return true 702 | } 703 | proxylog.WithFields(logrus.Fields{ 704 | "source": config.ProxyFor, 705 | "dest": p.name, 706 | "ts" : t.Format("2006-01-02T15:04:05.999999"), 707 | "testid": testID, 708 | }).Info("Test stop") 709 | return false 710 | } 711 | 712 | // readBody is shortcut method to get all bytes from a reader 713 | func readBody(r io.Reader) ([]byte, error) { 714 | result, err := ioutil.ReadAll(r) 715 | return result, err 716 | } 717 | 718 | // Take the raw bytes from a request (or response) and run them through a decompression 719 | // algorithm so we can run the regex on it or log it. 720 | func decodeBody(raw []byte, ct string, ce string) ([]byte, error) { 721 | if str.Contains(ce, "gzip") { 722 | gr, err := gzip.NewReader(bytes.NewBuffer(raw)) 723 | if err != nil { 724 | return []byte{}, err 725 | } 726 | result, err := ioutil.ReadAll(gr) 727 | return result, err 728 | } else if str.Contains(ce, "deflate") { 729 | zr, err := zlib.NewReader(bytes.NewBuffer(raw)) 730 | if err != nil { 731 | return []byte{}, err 732 | } 733 | result, err := ioutil.ReadAll(zr) 734 | return result, err 735 | } 736 | return raw, nil 737 | } 738 | 739 | // Try to read in an arbitrary json object in the body of the request (or response) 740 | // so we can log it. 741 | func tryGetJSON(raw []byte) interface{} { 742 | var o interface{} 743 | err := json.Unmarshal(raw, &o) 744 | if err != nil { 745 | globallog.Warn("Could not get JSON from byte array") 746 | return raw 747 | } 748 | return o 749 | } 750 | 751 | // drawAndDecide draws from a given distribution and compares (<) the result to a threshold. 752 | // This determines whether an action should be taken or not 753 | func drawAndDecide(distribution ProbabilityDistribution, probability float64) bool { 754 | // fmt.Printf("In draw and decide with dis %s, thresh %f", DistributionString(distribution), probability); 755 | 756 | switch (distribution) { 757 | case ProbUniform: 758 | return rand.Float64() < probability 759 | case ProbExponential: 760 | return rand.ExpFloat64() < probability 761 | case ProbNormal: 762 | return rand.NormFloat64() < probability 763 | default: 764 | // globallog.Warn("Unknown probability distribution " + distribution + ", defaulting to coin flip") 765 | return rand.Float64() < .5 766 | } 767 | } 768 | -------------------------------------------------------------------------------- /proxy/rule.go: -------------------------------------------------------------------------------- 1 | package proxy 2 | 3 | import ( 4 | "errors" 5 | "github.com/ResilienceTesting/gremlinproxy/config" 6 | str "strings" 7 | "time" 8 | 9 | ) 10 | 11 | // MessageType is just that a type: request or reply 12 | type MessageType uint 13 | 14 | // ActionType actions we can do to a request or reply: abort, delay,... 15 | //type ActionType uint 16 | 17 | // ActionMethod with respect to an action. In case of abort it's hang, reset,... 18 | type ActionMethod uint 19 | 20 | // ProbabilityDistribution is a type for probability distribution functions for rules 21 | type ProbabilityDistribution uint 22 | 23 | /* 24 | const ( 25 | // ActionAbort abort 26 | ActionAbort ActionType = 1 << iota 27 | // ActionDelay delay 28 | ActionDelay 29 | // ActionAbortOrDelay inject a abort or delay (for overload scenario) 30 | ActionAbortOrDelay 31 | // ActionNop do nothing 32 | ActionNop 33 | ) 34 | */ 35 | 36 | // //customizations within abort or delay 37 | // const ( 38 | // // MethodStatus return some protocol specific error 39 | // MethodError = iota 40 | // // MethodReset TCP reset 41 | // MethodReset 42 | // //Emulates slow connections 43 | // //MethodDrip 44 | // ) 45 | 46 | const ( 47 | ProbUniform = iota 48 | ProbExponential 49 | ProbNormal 50 | ) 51 | 52 | /* 53 | var actionMap = map[ActionType]string{ 54 | ActionAbort: "abort", 55 | ActionDelay: "delay", 56 | ActionAbortOrDelay: "abort_or_delay", 57 | ActionNop: "nop", 58 | } 59 | */ 60 | 61 | // var methodMap = map[ActionMethod]string{ 62 | // MethodError: "errorcode", 63 | // MethodReset: "connreset", 64 | // //MethodDrip: "drip", 65 | // } 66 | 67 | var distributionMap = map[ProbabilityDistribution]string{ 68 | ProbUniform: "uniform", 69 | ProbExponential: "exponential", 70 | ProbNormal: "normal", 71 | } 72 | 73 | //message channel type between client and server, via the proxy 74 | const ( 75 | MTypeUnknown MessageType = iota 76 | Request 77 | Response 78 | Publish 79 | Subscribe 80 | ) 81 | 82 | var rMap = map[MessageType]string{ 83 | MTypeUnknown: "unknown", 84 | Request: "request", 85 | Response: "response", 86 | Publish: "publish", 87 | Subscribe: "subscribe", 88 | } 89 | 90 | // Rule is a universal type for all rules. 91 | type Rule struct { 92 | Source string 93 | Dest string 94 | MType MessageType 95 | //Method ActionMethod 96 | 97 | //Select only messages that match pattens specified in these fields 98 | BodyPattern string 99 | HeaderPattern string 100 | 101 | // Probability float64 102 | // Distribution string 103 | 104 | // First delay, then mangle and then abort 105 | // One could set the probabilities of these variables to 0/1 to toggle them on or off 106 | // We effectively get 8 combinations but only few make sense. 107 | DelayProbability float64 108 | DelayDistribution ProbabilityDistribution 109 | MangleProbability float64 110 | MangleDistribution ProbabilityDistribution 111 | AbortProbability float64 112 | AbortDistribution ProbabilityDistribution 113 | 114 | //TestID string 115 | DelayTime time.Duration 116 | ErrorCode int 117 | SearchString string 118 | ReplaceString string 119 | Enabled bool 120 | } 121 | 122 | // NopRule is a rule that does nothing. Useful default return value 123 | var NopRule = Rule{Enabled: false} 124 | 125 | func getDistribution(distribution string) (ProbabilityDistribution, error) { 126 | 127 | if distribution == "" { 128 | return ProbUniform, nil 129 | } 130 | 131 | switch str.ToLower(distribution) { 132 | case "uniform": 133 | return ProbUniform, nil 134 | case "exponential": 135 | return ProbExponential, nil 136 | case "normal": 137 | return ProbNormal, nil 138 | default: 139 | return ProbUniform, errors.New("Unknown probability distribution") 140 | } 141 | } 142 | 143 | // NewRule return a new rule based on the config. 144 | func NewRule(c config.RuleConfig) (Rule, error) { 145 | var r Rule 146 | var err error 147 | /* 148 | // Convert actions into masks so we can check them quickly 149 | for _, a := range c.Actions { 150 | switch str.ToLower(a) { 151 | case "abort": 152 | r.Action = r.Action | ActionAbort 153 | break 154 | case "delay": 155 | r.Action = r.Action | ActionDelay 156 | break 157 | case "abort_or_delay": 158 | r.Action = r.Action | ActionAbortOrDelay 159 | break 160 | default: 161 | return NopRule, errors.New("Unsupported action") 162 | } 163 | }*/ 164 | // Convert request/reply types 165 | switch str.ToLower(c.MType) { 166 | case "request": 167 | r.MType = Request 168 | case "response": 169 | r.MType = Response 170 | case "publish": 171 | r.MType = Publish 172 | case "subscribe": 173 | r.MType = Subscribe 174 | default: 175 | return NopRule, errors.New("Unsupported request type") 176 | } 177 | r.BodyPattern = c.BodyPattern 178 | r.HeaderPattern = c.HeaderPattern 179 | //sanity check 180 | //atleast header or body pattern must be non-empty 181 | if r.HeaderPattern == "" { 182 | return NopRule, errors.New("HeaderPattern cannot be empty (specify * instead)") 183 | } 184 | 185 | if r.BodyPattern == "" { 186 | r.BodyPattern = "*" 187 | } 188 | 189 | r.DelayDistribution, err = getDistribution(c.DelayDistribution) 190 | if (err != nil) { 191 | return NopRule, err 192 | } 193 | r.MangleDistribution, err = getDistribution(c.MangleDistribution) 194 | if (err != nil) { 195 | return NopRule, err 196 | } 197 | 198 | r.AbortDistribution, err = getDistribution(c.AbortDistribution) 199 | if (err != nil) { 200 | return NopRule, err 201 | } 202 | 203 | r.DelayProbability = c.DelayProbability 204 | r.MangleProbability = c.MangleProbability 205 | r.AbortProbability = c.AbortProbability 206 | valid := ((r.DelayProbability > 0.0) || (r.MangleProbability > 0.0) || (r.AbortProbability > 0.0)) 207 | if (!valid) { 208 | return NopRule, errors.New("Atleast one of delayprobability, mangleprobability, abortprobability must be non-zero and <=1.0") 209 | } 210 | 211 | // valid = ((r.DelayProbability >1.0) || (r.MangleProbability > 1.0) || (r.AbortProbability > 1.0)) 212 | // if (!valid) { 213 | // globallog.WithFields(logrus.Fields{"delay", r.DelayProbability, "abort": r.AbortProbability, "mangle": r.MangleProbability}).Warn("Probability cannot be >1.0") 214 | // return NopRule, errors.New("Probability cannot be >1.0") 215 | // } 216 | 217 | //r.TestID = c.TestID 218 | // r.DelayTime = time.Duration(c.DelayTime) * time.Millisecond 219 | if c.DelayTime != "" { 220 | var err error 221 | r.DelayTime, err = time.ParseDuration(c.DelayTime) 222 | if err != nil { 223 | globallog.WithField("errmsg", err.Error()).Warn("Could not parse rule delay time") 224 | return NopRule, err 225 | } 226 | } else { 227 | r.DelayTime = time.Duration(0) 228 | } 229 | 230 | // switch str.ToLower(c.Method) { 231 | // case "hang": 232 | // r.Method = MethodHang 233 | // case "reset": 234 | // r.Method = MethodReset 235 | // case "error": 236 | // r.Method = MethodError 237 | // default: 238 | // return NopRule, errors.New("Unsupported method") 239 | // } 240 | r.ErrorCode = c.ErrorCode 241 | r.SearchString = c.SearchString 242 | r.ReplaceString = c.ReplaceString 243 | r.Source = c.Source 244 | r.Dest = c.Dest 245 | r.Enabled = true 246 | return r, nil 247 | } 248 | 249 | // ToConfig converts the rule into a human-readable string config. 250 | func (r *Rule) ToConfig() config.RuleConfig { 251 | var c config.RuleConfig 252 | 253 | // Converting combo actions back into separate strings using some bit manipulations 254 | // c.Actions = []string{} 255 | // for a := range actionMap { 256 | // if a&r.Action > 0 { 257 | // c.Actions = append(c.Actions, actionMap[a]) 258 | // } 259 | // } 260 | c.Source = r.Source 261 | c.Dest = r.Dest 262 | c.MType = rMap[r.MType] 263 | 264 | c.HeaderPattern = r.HeaderPattern 265 | // c.TestID = r.TestID 266 | c.BodyPattern = r.BodyPattern 267 | 268 | 269 | c.DelayDistribution = distributionMap[r.DelayDistribution] 270 | c.MangleDistribution = distributionMap[r.MangleDistribution] 271 | c.AbortDistribution = distributionMap[r.AbortDistribution] 272 | 273 | c.DelayProbability = r.DelayProbability 274 | c.MangleProbability = r.MangleProbability 275 | c.AbortProbability = r.AbortProbability 276 | 277 | // c.Method = methodMap[r.Method] 278 | 279 | c.DelayTime = r.DelayTime.String() 280 | c.ErrorCode = r.ErrorCode 281 | c.SearchString = r.SearchString 282 | c.ReplaceString = r.ReplaceString 283 | 284 | return c 285 | } 286 | 287 | // ActionString returns the string that represents this action. 288 | // Warning: does not work on compound actions 289 | // func ActionString(a ActionType) string { 290 | // return actionMap[a] 291 | // } 292 | 293 | // ReqString returns the string represtation of MessageType: either "request" or "reply" 294 | func ReqString(r MessageType) string { 295 | return rMap[r] 296 | } 297 | 298 | // MethodString returns the string version of a abort method: hang, reset, httpstatus... 299 | // func MethodString(m ActionMethod) string { 300 | // return methodMap[m] 301 | // } 302 | 303 | func DistributionString(p ProbabilityDistribution) string { 304 | return distributionMap[p] 305 | } 306 | -------------------------------------------------------------------------------- /router/router.go: -------------------------------------------------------------------------------- 1 | package router 2 | 3 | import ( 4 | // "bufio" 5 | // "bytes" 6 | "encoding/json" 7 | "errors" 8 | "github.com/ResilienceTesting/gremlinproxy/config" 9 | "github.com/ResilienceTesting/gremlinproxy/proxy" 10 | "github.com/ResilienceTesting/gremlinproxy/services" 11 | // "io" 12 | "net/http" 13 | // "os" 14 | "strconv" 15 | str "strings" 16 | // "sync" 17 | "github.com/julienschmidt/httprouter" 18 | ) 19 | 20 | var logstashHost string 21 | var log = config.GlobalLogger 22 | 23 | // Router maintains all the state. It keeps a list or remote services we talk 24 | // to, and exposes a REST 25 | // API for configuring router rules 26 | type Router struct { 27 | services []*services.Service 28 | RESTPort uint16 29 | // map names of remote services to objects 30 | serviceNameMap map[string]*services.Service 31 | } 32 | 33 | // NewRouter creates a new router and configures unerlying services 34 | func NewRouter(conf config.Config) Router { 35 | var r Router 36 | r.services = make([]*services.Service, len(conf.Services)) 37 | r.serviceNameMap = make(map[string]*services.Service) 38 | for i, sconf := range conf.Services { 39 | s := services.NewService(sconf) 40 | r.services[i] = s 41 | r.serviceNameMap[s.Name] = s 42 | } 43 | r.RESTPort = conf.Router.Port 44 | logstashHost = conf.LogstashHost 45 | config.ProxyFor = conf.Router.Name 46 | return r 47 | } 48 | 49 | // Run starts up the control loop of the router. listening for any REST messages 50 | func (r *Router) Run() { 51 | for _, service := range r.services { 52 | go service.Proxy.Run() 53 | } 54 | log.Info("Router initialized") 55 | // blocking call here 56 | r.exposeREST() 57 | } 58 | 59 | func (r *Router) exposeREST() { 60 | // Expose a REST configuration interface 61 | hr := httprouter.New() 62 | hr.GET("/gremlin/v1", restHello) 63 | hr.POST("/gremlin/v1/rules/add", r.AddRule) 64 | hr.POST("/gremlin/v1/rules/remove", r.RemoveRule) 65 | hr.GET("/gremlin/v1/rules/list", r.ListRules) 66 | hr.DELETE("/gremlin/v1/rules", r.Reset) 67 | hr.GET("/gremlin/v1/proxy/:service/instances", r.GetInstances) 68 | hr.PUT("/gremlin/v1/proxy/:service/:instances", r.SetInstances) 69 | hr.DELETE("/gremlin/v1/proxy/:service/instances", r.RemoveInstances) 70 | hr.PUT("/gremlin/v1/test/:id", r.SetTest) 71 | hr.DELETE("/gremlin/v1/test/:id", r.RemoveTest) 72 | log.WithField("port", r.RESTPort).Debug("Running REST server") 73 | http.ListenAndServe(":"+strconv.Itoa(int(r.RESTPort)), hr) 74 | } 75 | 76 | // AddRule adds a rule to the list of active rules 77 | func (r *Router) AddRule(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { 78 | s, rule, err := r.readRule(req) 79 | if err != nil { 80 | w.WriteHeader(http.StatusBadRequest) 81 | w.Write([]byte(err.Error())) 82 | return 83 | } 84 | // Put the rule into the proxy that belongs to the correct service 85 | s.Proxy.AddRule(*rule) 86 | // Everything went well 87 | w.Write([]byte(config.OK)) 88 | log.Debug("Added rule") 89 | } 90 | 91 | // ListRules retuns a list of rules at all proxies in JSON format 92 | func (r *Router) ListRules(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { 93 | var readableRules []config.RuleConfig 94 | for _, s := range r.services { 95 | for _, rule := range s.Proxy.GetRules() { 96 | // Convert to human-readable 97 | c := rule.ToConfig() 98 | readableRules = append(readableRules, c) 99 | } 100 | } 101 | log.WithField("rules", readableRules).Debug("List rules:") 102 | // Write it out 103 | e := json.NewEncoder(w) 104 | if e.Encode(readableRules) != nil { 105 | log.Error("Error encoding router rules to JSON") 106 | w.WriteHeader(http.StatusInternalServerError) 107 | w.Write([]byte("Encoding rules problem")) 108 | } 109 | } 110 | 111 | // RemoveRule removes the rule from the list of active rules 112 | // FIXME: Currently this relies on internal Go equal semantics. Not ideal, but works for now 113 | func (r *Router) RemoveRule(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { 114 | s, rule, err := r.readRule(req) 115 | if err != nil { 116 | w.WriteHeader(http.StatusBadRequest) 117 | w.Write([]byte(err.Error())) 118 | return 119 | } 120 | 121 | res := s.Proxy.RemoveRule(*rule) 122 | // Everything went well 123 | w.Write([]byte(config.OK + "\n" + strconv.FormatBool(res))) 124 | log.Debug("Removed rule") 125 | } 126 | 127 | // Reset clears router state. This means all active proxies and their rules get cleared. 128 | func (r *Router) Reset(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { 129 | for _, s := range r.services { 130 | s.Proxy.Reset() 131 | } 132 | w.Write([]byte(config.OK)) 133 | } 134 | 135 | // restHello is just a demo REST API function 136 | func restHello(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { 137 | w.Write([]byte("Hello, I am " + config.NAME)) 138 | } 139 | 140 | // Overwrite the load balancer pool for a given service name 141 | func (r *Router) SetInstances(w http.ResponseWriter, req *http.Request, params httprouter.Params) { 142 | serviceName := params.ByName("service") 143 | hostlist := params.ByName("hosts") 144 | log.Debug("name="+serviceName+", hosts="+hostlist) 145 | s, exists := r.serviceNameMap[serviceName] 146 | if !exists { 147 | w.WriteHeader(http.StatusNotFound) 148 | w.Write([]byte("No such service " + serviceName)) 149 | return 150 | } 151 | if hostlist == "" { 152 | w.WriteHeader(http.StatusBadRequest) 153 | w.Write([]byte("Empty host list for " + serviceName)) 154 | return 155 | } 156 | hosts := str.Split(hostlist, ",") 157 | s.Proxy.SetInstances(hosts) 158 | w.Write([]byte(config.OK)) 159 | } 160 | 161 | func (r *Router) GetInstances(w http.ResponseWriter, req *http.Request, params httprouter.Params) { 162 | serviceName := params.ByName("service") 163 | s, exists := r.serviceNameMap[serviceName] 164 | if !exists { 165 | w.WriteHeader(http.StatusNotFound) 166 | w.Write([]byte("No such service " + serviceName)) 167 | return 168 | } 169 | 170 | hosts := s.Proxy.GetInstances() 171 | hostlist := str.Join(hosts, ",") 172 | w.Write([]byte(config.OK)) 173 | w.Write([]byte(hostlist)) 174 | } 175 | 176 | func (r *Router) RemoveInstances(w http.ResponseWriter, req *http.Request, params httprouter.Params) { 177 | serviceName := params.ByName("service") 178 | s, exists := r.serviceNameMap[serviceName] 179 | if !exists { 180 | w.WriteHeader(http.StatusNotFound) 181 | w.Write([]byte("No such service " + serviceName)) 182 | return 183 | } 184 | 185 | s.Proxy.SetInstances(make([]string,0)) 186 | w.Write([]byte(config.OK)) 187 | } 188 | 189 | /** 190 | // Expect is a special form of rule/request where we are waiting for a certain pattern on the wire. 191 | // This blocks until we see the pattern, or we timeout. HTTP keep-alives must be enabled. 192 | func (r *Router) Expect(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { 193 | s, rule, err := r.readRule(req) 194 | if err != nil { 195 | w.WriteHeader(http.StatusBadRequest) 196 | w.Write([]byte(err.Error())) 197 | return 198 | } 199 | 200 | val := s.Proxy.Expect((*rule).BodyPattern, (*rule).DelayTime) 201 | if val >= 0 { //all is good, we saw the pattern 202 | w.WriteHeader(http.StatusOK) 203 | } else { // we timed out 204 | w.WriteHeader(http.StatusRequestTimeout) 205 | } 206 | } 207 | **/ 208 | 209 | // readRule converts JSON rule POSTed to us to a Rule object 210 | func (r *Router) readRule(req *http.Request) (*services.Service, *proxy.Rule, error) { 211 | d := json.NewDecoder(req.Body) 212 | // defer req.Body.Close() 213 | 214 | // Try to automatically decode this from JSON into a config 215 | var ruleconf config.RuleConfig 216 | err := d.Decode(&ruleconf) 217 | if err != nil { 218 | log.Warning("Could not read JSON request\n" + err.Error()) 219 | return nil, nil, err 220 | } 221 | 222 | //check if source matches the router name 223 | if (ruleconf.Source != config.ProxyFor) { 224 | log.WithField("Source",ruleconf.Source).Warning("Rule not targeted for this Router") 225 | return nil, nil, errors.New("Router name does not match Source " + ruleconf.Source) 226 | } 227 | 228 | // Check if we have the desired service for which this rule applies. 229 | s, exists := r.serviceNameMap[ruleconf.Dest] 230 | if !exists { 231 | log.WithField("Dest", ruleconf.Dest).Warning("Service specified in rule not found") 232 | return nil, nil, errors.New("Dest service " + ruleconf.Dest + " not known") 233 | } 234 | 235 | //sanity checks in rule.go 236 | // Create a new rule 237 | rule, err := proxy.NewRule(ruleconf) 238 | if err != nil { 239 | log.WithField("errmsg", err.Error()).Info("Badly formed rule ignored") 240 | return nil, nil, err 241 | } 242 | return s, &rule, nil 243 | } 244 | 245 | // SetTest tells the router that a test with given ID will be happening 246 | func (r *Router) SetTest(w http.ResponseWriter, req *http.Request, 247 | params httprouter.Params) { 248 | testid := params.ByName("id") 249 | for _, s := range r.services { 250 | s.Proxy.SetTestID(testid) 251 | } 252 | w.Write([]byte(config.OK)) 253 | } 254 | 255 | func (r *Router) RemoveTest(w http.ResponseWriter, req *http.Request, 256 | params httprouter.Params) { 257 | testid := params.ByName("id") 258 | for _, s := range r.services { 259 | s.Proxy.StopTest(testid) 260 | } 261 | w.Write([]byte(config.OK)) 262 | } 263 | -------------------------------------------------------------------------------- /services/service.go: -------------------------------------------------------------------------------- 1 | package services 2 | 3 | import ( 4 | "github.com/ResilienceTesting/gremlinproxy/config" 5 | "github.com/ResilienceTesting/gremlinproxy/proxy" 6 | ) 7 | 8 | // Service is an encapsulation of a remote endpoint. It contains the proxy 9 | // instance doing the forwarding. 10 | // It could potentially be augmented to encapsulate functionality for discovering service 11 | // instances through zookeper (or potentially other services) 12 | type Service struct { 13 | Name string 14 | Proxy *proxy.Proxy 15 | } 16 | 17 | // NewService returns a new service given the config 18 | func NewService(conf config.ServiceConfig) *Service { 19 | var s = Service{ 20 | Name: conf.Name, 21 | Proxy: proxy.NewProxy(conf.Name, conf.Proxyconf, conf.LBConfig), 22 | } 23 | return &s 24 | } 25 | 26 | // GetInstances gets the service instances for a given service 27 | func (s *Service) GetInstances() []string { 28 | return s.Proxy.GetInstances() 29 | } 30 | 31 | // SetInstances sets the service instances for a given service 32 | func (s *Service) SetInstances(hosts []string) { 33 | s.Proxy.SetInstances(hosts) 34 | } 35 | --------------------------------------------------------------------------------