├── .gitignore ├── .travis.yml ├── glide.yaml ├── glide.lock ├── TODO.md ├── Makefile ├── LICENSE ├── report ├── doc.go ├── formatted_test.go └── formatted.go ├── CHANGELOG.md ├── source ├── info.go ├── tap │ ├── doc.go │ ├── tracer_test.go │ ├── emitter.go │ ├── tracer_example_test.go │ └── tracer.go ├── sources.go ├── item_data_source.go ├── source.go └── generic.go ├── internal ├── format_func.go ├── multi_error.go ├── marshaled │ ├── json.go │ ├── template.go │ ├── source_test.go │ ├── watcher.go │ └── source.go ├── protocol │ ├── chan_buf.go │ ├── item_buf.go │ ├── http_rest.go │ └── redis.go ├── resp │ ├── server.go │ ├── value.go │ ├── handler.go │ └── connection.go ├── test │ └── watcher.go └── meta │ ├── noun_data_source.go │ └── noun_data_source_test.go ├── config_test.go ├── doc.go ├── example_server ├── req_logger.go ├── res_logger.go └── main.go ├── data_source.go ├── README.md ├── serve.go ├── config.go └── example_access_log_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | vendor/ 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - 1.5 4 | - 1.6 5 | - tip 6 | env: 7 | global: 8 | - GO15VENDOREXPERIMENT=1 9 | install: make install_ci 10 | script: make test 11 | cache: 12 | directories: 13 | - vendor 14 | -------------------------------------------------------------------------------- /glide.yaml: -------------------------------------------------------------------------------- 1 | package: github.com/uber-go/gwr 2 | import: 3 | - package: github.com/uber-common/stacked 4 | version: ^1.0.2 5 | - package: github.com/stretchr/testify 6 | subpackages: 7 | - assert 8 | - package: github.com/uber/uber-licence 9 | - package: github.com/golang/lint 10 | - package: golang.org/x/tools 11 | subpackages: 12 | - go/gcimporter15 13 | -------------------------------------------------------------------------------- /glide.lock: -------------------------------------------------------------------------------- 1 | hash: bed765da2127646b76aafc551c153306d294e1569a93eded9a26639b1d74c739 2 | updated: 2016-05-20T13:43:53.815898766-07:00 3 | imports: 4 | - name: github.com/golang/lint 5 | version: c7bacac2b21ca01afa1dee0acf64df3ce047c28f 6 | - name: github.com/stretchr/testify 7 | version: 6cb3b85ef5a0efef77caef88363ec4d4b5c0976d 8 | subpackages: 9 | - assert 10 | - name: github.com/uber-common/stacked 11 | version: f68dcbe9559e6669e6f73c4dcdb8bfdbf13712bb 12 | - name: github.com/uber/uber-licence 13 | version: e35b7f99af2d110505bc81d8a9449dafaa326730 14 | - name: golang.org/x/tools 15 | version: 9ae4729fba20b3533d829a9c6ba8195b068f2abc 16 | subpackages: 17 | - go/gcimporter15 18 | devImports: [] 19 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | # For v1.0.0 2 | 3 | - pull a design document out into docs 4 | - resolve the config listen v port issue 5 | - implement /meta/entity 6 | - rework GenericDataFormat interface 7 | - allow preserving strings thru, rather than just []byte 8 | - consider splitting out the framing; either leave it always up to the wire 9 | protocol, or call it a "format default framing"? 10 | - add exported convenience types 11 | 12 | # For after v1.0.0 13 | 14 | - reporting support 15 | - redis pub/sub pattern may provide better experience than monitor 16 | - a collection agent based on watch and then later report 17 | 18 | # Anytime 19 | 20 | - more test coverage 21 | - benchmarks 22 | - more example integrations 23 | - metrics 24 | - runtime/trace 25 | - app logging 26 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PACKAGES=$(shell glide novendor) 2 | 3 | .PHONY: lint 4 | 5 | lint: 6 | go vet $(PACKAGES) 7 | 8 | .PHONY: test 9 | 10 | test: check-license lint 11 | find . -type f -name '*.go' -not -name '*_string.go' | xargs golint 12 | go test $(PACKAGES) 13 | 14 | vendor: glide.lock 15 | glide install 16 | 17 | .PHONY: install_ci 18 | install_ci: 19 | glide --version || go get -u -f github.com/Masterminds/glide 20 | make vendor 21 | glide install 22 | go install ./vendor/github.com/golang/lint/golint 23 | 24 | vendor/github.com/uber/uber-licence: vendor 25 | [ -d vendor/github.com/uber/uber-licence ] || glide install 26 | 27 | vendor/github.com/uber/uber-licence/node_modules: vendor/github.com/uber/uber-licence 28 | cd vendor/github.com/uber/uber-licence && npm install 29 | 30 | .PHONY: check-license add-license 31 | 32 | check-license: vendor/github.com/uber/uber-licence/node_modules 33 | ./vendor/github.com/uber/uber-licence/bin/licence --dry --file '*.go' 34 | 35 | add-license: vendor/github.com/uber/uber-licence/node_modules 36 | ./vendor/github.com/uber/uber-licence/bin/licence --verbose --file '*.go' 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Uber Technologies, Inc 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /report/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | /* 22 | Package report implements reporting support around watchable source.DataSource. 23 | */ 24 | package report 25 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | v0.7.1 2 | 3 | - Deadlock fix 4 | - Make `(*"internal/marshaled".DataSource).Active` less contentious 5 | 6 | v0.7.0 7 | - Many improvements to the tracer 8 | - Ability to walk up the scope chain with the scope.Root() and scope.Parent() 9 | methods. 10 | - Track begin and end time for every scope. 11 | - Added a standalone example fibonacci test 12 | - Added a default stringer text format for sources that define no text template. 13 | - Added a simple formatted reporter, which can be used to wire up a GWR source 14 | to a file or log stream (any Printf-like func). 15 | - Removed the extraneous "OK" response from the RESP monitor command; this 16 | makes it easier to process JSON output. 17 | - Added ability to drain a source; this allows a test, or other non-daemon 18 | program, to block until one or more GWR sources have finished sending any 19 | pending items to their watcher(s). 20 | - Added a convenience implementation of marshaled data source format around a 21 | single function. 22 | 23 | v0.6.5 24 | - Fixed a bug in tracer that was causing it to active even with no watchers. 25 | 26 | v0.6.4 27 | - Fix potential problems with marshaled data source state under concurrency 28 | - Fixed text output from /meta/nouns 29 | - Improve tracer text output 30 | - Other minor fixes to tests and documenetation 31 | 32 | v0.6.3 33 | - Fix potential concurrency problem for tap trace ids 34 | - Fix subtle problems around marshaled data source channels 35 | - Fix potential race condition 36 | 37 | v0.6.2 38 | - Initial intended release 39 | -------------------------------------------------------------------------------- /source/info.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package source 22 | 23 | // Info is a convenience info descriptor about a data source. 24 | type Info struct { 25 | Formats []string `json:"formats"` 26 | Attrs map[string]interface{} `json:"attrs"` 27 | } 28 | 29 | // GetInfo returns a structure that contains format and other information about 30 | // a given data source. 31 | func GetInfo(ds DataSource) Info { 32 | return Info{ 33 | Formats: ds.Formats(), 34 | Attrs: ds.Attrs(), 35 | } 36 | } 37 | 38 | // Info returns a map of info about all sources. 39 | func (dss *DataSources) Info() map[string]Info { 40 | info := make(map[string]Info, len(dss.sources)) 41 | for name, ds := range dss.sources { 42 | info[name] = GetInfo(ds) 43 | } 44 | return info 45 | } 46 | -------------------------------------------------------------------------------- /internal/format_func.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package internal 22 | 23 | // FormatFunc is a convenience type to define simple GenericDataFormats. 24 | type FormatFunc func(interface{}) ([]byte, error) 25 | 26 | // MarshalGet calls the wrapped function. 27 | func (ff FormatFunc) MarshalGet(item interface{}) ([]byte, error) { 28 | return ff(item) 29 | } 30 | 31 | // MarshalInit calls the wrapped function. 32 | func (ff FormatFunc) MarshalInit(item interface{}) ([]byte, error) { 33 | return ff(item) 34 | } 35 | 36 | // MarshalItem calls the wrapped function. 37 | func (ff FormatFunc) MarshalItem(item interface{}) ([]byte, error) { 38 | return ff(item) 39 | } 40 | 41 | // FrameItem simply adds a newline. 42 | func (ff FormatFunc) FrameItem(buf []byte) ([]byte, error) { 43 | n := len(buf) 44 | frame := make([]byte, n+1) 45 | copy(frame, buf) 46 | frame[n] = '\n' 47 | return frame, nil 48 | } 49 | -------------------------------------------------------------------------------- /report/formatted_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package report_test 22 | 23 | import ( 24 | "fmt" 25 | "testing" 26 | 27 | "github.com/stretchr/testify/assert" 28 | "github.com/uber-go/gwr" 29 | "github.com/uber-go/gwr/report" 30 | "github.com/uber-go/gwr/source" 31 | "github.com/uber-go/gwr/source/tap" 32 | ) 33 | 34 | var dummy = tap.AddEmitter("testDummy", nil) 35 | 36 | func TestLogfReporter(t *testing.T) { 37 | src := gwr.DefaultDataSources.Get("/tap/testDummy") 38 | 39 | var coll []string 40 | rep := report.NewLogfReporter(src, func(format string, args ...interface{}) { 41 | coll = append(coll, fmt.Sprintf(format, args...)) 42 | }) 43 | rep.Start() 44 | defer rep.Stop() 45 | 46 | dummy.Emit(42) 47 | dummy.Emit(struct{ Lol int }{99}) 48 | 49 | src.(source.DrainableSource).Drain() 50 | 51 | assert.Equal(t, []string{ 52 | "/tap/testDummy: 42", 53 | "/tap/testDummy: struct { Lol int }{Lol:99}", 54 | }, coll) 55 | } 56 | -------------------------------------------------------------------------------- /internal/multi_error.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package internal 22 | 23 | import ( 24 | "fmt" 25 | "strings" 26 | ) 27 | 28 | // MultiErr bundles more than one error together into a single error. 29 | type MultiErr []error 30 | 31 | // Error returns a string like "[E1, E2, ...]" where each Ex is the Error() of 32 | // each error in the slice. 33 | func (mienErrs MultiErr) Error() string { 34 | parts := make([]string, len(mienErrs)) 35 | for i, err := range mienErrs { 36 | parts[i] = err.Error() 37 | } 38 | return fmt.Sprintf("[%s]", strings.Join(parts, ", ")) 39 | } 40 | 41 | // AsError returns either: nil, the only error, or the MultiErr instance itself 42 | // if there are 0, 1, or more errors in the slice respectively. This method is 43 | // useful for contexts that want to do a simple return 44 | // MultiError(errs).AsError(). 45 | func (mienErrs MultiErr) AsError() error { 46 | switch len(mienErrs) { 47 | case 0: 48 | return nil 49 | case 1: 50 | return mienErrs[0] 51 | default: 52 | return mienErrs 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /internal/marshaled/json.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package marshaled 22 | 23 | import "encoding/json" 24 | 25 | // LDJSONMarshal is the usual Line-Delimited JSON 26 | var LDJSONMarshal = ldJSONMarshal(0) 27 | 28 | type ldJSONMarshal int 29 | 30 | // MarshalGet marhshals data through the standard json module. 31 | func (x ldJSONMarshal) MarshalGet(data interface{}) ([]byte, error) { 32 | return json.Marshal(data) 33 | } 34 | 35 | // MarshalInit marhshals data through the standard json module. 36 | func (x ldJSONMarshal) MarshalInit(data interface{}) ([]byte, error) { 37 | return json.Marshal(data) 38 | } 39 | 40 | // MarshalItem marhshals data through the standard json module. 41 | func (x ldJSONMarshal) MarshalItem(data interface{}) ([]byte, error) { 42 | return json.Marshal(data) 43 | } 44 | 45 | // FrameItem appends the newline record delimiter 46 | func (x ldJSONMarshal) FrameItem(json []byte) ([]byte, error) { 47 | n := len(json) 48 | frame := make([]byte, n+1) 49 | copy(frame, json) 50 | frame[n] = '\n' 51 | return frame, nil 52 | } 53 | -------------------------------------------------------------------------------- /source/tap/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | /* 22 | Package tap provides a simple item emitter source and a tracing source. 23 | 24 | TODO: break up this package: split emitter from tracer 25 | 26 | Emitter 27 | 28 | The Emitter source is useful for adding watchable-sources for existing 29 | data in your application, and where it wasn't worth it to define a 30 | specialized source. 31 | 32 | All emitter sources will be named like "/tap/...", to emphasize their generic 33 | nature. The normal use case here is for adding adhoc taps into existing 34 | program data. 35 | 36 | Tracer 37 | 38 | The Tracer source is useful for tracing program execution. It can be used 39 | to trace things like function calls, goroutine work units, and anything else 40 | where you can define a scope of work. 41 | 42 | All tracer sources will be named like "/tap/trace/...". A default tracer is 43 | provided at "/tap/trace", however the normal usage pattern is to declare 44 | one-or-more tracers within a package, and to name the appropriately to the area 45 | of the code that is traced. 46 | 47 | */ 48 | package tap 49 | -------------------------------------------------------------------------------- /config_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package gwr_test 22 | 23 | import ( 24 | "os" 25 | "testing" 26 | 27 | "github.com/uber-go/gwr" 28 | 29 | "github.com/stretchr/testify/assert" 30 | ) 31 | 32 | func TestConfiguredServer(t *testing.T) { 33 | os.Unsetenv("GWR_LISTEN") 34 | srv := gwr.NewConfiguredServer(gwr.Config{}) 35 | assert.Equal(t, srv.ListenAddr(), "", "no default listen address") 36 | 37 | os.Setenv("GWR_LISTEN", ":1234") 38 | srv = gwr.NewConfiguredServer(gwr.Config{}) 39 | assert.Equal(t, srv.ListenAddr(), ":1234", "env variable works") 40 | os.Unsetenv("GWR_LISTEN") 41 | 42 | srv = gwr.NewConfiguredServer(gwr.Config{ListenAddr: ":0"}) 43 | assert.Equal(t, srv.ListenAddr(), ":0", "server took ListenAddr config") 44 | 45 | assert.NoError(t, srv.Start(), "no start error") 46 | assert.Equal(t, srv.Start(), gwr.ErrAlreadyStarted, "double start error") 47 | assert.NotNil(t, srv.Addr(), "non-nil addr after start") 48 | assert.NoError(t, srv.Stop(), "no stop error") 49 | assert.Nil(t, srv.Addr(), "nil addr after stop") 50 | assert.NoError(t, srv.Stop(), "stop is idempotent") 51 | } 52 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | /* 22 | 23 | Package gwr provides on demand operational data sources in Go. A typical use 24 | is adding in-depth debug tracing to your program that only turns on when there 25 | are any active consumer(s). 26 | 27 | Basics 28 | 29 | A gwr data source is a named subset data that is Get-able and/or Watch-able. 30 | The gwr library can then build reporting on top of Watch-able sources, or by 31 | falling back to polling Get-able sources. 32 | 33 | For example a request log source would be naturally Watch-able for future 34 | requests as they come in. An implementation could go further and add a 35 | "last-10" buffer to also become Get-able. 36 | 37 | Integrating 38 | 39 | To bootstrap gwr and start its server listening on port 4040: 40 | 41 | gwr.Configure(&gwr.Config{ListenAddr: ":4040"}) 42 | 43 | 44 | GWR also adds a handler to the default http server; so if you already have a 45 | default http server like: 46 | 47 | log.Fatal(http.ListenAndServe(":8080", nil)) 48 | 49 | Then gwr will already be accessible at "/gwr/..." on port 8080; you should 50 | still call gwr.Configure: 51 | 52 | gwr.Configure(nil) 53 | 54 | */ 55 | package gwr 56 | -------------------------------------------------------------------------------- /example_server/req_logger.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package main 22 | 23 | import ( 24 | "net/http" 25 | "text/template" 26 | 27 | "github.com/uber-go/gwr/source" 28 | ) 29 | 30 | type reqLogger struct { 31 | handler http.Handler 32 | watcher source.GenericDataWatcher 33 | } 34 | 35 | func logged(handler http.Handler) *reqLogger { 36 | return &reqLogger{ 37 | handler: handler, 38 | } 39 | } 40 | 41 | var reqLogTextTemplate = template.Must(template.New("req_logger_text").Parse(` 42 | {{ define "item" }}{{ .Method }} {{ .Path }} {{ .Query }}{{ end }} 43 | `)) 44 | 45 | type reqInfo struct { 46 | Method string `json:"method"` 47 | Path string `json:"path"` 48 | Query string `json:"query"` 49 | } 50 | 51 | func (rl *reqLogger) ServeHTTP(w http.ResponseWriter, r *http.Request) { 52 | if watcher := rl.watcher; watcher.Active() { 53 | watcher.HandleItem(reqInfo{ 54 | Method: r.Method, 55 | Path: r.URL.Path, 56 | Query: r.URL.RawQuery, 57 | }) 58 | } 59 | rl.handler.ServeHTTP(w, r) 60 | } 61 | 62 | func (rl *reqLogger) Name() string { 63 | return "/request_log" 64 | } 65 | 66 | func (rl *reqLogger) TextTemplate() *template.Template { 67 | return reqLogTextTemplate 68 | } 69 | 70 | func (rl *reqLogger) SetWatcher(watcher source.GenericDataWatcher) { 71 | rl.watcher = watcher 72 | } 73 | -------------------------------------------------------------------------------- /data_source.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package gwr 22 | 23 | import ( 24 | "github.com/uber-go/gwr/internal/marshaled" 25 | "github.com/uber-go/gwr/internal/meta" 26 | "github.com/uber-go/gwr/source" 27 | ) 28 | 29 | // DefaultDataSources is default data sources registry which data sources are 30 | // added to by the module-level Add* functions. It is used by all of the 31 | // protocol servers if no data sources are provided. 32 | var DefaultDataSources *source.DataSources 33 | 34 | func init() { 35 | DefaultDataSources = source.NewDataSources() 36 | metaNouns := meta.NewNounDataSource(DefaultDataSources) 37 | DefaultDataSources.Add(marshaled.NewDataSource(metaNouns, nil)) 38 | DefaultDataSources.SetObserver(metaNouns) 39 | } 40 | 41 | // AddDataSource adds a data source to the default data sources registry. It 42 | // returns an error if there's already a data source defined with the same 43 | // name. 44 | func AddDataSource(ds source.DataSource) error { 45 | return DefaultDataSources.Add(ds) 46 | } 47 | 48 | // AddGenericDataSource adds a generic data source to the default data sources 49 | // registry. It returns an error if there's already a data source defined with 50 | // the same name. 51 | func AddGenericDataSource(gds source.GenericDataSource) error { 52 | mds := marshaled.NewDataSource(gds, nil) 53 | return DefaultDataSources.Add(mds) 54 | } 55 | -------------------------------------------------------------------------------- /internal/protocol/chan_buf.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package protocol 22 | 23 | import ( 24 | "bytes" 25 | "errors" 26 | "io" 27 | "sync" 28 | ) 29 | 30 | var errBufClosed = errors.New("buffer closed") 31 | 32 | type chanBuf struct { 33 | sync.Mutex 34 | bytes.Buffer 35 | ready chan<- *chanBuf 36 | closed bool 37 | pending bool 38 | p []byte 39 | // TODO: limit 40 | } 41 | 42 | func (cb *chanBuf) Reset() { 43 | cb.pending = false 44 | cb.Buffer.Reset() 45 | } 46 | 47 | func (cb *chanBuf) Write(p []byte) (int, error) { 48 | var send bool 49 | cb.Lock() 50 | 51 | if cb.closed { 52 | cb.Unlock() 53 | return 0, errBufClosed 54 | } 55 | 56 | n, err := cb.Buffer.Write(p) 57 | if n > 0 && !cb.pending { 58 | cb.pending = true 59 | send = true 60 | } 61 | cb.Unlock() 62 | 63 | if send { 64 | cb.ready <- cb 65 | } 66 | return n, err 67 | } 68 | 69 | func (cb *chanBuf) Close() error { 70 | if !cb.closed { 71 | cb.Lock() 72 | cb.closed = true 73 | cb.Unlock() 74 | } 75 | return nil 76 | } 77 | 78 | func (cb *chanBuf) writeTo(w io.Writer) (int, error) { 79 | return w.Write(cb.drain()) 80 | } 81 | 82 | func (cb *chanBuf) drain() []byte { 83 | cb.Lock() 84 | if cap(cb.p) < cb.Len() { 85 | cb.p = make([]byte, cb.Len()) 86 | } 87 | 88 | n := copy(cb.p[:cap(cb.p)], cb.Bytes()) 89 | cb.p = cb.p[:n] 90 | cb.Reset() 91 | cb.Unlock() 92 | return cb.p 93 | } 94 | -------------------------------------------------------------------------------- /internal/protocol/item_buf.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package protocol 22 | 23 | import ( 24 | "errors" 25 | "sync" 26 | ) 27 | 28 | var errItemBufClosed = errors.New("item buffer closed") 29 | 30 | type itemBuf struct { 31 | sync.Mutex 32 | ready chan<- *itemBuf 33 | closed bool 34 | pending bool 35 | buffer [][]byte 36 | takeBuf [][]byte 37 | // TODO: limit 38 | } 39 | 40 | func newItemBuf(ready chan<- *itemBuf) *itemBuf { 41 | return &itemBuf{ 42 | ready: ready, 43 | } 44 | } 45 | 46 | func (ib *itemBuf) put(items ...[]byte) (int, error) { 47 | if ib.closed { 48 | return 0, errItemBufClosed 49 | } 50 | ib.buffer = append(ib.buffer, items...) 51 | return len(items), nil 52 | } 53 | 54 | func (ib *itemBuf) HandleItem(item []byte) error { 55 | ib.Lock() 56 | n, err := ib.put(item) 57 | ib.Unlock() 58 | if n > 0 { 59 | ib.ready <- ib 60 | } 61 | return err 62 | } 63 | 64 | func (ib *itemBuf) HandleItems(items [][]byte) error { 65 | ib.Lock() 66 | n, err := ib.put(items...) 67 | ib.Unlock() 68 | if n > 0 { 69 | ib.ready <- ib 70 | } 71 | return err 72 | } 73 | 74 | func (ib *itemBuf) Close() error { 75 | if !ib.closed { 76 | ib.Lock() 77 | ib.closed = true 78 | ib.Unlock() 79 | } 80 | return nil 81 | } 82 | 83 | func (ib *itemBuf) drain() [][]byte { 84 | ib.Lock() 85 | ib.takeBuf = append(ib.takeBuf[:0], ib.buffer...) 86 | ib.buffer = ib.buffer[:0] 87 | ib.Unlock() 88 | return ib.takeBuf 89 | } 90 | -------------------------------------------------------------------------------- /example_server/res_logger.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package main 22 | 23 | import ( 24 | "net/http" 25 | "net/http/httptest" 26 | "text/template" 27 | 28 | "github.com/uber-go/gwr/source" 29 | ) 30 | 31 | type resLogger struct { 32 | handler http.Handler 33 | watcher source.GenericDataWatcher 34 | } 35 | 36 | type resInfo struct { 37 | Code int `json:"code"` 38 | Bytes int `json:"bytes"` 39 | ContentType string `json:"content_type"` 40 | } 41 | 42 | var resLogTextTemplate = template.Must(template.New("res_logger_text").Parse(` 43 | {{ define "item" }}{{ .Code }} {{ .Bytes }} {{ .ContentType }}{{ end }} 44 | `)) 45 | 46 | func (rl *resLogger) ServeHTTP(w http.ResponseWriter, r *http.Request) { 47 | rl.watcher.Active() 48 | 49 | rec := httptest.NewRecorder() 50 | rl.handler.ServeHTTP(rec, r) 51 | 52 | rl.watcher.HandleItem(resInfo{ 53 | Code: rec.Code, 54 | Bytes: rec.Body.Len(), 55 | ContentType: rec.HeaderMap.Get("Content-Type"), 56 | }) 57 | 58 | hdr := w.Header() 59 | for key, vals := range rec.HeaderMap { 60 | hdr[key] = vals 61 | } 62 | w.WriteHeader(rec.Code) 63 | rec.Body.WriteTo(w) 64 | } 65 | 66 | func (rl *resLogger) Name() string { 67 | return "/response_log" 68 | } 69 | 70 | func (rl *resLogger) TextTemplate() *template.Template { 71 | return resLogTextTemplate 72 | } 73 | 74 | func (rl *resLogger) SetWatcher(watcher source.GenericDataWatcher) { 75 | rl.watcher = watcher 76 | } 77 | -------------------------------------------------------------------------------- /internal/resp/server.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package resp 22 | 23 | import ( 24 | "fmt" 25 | "net" 26 | ) 27 | 28 | // RedisServer serves a RedisHandler on a listening socket. 29 | type RedisServer struct { 30 | consumer RedisHandler 31 | } 32 | 33 | // NewRedisServer creates a new RedisServer. 34 | func NewRedisServer(consumer RedisHandler) *RedisServer { 35 | return &RedisServer{ 36 | consumer: consumer, 37 | } 38 | } 39 | 40 | // ListenAndServe listens on the given hostPort, and then serves on that 41 | // listener. 42 | func (h RedisServer) ListenAndServe(hostPort string) error { 43 | ln, err := net.Listen("tcp", hostPort) 44 | if err != nil { 45 | return err 46 | } 47 | return h.Serve(ln) 48 | } 49 | 50 | // Serve serves the given listener. 51 | func (h RedisServer) Serve(ln net.Listener) error { 52 | for { 53 | conn, err := ln.Accept() 54 | if err != nil { 55 | // TODO: deal better with accept errors 56 | fmt.Printf("ERROR: accept error: %v", err) 57 | continue 58 | } 59 | go NewRedisConnection(conn, nil).Handle(h.consumer) 60 | } 61 | } 62 | 63 | // IsFirstByteRespTag returns true if the first byte in the passed slice is a 64 | // valid RESP tag character (i.e. if it is one of "-:+$*"). 65 | func IsFirstByteRespTag(p []byte) bool { 66 | switch p[0] { 67 | case '-': 68 | fallthrough 69 | case ':': 70 | fallthrough 71 | case '+': 72 | fallthrough 73 | case '$': 74 | fallthrough 75 | case '*': 76 | return true 77 | default: 78 | return false 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GWR: Get / Watch / Report -ing of operational data 2 | 3 | GWR provides on demand access to operational data: 4 | - define your data sources 5 | - poll and/or watch them over HTTP or Redis Protocol 6 | 7 | ## Status: beta 8 | 9 | GWR is currently in beta devolopment: 10 | - basic support for get and watch are done, with a couple simple sources; these 11 | interfaces are not likely to change before 1.0 12 | - reporting is not yet started, and is the major blocker before 1.0 13 | 14 | # Using 15 | 16 | GWR exposes a dual HTTP and RESP (Redis Protocol) interface. Integrators may 17 | specify the port, the example below uses 4040. 18 | 19 | The following examples are against a running instance of `example_server/`. 20 | 21 | ## HTTP 22 | 23 | Example for http: 24 | 25 | ``` 26 | $ curl localhost:4040/meta/nouns 27 | - /meta/nouns formats: 28 | - /request_log formats: 29 | - /response_log formats: 30 | 31 | $ curl -X WATCH localhost:4040/request_log& 32 | $ curl -X WATCH localhost:4040/response_log& 33 | 34 | $ curl localhost:8080/foo 35 | 404 page not found # this is the normal curl output 36 | GET /foo # this comes from the first watch-curl 37 | 404 19 text/plain; charset=utf-8 # this comes from the first watch-curl 38 | ``` 39 | 40 | ## Resp 41 | 42 | ``` 43 | $ redis-cli -p 4040 ls # this is a convenience alias for "get /meta/nouns" 44 | 1) - /meta/nouns formats: 45 | 2) - /request_log formats: 46 | 3) - /response_log formats: 47 | 48 | $ redis-cli -p 4040 monitor /request_log text /response_log text& 49 | OK 50 | 51 | $ curl localhost:8080/bar 52 | 404 page not found # this is the curl output 53 | /request_log> GET /bar # this is from redis-cli 54 | /response_log> 404 19 text/plain; charset=utf-8 # so is this, ordering not guaranteed 55 | ``` 56 | 57 | # Integration 58 | 59 | To add gwr to a program, all you need to do is call: 60 | 61 | ``` 62 | gwrProto.ListenAndServe(":4040", nil) 63 | ``` 64 | 65 | This hosts dual protocol HTTP and RESP server on port 4040. 66 | 67 | # Defining data sources 68 | 69 | To define a data source, the easiest way is to implement the 70 | `gwr.GenericDataSource` interface. 71 | 72 | `TODO: example` 73 | 74 | For now see `example_server/req_logger.go` and `example_server/res_logger.go` 75 | 76 | # Running the example server 77 | 78 | Should work by: 79 | ``` 80 | $ go run example_server/*.go 81 | ``` 82 | 83 | The example server hosts a dummy 404-ing web server on port `8080` and exposes 84 | a request and response log GWR noun. The HTTP and Resp usage examples above 85 | are against it. 86 | -------------------------------------------------------------------------------- /source/tap/tracer_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package tap_test 22 | 23 | import ( 24 | "fmt" 25 | "strings" 26 | "testing" 27 | 28 | "github.com/stretchr/testify/assert" 29 | "github.com/stretchr/testify/require" 30 | 31 | "github.com/uber-go/gwr/internal/test" 32 | "github.com/uber-go/gwr/source/tap" 33 | ) 34 | 35 | func TestTracer_collatz(t *testing.T) { 36 | tap.ResetTraceID() 37 | tracer := tap.NewTracer("test") 38 | wat := test.NewWatcher() 39 | tracer.SetWatcher(wat) 40 | sc := tracer.Scope("collatzTest").Open() 41 | n := collatz(5, sc) 42 | sc.Close(n) 43 | require.Equal(t, 1, n) 44 | assert.Equal(t, recodeTimeField(wat.AllStrings()), []string{ 45 | "--> t0 [1::1] collatzTest: ", 46 | "--> t1 [1:1:2] collatz: 5", 47 | "<-- t2 [1:1:2] 16", 48 | "--> t3 [1:2:3] collatz: 16", 49 | "<-- t4 [1:2:3] 8", 50 | "--> t5 [1:3:4] collatz: 8", 51 | "<-- t6 [1:3:4] 4", 52 | "--> t7 [1:4:5] collatz: 4", 53 | "<-- t8 [1:4:5] 2", 54 | "--> t9 [1:5:6] collatz: 2", 55 | "<-- t10 [1:5:6] 1", 56 | "<-- t11 [1::1] 1", 57 | }) 58 | } 59 | 60 | func recodeTimeField(strs []string) []string { 61 | for n, str := range strs { 62 | var head string 63 | rest := str 64 | for k := 0; k < 5; k++ { 65 | i := strings.IndexByte(rest, ' ') 66 | if i < 0 { 67 | continue 68 | } 69 | if len(head) == 0 { 70 | head = rest[:i] 71 | } 72 | rest = rest[i+1:] 73 | } 74 | strs[n] = strings.Join([]string{head, rest}, fmt.Sprintf(" t%d ", n)) 75 | } 76 | return strs 77 | } 78 | 79 | func collatz(n int, sc *tap.TraceScope) int { 80 | if n <= 1 { 81 | return n 82 | } 83 | sc = sc.Sub("collatz").Open(n) 84 | if n&1 == 1 { 85 | n = 3*n + 1 86 | sc.Close(n) 87 | return collatz(n, sc) 88 | } 89 | n = n >> 1 90 | sc.Close(n) 91 | return collatz(n, sc) 92 | } 93 | -------------------------------------------------------------------------------- /internal/test/watcher.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package test 22 | 23 | import "fmt" 24 | 25 | type element struct { 26 | Item interface{} 27 | Items []interface{} 28 | } 29 | 30 | // Watcher implements a source.GenericDataWatcher that appends to a slice for 31 | // testing purposes. 32 | type Watcher struct { 33 | Q []element 34 | } 35 | 36 | // NewWatcher creates a new test watcher. 37 | func NewWatcher() *Watcher { 38 | return &Watcher{} 39 | } 40 | 41 | // Active always returns true. 42 | func (wat *Watcher) Active() bool { 43 | return true 44 | } 45 | 46 | // HandleItem appends the item to the Q and always returns true. 47 | func (wat *Watcher) HandleItem(item interface{}) bool { 48 | wat.Q = append(wat.Q, element{Item: item}) 49 | return true 50 | } 51 | 52 | // HandleItems appends the items to the Q and always returns true. 53 | func (wat *Watcher) HandleItems(items []interface{}) bool { 54 | wat.Q = append(wat.Q, element{Items: items}) 55 | return true 56 | } 57 | 58 | // AllItems returns a list containing all of the items in the Q flattened out. 59 | func (wat *Watcher) AllItems() (items []interface{}) { 60 | for _, el := range wat.Q { 61 | if el.Item != nil { 62 | items = append(items, el.Item) 63 | } else { 64 | items = append(items, el.Items...) 65 | } 66 | } 67 | return 68 | } 69 | 70 | // AllStrings retruns a list of strings called by either .String-ing each item 71 | // in AllItems, or fmting it with "%#v". 72 | func (wat *Watcher) AllStrings() (strs []string) { 73 | each := func(items ...interface{}) { 74 | for _, item := range items { 75 | if strer, ok := item.(fmt.Stringer); ok { 76 | strs = append(strs, strer.String()) 77 | } else { 78 | strs = append(strs, fmt.Sprintf("%#v", item)) 79 | } 80 | } 81 | } 82 | for _, el := range wat.Q { 83 | if el.Item != nil { 84 | each(el.Item) 85 | } else { 86 | each(el.Items...) 87 | } 88 | } 89 | return 90 | } 91 | -------------------------------------------------------------------------------- /example_server/main.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package main 22 | 23 | import ( 24 | "fmt" 25 | "io" 26 | "log" 27 | "net/http" 28 | "strconv" 29 | 30 | gwr "github.com/uber-go/gwr" 31 | "github.com/uber-go/gwr/source/tap" 32 | ) 33 | 34 | func main() { 35 | if err := gwr.Configure(nil); err != nil { 36 | log.Fatal(err) 37 | } 38 | 39 | resLog := &resLogger{handler: http.DefaultServeMux} 40 | reqLog := &reqLogger{handler: resLog} 41 | 42 | gwr.AddGenericDataSource(reqLog) 43 | gwr.AddGenericDataSource(resLog) 44 | 45 | fb := fibber{ 46 | naive: tap.AddNewTracer("fib/naive"), 47 | } 48 | http.HandleFunc("/fib/naive", fb.handleNaive) 49 | 50 | log.Fatal(http.ListenAndServe(":8080", reqLog)) 51 | } 52 | 53 | type fibber struct { 54 | naive *tap.Tracer 55 | } 56 | 57 | func (fb *fibber) handleNaive(w http.ResponseWriter, r *http.Request) { 58 | trc := fb.naive.Scope("handleNaive").Open(map[string]interface{}{ 59 | "method": r.Method, 60 | "proto": r.Proto, 61 | "url": r.URL, 62 | "header": r.Header, 63 | "host": r.Host, 64 | }) 65 | defer trc.Close() 66 | 67 | if err := r.ParseForm(); err != nil { 68 | http.Error(w, "400 Bad Request", http.StatusBadRequest) 69 | trc.ErrorName("ParseForm", err) 70 | return 71 | } 72 | trc.Info("parsed form", r.Form) 73 | 74 | i, err := strconv.Atoi(r.Form.Get("n")) 75 | if err != nil { 76 | http.Error(w, "400 Bad Request", http.StatusBadRequest) 77 | trc.ErrorName("Atoi", err) 78 | return 79 | } 80 | 81 | n := naiveFib(i, trc) 82 | io.WriteString(w, fmt.Sprintf("fib(%d) = %d\n", i, n)) 83 | } 84 | 85 | func naiveFib(i int, trc *tap.TraceScope) (n int) { 86 | sc := trc.Sub("naiveFib").Open(i) 87 | defer func() { 88 | sc.Close(n) 89 | }() 90 | 91 | if i <= 0 { 92 | n = 0 93 | return 94 | } 95 | 96 | if i <= 2 { 97 | n = 1 98 | return 99 | } 100 | 101 | n = naiveFib(i-1, sc) + naiveFib(i-2, sc) 102 | return 103 | } 104 | -------------------------------------------------------------------------------- /source/sources.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package source 22 | 23 | import "errors" 24 | 25 | var ErrSourceAlreadyDefined = errors.New("data source already defined") 26 | 27 | // DataSourcesObserver is an interface to observe data sources changes. 28 | // 29 | // Observation happens after after the source has been added (resp. removed). 30 | type DataSourcesObserver interface { 31 | SourceAdded(ds DataSource) 32 | SourceRemoved(ds DataSource) 33 | } 34 | 35 | // DataSources is a flat collection of DataSources 36 | // with a meta introspection data source. 37 | type DataSources struct { 38 | sources map[string]DataSource 39 | obs DataSourcesObserver 40 | } 41 | 42 | // NewDataSources creates a DataSources structure 43 | // an sets up its "/meta/nouns" data source. 44 | func NewDataSources() *DataSources { 45 | dss := &DataSources{ 46 | sources: make(map[string]DataSource, 2), 47 | } 48 | return dss 49 | } 50 | 51 | // SetObserver sets the (single!) observer of data source changes; if nil is 52 | // passed, observation is disabled. 53 | func (dss *DataSources) SetObserver(obs DataSourcesObserver) { 54 | dss.obs = obs 55 | } 56 | 57 | // Get returns the named data source or nil if none is defined. 58 | func (dss *DataSources) Get(name string) DataSource { 59 | source, ok := dss.sources[name] 60 | if ok { 61 | return source 62 | } 63 | return nil 64 | } 65 | 66 | // Add a DataSource, if none is already defined for the given name. 67 | func (dss *DataSources) Add(ds DataSource) error { 68 | name := ds.Name() 69 | if _, ok := dss.sources[name]; ok { 70 | return ErrSourceAlreadyDefined 71 | } 72 | dss.sources[name] = ds 73 | if dss.obs != nil { 74 | dss.obs.SourceAdded(ds) 75 | } 76 | return nil 77 | } 78 | 79 | // Remove a DataSource by name, if any exsits. Returns the source removed, nil 80 | // if none was defined. 81 | func (dss *DataSources) Remove(name string) DataSource { 82 | ds, ok := dss.sources[name] 83 | if ok { 84 | delete(dss.sources, name) 85 | if dss.obs != nil { 86 | dss.obs.SourceRemoved(ds) 87 | } 88 | } 89 | return ds 90 | } 91 | -------------------------------------------------------------------------------- /internal/resp/value.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package resp 22 | 23 | import ( 24 | "fmt" 25 | "strconv" 26 | ) 27 | 28 | // RedisValue implements a scalar value from the redis protocol (string or int) 29 | type RedisValue struct { 30 | isNum bool 31 | num int 32 | buf []byte 33 | } 34 | 35 | var NilRedisValue = RedisValue{} 36 | 37 | func NewIntRedisValue(num int) RedisValue { 38 | return RedisValue{ 39 | isNum: true, 40 | num: num, 41 | } 42 | } 43 | 44 | func NewBytesRedisValue(buf []byte) RedisValue { 45 | return RedisValue{ 46 | isNum: false, 47 | buf: buf, 48 | } 49 | } 50 | 51 | func NewStringRedisValue(str string) RedisValue { 52 | buf := []byte(str) 53 | return RedisValue{ 54 | isNum: false, 55 | buf: buf, 56 | } 57 | } 58 | 59 | func (rv RedisValue) IsNumber() bool { 60 | return rv.isNum 61 | } 62 | 63 | func (rv RedisValue) IsNull() bool { 64 | if rv.isNum { 65 | return false 66 | } 67 | return rv.buf == nil 68 | } 69 | 70 | func (rv RedisValue) GetNumber() (int, bool) { 71 | if !rv.isNum { 72 | return 0, false 73 | } 74 | return rv.num, true 75 | } 76 | 77 | func (rv RedisValue) GetBytes() ([]byte, bool) { 78 | if rv.isNum { 79 | return nil, false 80 | } 81 | if rv.buf == nil { 82 | return nil, false 83 | } 84 | return rv.buf, true 85 | } 86 | 87 | func (rv RedisValue) GetString() (string, bool) { 88 | if rv.isNum { 89 | return "", false 90 | } 91 | if rv.buf == nil { 92 | return "", false 93 | } 94 | return string(rv.buf), true 95 | } 96 | 97 | func (rv RedisValue) WriteTo(rconn *RedisConnection) error { 98 | if rv.isNum { 99 | return rconn.WriteInteger(rv.num) 100 | } 101 | if rv.buf != nil { 102 | return rconn.WriteBulkBytes(rv.buf) 103 | } 104 | return rconn.WriteNull() 105 | } 106 | 107 | func (rv RedisValue) String() string { 108 | if rv.isNum { 109 | return strconv.Itoa(rv.num) 110 | } else if rv.buf != nil { 111 | return fmt.Sprintf("%#v", string(rv.buf)) 112 | } else { 113 | return "null" 114 | } 115 | } 116 | 117 | type RedisArray []RedisValue 118 | 119 | func (ra RedisArray) WriteTo(rconn *RedisConnection) error { 120 | if err := rconn.WriteArrayHeader(len(ra)); err != nil { 121 | return err 122 | } 123 | for _, val := range ra { 124 | if err := val.WriteTo(rconn); err != nil { 125 | return err 126 | } 127 | } 128 | return nil 129 | } 130 | -------------------------------------------------------------------------------- /source/item_data_source.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package source 22 | 23 | // ItemDataSource is an interface implemented by a data source to provide 24 | // marshaled but unframed streams of Watch items. When implemented protocol 25 | // libraries can add protocol-specific builtin framing. 26 | type ItemDataSource interface { 27 | // WatchItems implementations have all of the semantics of 28 | // DataSource.Watch, just over an ItemWatcher instead of an io.Writer. 29 | // 30 | // If the data source naturally has batches of items on hand it may call 31 | // watcher.HandleItems. 32 | // 33 | // Passed watchers must be discarded after either HandleItem or 34 | // HandleItems returns a non-nil error. 35 | WatchItems(format string, watcher ItemWatcher) error 36 | } 37 | 38 | // ItemWatcher is the interface passed to ItemSource.WatchItems. Any 39 | // error returned by either HandleItem or HandleItems indicates that this 40 | // watcher should not be called with more items. 41 | type ItemWatcher interface { 42 | // HandleItem gets a single marshaled item, and should return any framing 43 | // or write error. 44 | HandleItem(item []byte) error 45 | 46 | // HandleItems gets a batch of marshaled items, and should return any 47 | // framing or write error. 48 | HandleItems(items [][]byte) error 49 | } 50 | 51 | // ItemWatcherFunc is a convenience type for watching with a simple 52 | // per-item function. The item function should return any framing or write 53 | // error. 54 | type ItemWatcherFunc func([]byte) error 55 | 56 | // HandleItem just calls the wrapped function. 57 | func (itemFunc ItemWatcherFunc) HandleItem(item []byte) error { 58 | return itemFunc(item) 59 | } 60 | 61 | // HandleItems calls the wrapped function for each item in the batch, stopping 62 | // on and returning the first error. 63 | func (itemFunc ItemWatcherFunc) HandleItems(items [][]byte) error { 64 | for _, item := range items { 65 | if err := itemFunc(item); err != nil { 66 | return err 67 | } 68 | } 69 | return nil 70 | } 71 | 72 | // ItemWatcherBatchFunc is a convenience type for watching with a simple 73 | // batch function. The batch function should return any framing or write 74 | // error. 75 | type ItemWatcherBatchFunc func([][]byte) error 76 | 77 | // HandleItem calls the wrapped function with a singleton batch. 78 | func (batchFunc ItemWatcherBatchFunc) HandleItem(item []byte) error { 79 | return batchFunc([][]byte{item}) 80 | } 81 | 82 | // HandleItems just calls the wrapped function. 83 | func (batchFunc ItemWatcherBatchFunc) HandleItems(items [][]byte) error { 84 | return batchFunc(items) 85 | } 86 | -------------------------------------------------------------------------------- /serve.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package gwr 22 | 23 | import ( 24 | "bufio" 25 | "errors" 26 | "net" 27 | "net/http" 28 | 29 | "github.com/uber-go/gwr/internal/protocol" 30 | "github.com/uber-go/gwr/internal/resp" 31 | "github.com/uber-go/gwr/source" 32 | 33 | "github.com/uber-common/stacked" 34 | ) 35 | 36 | var errNoServer = errors.New("no server configured") 37 | 38 | type indirectServer struct { 39 | cs **ConfiguredServer 40 | } 41 | 42 | func (is indirectServer) Addr() net.Addr { 43 | srv := *(is.cs) 44 | if srv == nil { 45 | return nil 46 | } 47 | return srv.Addr() 48 | } 49 | 50 | func (is indirectServer) StartOn(laddr string) error { 51 | srv := *(is.cs) 52 | if srv == nil { 53 | return errNoServer 54 | } 55 | return srv.StartOn(laddr) 56 | } 57 | 58 | func (is indirectServer) Stop() error { 59 | srv := *(is.cs) 60 | if srv == nil { 61 | return errNoServer 62 | } 63 | return srv.Stop() 64 | } 65 | 66 | func init() { 67 | hh := protocol.NewHTTPRest(DefaultDataSources, "/gwr", indirectServer{&theServer}) 68 | http.Handle("/gwr/", hh) 69 | } 70 | 71 | // ListenAndServeResp starts a resp protocol gwr server. 72 | func ListenAndServeResp(hostPort string, dss *source.DataSources) error { 73 | if dss == nil { 74 | dss = DefaultDataSources 75 | } 76 | return protocol.NewRedisServer(dss).ListenAndServe(hostPort) 77 | } 78 | 79 | // ListenAndServeHTTP starts an http protocol gwr server. 80 | func ListenAndServeHTTP(hostPort string, dss *source.DataSources) error { 81 | if dss == nil { 82 | dss = DefaultDataSources 83 | } 84 | hh := protocol.NewHTTPRest(dss, "", indirectServer{&theServer}) 85 | return http.ListenAndServe(hostPort, hh) 86 | } 87 | 88 | // NewServer creates an "auto" protocol server that will respond to HTTP or 89 | // RESP requests. 90 | func NewServer(dss *source.DataSources) stacked.Server { 91 | if dss == nil { 92 | dss = DefaultDataSources 93 | } 94 | hh := protocol.NewHTTPRest(dss, "", indirectServer{&theServer}) 95 | rh := protocol.NewRedisHandler(dss) 96 | return stacked.NewServer( 97 | respDetector(rh), 98 | stacked.DefaultHTTPHandler(hh), 99 | ) 100 | } 101 | 102 | func respDetector(respHandler resp.RedisHandler) stacked.Detector { 103 | hndl := stacked.HandlerFunc(func(conn net.Conn, bufr *bufio.Reader) { 104 | resp.NewRedisConnection(conn, bufr).Handle(respHandler) 105 | }) 106 | return stacked.Detector{ 107 | Needed: 1, 108 | Test: resp.IsFirstByteRespTag, 109 | Handler: hndl, 110 | } 111 | } 112 | 113 | // ListenAndServe starts an "auto" protocol server that will respond to HTTP or 114 | // RESP on the given hostPort. 115 | func ListenAndServe(hostPort string, dss *source.DataSources) error { 116 | return NewServer(dss).ListenAndServe(hostPort) 117 | } 118 | -------------------------------------------------------------------------------- /internal/marshaled/template.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package marshaled 22 | 23 | import ( 24 | "bytes" 25 | "fmt" 26 | "text/template" 27 | ) 28 | 29 | // TemplatedMarshal hooks together text/template to create a data source. 30 | type TemplatedMarshal struct { 31 | tmpl *template.Template 32 | getName, initName, itemName string 33 | } 34 | 35 | // NewTemplatedMarshal creates a TemplatedMarshal from a template. The default 36 | // template names "get", "init", and "item" are used, if defined in the parsed 37 | // template. 38 | func NewTemplatedMarshal(tmpl *template.Template) *TemplatedMarshal { 39 | var getName, initName, itemName string 40 | if tmpl.Lookup("get") != nil { 41 | getName = "get" 42 | } 43 | if tmpl.Lookup("init") != nil { 44 | initName = "init" 45 | } 46 | if tmpl.Lookup("item") != nil { 47 | itemName = "item" 48 | } 49 | return &TemplatedMarshal{ 50 | tmpl: tmpl, 51 | getName: getName, 52 | initName: initName, 53 | itemName: itemName, 54 | } 55 | } 56 | 57 | // TODO: need accessors for the names? 58 | 59 | // MarshalGet returns the rendered bytes from the get template. If no get 60 | // template is defined, an error is returned. 61 | func (tm *TemplatedMarshal) MarshalGet(data interface{}) ([]byte, error) { 62 | if len(tm.getName) == 0 { 63 | return nil, fmt.Errorf("only streaming is supported by the data format; no get template defined") 64 | } 65 | var buf bytes.Buffer 66 | if err := tm.tmpl.ExecuteTemplate(&buf, tm.getName, data); err != nil { 67 | return nil, err 68 | } 69 | return buf.Bytes(), nil 70 | } 71 | 72 | // MarshalInit returns the rendered bytes from the init template. If no init 73 | // template is defined, an error is returned. 74 | func (tm *TemplatedMarshal) MarshalInit(data interface{}) ([]byte, error) { 75 | if len(tm.initName) == 0 { 76 | return nil, fmt.Errorf("streaming is unsupported by the data format; no init template defined") 77 | } 78 | var buf bytes.Buffer 79 | if err := tm.tmpl.ExecuteTemplate(&buf, tm.initName, data); err != nil { 80 | return nil, err 81 | } 82 | return buf.Bytes(), nil 83 | } 84 | 85 | // MarshalItem returns the rendered bytes from the item template. If no item 86 | // template is defined, an error is returned. 87 | func (tm *TemplatedMarshal) MarshalItem(data interface{}) ([]byte, error) { 88 | if len(tm.itemName) == 0 { 89 | return nil, fmt.Errorf("streaming is unsupported by the data format; no item template defined") 90 | } 91 | var buf bytes.Buffer 92 | if err := tm.tmpl.ExecuteTemplate(&buf, tm.itemName, data); err != nil { 93 | return nil, err 94 | } 95 | return buf.Bytes(), nil 96 | } 97 | 98 | // FrameItem appends a newline 99 | func (tm *TemplatedMarshal) FrameItem(json []byte) ([]byte, error) { 100 | n := len(json) 101 | frame := make([]byte, n+1) 102 | copy(frame, json) 103 | frame[n] = '\n' 104 | return frame, nil 105 | } 106 | -------------------------------------------------------------------------------- /internal/meta/noun_data_source.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package meta 22 | 23 | import ( 24 | "strings" 25 | "text/template" 26 | 27 | "github.com/uber-go/gwr/source" 28 | ) 29 | 30 | // NounsName is the name of the meta nouns data source. 31 | const NounsName = "/meta/nouns" 32 | 33 | var nounsTextTemplate = template.Must(template.New("meta_nouns_text").Parse(strings.TrimSpace(` 34 | {{ define "get" }}Data Sources: 35 | {{ range $name, $info := . }}{{ $name }} formats: {{ $info.Formats }} 36 | {{ end }}{{ end }} 37 | `))) 38 | 39 | // NounDataSource provides a data source that describes other data sources. It 40 | // is used to implement the "/meta/nouns" data source. 41 | type NounDataSource struct { 42 | sources *source.DataSources 43 | watcher source.GenericDataWatcher 44 | } 45 | 46 | // NewNounDataSource creates a new data source that gets information on other 47 | // data sources and streams updates about them. 48 | func NewNounDataSource(dss *source.DataSources) *NounDataSource { 49 | return &NounDataSource{ 50 | sources: dss, 51 | } 52 | } 53 | 54 | // Name returns the static "/meta/nouns" string; currently using more than one 55 | // NounDataSource in a single DataSources is unsupported. 56 | func (nds *NounDataSource) Name() string { 57 | return NounsName 58 | } 59 | 60 | // TextTemplate returns a text/template to implement the GenericDataSource with 61 | // a "text" format option. 62 | func (nds *NounDataSource) TextTemplate() *template.Template { 63 | return nounsTextTemplate 64 | } 65 | 66 | // Get returns all currently knows data sources. 67 | func (nds *NounDataSource) Get() interface{} { 68 | return nds.sources.Info() 69 | } 70 | 71 | // WatchInit returns identical data to Get so that all Watch streams start out 72 | // with a snapshot of the world. 73 | func (nds *NounDataSource) WatchInit() interface{} { 74 | return nds.Get() 75 | } 76 | 77 | // SetWatcher implements GenericDataSource by retaining a reference to the 78 | // passed watcher. Updates are later sent to the watcher when new data sources 79 | // are added and removed. 80 | func (nds *NounDataSource) SetWatcher(watcher source.GenericDataWatcher) { 81 | nds.watcher = watcher 82 | } 83 | 84 | // SourceAdded is called whenever a source is added to the DataSources. 85 | func (nds *NounDataSource) SourceAdded(ds source.DataSource) { 86 | if !nds.watcher.Active() { 87 | return 88 | } 89 | nds.watcher.HandleItem(struct { 90 | Type string `json:"type"` 91 | Name string `json:"name"` 92 | Info source.Info `json:"info"` 93 | }{"add", ds.Name(), source.GetInfo(ds)}) 94 | } 95 | 96 | // SourceRemoved is called whenever a source is removed from the DataSources. 97 | func (nds *NounDataSource) SourceRemoved(ds source.DataSource) { 98 | if !nds.watcher.Active() { 99 | return 100 | } 101 | nds.watcher.HandleItem(struct { 102 | Type string `json:"type"` 103 | Name string `json:"name"` 104 | }{"remove", ds.Name()}) 105 | } 106 | -------------------------------------------------------------------------------- /source/tap/emitter.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package tap 22 | 23 | import ( 24 | "fmt" 25 | "text/template" 26 | 27 | "github.com/uber-go/gwr" 28 | "github.com/uber-go/gwr/internal" 29 | "github.com/uber-go/gwr/source" 30 | ) 31 | 32 | type stringer interface { 33 | String() string 34 | } 35 | 36 | var defaultTextFormat = internal.FormatFunc(func(val interface{}) ([]byte, error) { 37 | if str, ok := val.(stringer); ok { 38 | return []byte(str.String()), nil 39 | } 40 | return []byte(fmt.Sprintf("%#v", val)), nil 41 | }) 42 | 43 | // TODO: simple sampling extension to Emitter to implement Get 44 | 45 | // Emitter provides a simple watchable data source with easy emission. 46 | type Emitter struct { 47 | name string 48 | tmpl *template.Template 49 | watcher source.GenericDataWatcher 50 | } 51 | 52 | // NewEmitter creates an Emitter with a given name and text template; if the 53 | // template is nil, than a default template which just uses the default textual 54 | // representation is used. 55 | // 56 | // The given name will be prefixed with "/tap/" automatically. 57 | // 58 | // Any templated passed must define an "item" block. 59 | func NewEmitter(name string, tmpl *template.Template) *Emitter { 60 | name = fmt.Sprintf("/tap/%s", name) 61 | return &Emitter{ 62 | name: name, 63 | tmpl: tmpl, 64 | } 65 | } 66 | 67 | // AddEmitter creates an emitter source and adds it to the default gwr sources. 68 | func AddEmitter(name string, tmpl *template.Template) *Emitter { 69 | tap := NewEmitter(name, tmpl) 70 | gwr.AddGenericDataSource(tap) 71 | return tap 72 | } 73 | 74 | // Name returns the full name of the emitter source; this will be 75 | // "/tap/name_given_to_New_Emitter". 76 | func (em *Emitter) Name() string { 77 | return em.name 78 | } 79 | 80 | // TextTemplate returns the template used to marshal items human friendily. 81 | func (em *Emitter) TextTemplate() *template.Template { 82 | return em.tmpl 83 | } 84 | 85 | // Formats returns emitter-specific formats. 86 | func (em *Emitter) Formats() map[string]source.GenericDataFormat { 87 | if em.tmpl != nil { 88 | return nil 89 | } 90 | return map[string]source.GenericDataFormat{ 91 | "text": defaultTextFormat, 92 | } 93 | } 94 | 95 | // SetWatcher sets the watcher at source addition time. 96 | func (em *Emitter) SetWatcher(watcher source.GenericDataWatcher) { 97 | em.watcher = watcher 98 | } 99 | 100 | // Active retruns true if there are any active watchers. 101 | func (em *Emitter) Active() bool { 102 | return em.watcher.Active() 103 | } 104 | 105 | // Emit emits item(s) to any active watchers. Returns true if the watcher is 106 | // (still) active. 107 | func (em *Emitter) Emit(items ...interface{}) bool { 108 | if !em.watcher.Active() { 109 | return false 110 | } 111 | switch len(items) { 112 | case 0: 113 | return true 114 | case 1: 115 | return em.watcher.HandleItem(items[0]) 116 | default: 117 | return em.watcher.HandleItems(items) 118 | } 119 | } 120 | 121 | // EmitBatch emits batch of items. Returns true if the watcher is (still) 122 | // active. 123 | func (em *Emitter) EmitBatch(items []interface{}) bool { 124 | if !em.watcher.Active() { 125 | return false 126 | } 127 | return em.watcher.HandleItems(items) 128 | } 129 | -------------------------------------------------------------------------------- /internal/marshaled/source_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package marshaled_test 22 | 23 | import ( 24 | "bufio" 25 | "fmt" 26 | "os" 27 | "strings" 28 | "testing" 29 | "text/template" 30 | 31 | "github.com/stretchr/testify/assert" 32 | "github.com/stretchr/testify/require" 33 | 34 | "github.com/uber-go/gwr/internal/marshaled" 35 | "github.com/uber-go/gwr/source" 36 | ) 37 | 38 | type testDataSource struct { 39 | watcher source.GenericDataWatcher 40 | activated chan struct{} 41 | } 42 | 43 | func (tds *testDataSource) Name() string { 44 | return "/test" 45 | } 46 | 47 | func (tds *testDataSource) TextTemplate() *template.Template { 48 | return nil 49 | } 50 | 51 | func (tds *testDataSource) SetWatcher(watcher source.GenericDataWatcher) { 52 | tds.watcher = watcher 53 | } 54 | 55 | func (tds *testDataSource) Activate() { 56 | tds.activated <- struct{}{} 57 | } 58 | 59 | func (tds *testDataSource) emit(item interface{}) { 60 | if tds.watcher.Active() { 61 | tds.watcher.HandleItem(item) 62 | } 63 | } 64 | 65 | func (tds *testDataSource) hasActivated() bool { 66 | select { 67 | case <-tds.activated: 68 | return true 69 | default: 70 | return false 71 | } 72 | } 73 | 74 | func TestDataSource_Watch_activation(t *testing.T) { 75 | tds := &testDataSource{} 76 | tds.activated = make(chan struct{}, 1) 77 | mds := marshaled.NewDataSource(tds, nil) 78 | 79 | var ps pipeSet 80 | defer ps.close() 81 | 82 | watchit := func() { 83 | w, err := ps.add() 84 | require.NoError(t, err) 85 | require.NoError(t, mds.Watch("json", w)) 86 | } 87 | 88 | watchit() 89 | assert.True(t, tds.hasActivated(), "first watcher causes activation") 90 | 91 | // observe one 92 | tds.emit(map[string]interface{}{"hello": "world"}) 93 | ps.assertGotJSON(t, 1, `{"hello":"world"}`) 94 | 95 | watchit() 96 | assert.False(t, tds.hasActivated(), "second watcher does not cause activation") 97 | 98 | // observe two 99 | tds.emit(map[string]interface{}{"hello": "world2"}) 100 | ps.assertGotJSON(t, 2, `{"hello":"world2"}`) 101 | 102 | // TODO: further testing has synchronization needs: need to be able to wait 103 | // for mds to "drain" when it's watcher-less before moving on to next phase 104 | } 105 | 106 | type pipeSet struct { 107 | rs []*os.File 108 | scs []*bufio.Scanner 109 | } 110 | 111 | func (ps *pipeSet) add() (*os.File, error) { 112 | r, w, err := os.Pipe() 113 | if err == nil { 114 | ps.rs = append(ps.rs, r) 115 | ps.scs = append(ps.scs, bufio.NewScanner(r)) 116 | } 117 | return w, err 118 | } 119 | 120 | func (ps *pipeSet) closeOne(i int) error { 121 | if i >= len(ps.rs) { 122 | return fmt.Errorf("invalid index") 123 | } 124 | r := ps.rs[i] 125 | ps.rs = append(ps.rs[:i], ps.rs[i+1:]...) 126 | ps.scs = append(ps.scs[:i], ps.scs[i+1:]...) 127 | return r.Close() 128 | } 129 | 130 | func (ps *pipeSet) close() { 131 | for _, r := range ps.rs { 132 | r.Close() 133 | } 134 | } 135 | 136 | func (ps *pipeSet) assertGotJSON(t *testing.T, n int, expected string, msgAndArgs ...interface{}) { 137 | var m int 138 | for _, sc := range ps.scs { 139 | assertJSONScanLine(t, sc, expected, msgAndArgs...) 140 | m++ 141 | } 142 | assert.Equal(t, n, m, msgAndArgs...) 143 | } 144 | 145 | func assertJSONScanLine(t *testing.T, sc *bufio.Scanner, expected string, msgAndArgs ...interface{}) { 146 | if !sc.Scan() { 147 | assert.Fail(t, "expected to scan a JSON line", msgAndArgs...) 148 | } else { 149 | expected = strings.Join([]string{expected, "\n"}, "") 150 | assert.JSONEq(t, expected, sc.Text(), msgAndArgs...) 151 | } 152 | assert.NoError(t, sc.Err()) 153 | } 154 | -------------------------------------------------------------------------------- /internal/meta/noun_data_source_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package meta_test 22 | 23 | import ( 24 | "bufio" 25 | "bytes" 26 | "os" 27 | "strings" 28 | "testing" 29 | "text/template" 30 | 31 | "github.com/uber-go/gwr/internal/marshaled" 32 | "github.com/uber-go/gwr/internal/meta" 33 | "github.com/uber-go/gwr/source" 34 | 35 | "github.com/stretchr/testify/assert" 36 | ) 37 | 38 | type dummyDataSource struct { 39 | name string 40 | tmpl *template.Template 41 | } 42 | 43 | func (dds *dummyDataSource) Name() string { 44 | return dds.name 45 | } 46 | 47 | func (dds *dummyDataSource) TextTemplate() *template.Template { 48 | return dds.tmpl 49 | } 50 | 51 | func setup() *source.DataSources { 52 | dss := source.NewDataSources() 53 | nds := meta.NewNounDataSource(dss) 54 | dss.Add(marshaled.NewDataSource(nds, nil)) 55 | dss.SetObserver(nds) 56 | return dss 57 | } 58 | 59 | func TestNounDataSource_Watch(t *testing.T) { 60 | dss := setup() 61 | mds := dss.Get("/meta/nouns") 62 | 63 | r, w, err := os.Pipe() 64 | if err != nil { 65 | t.Fatal(err) 66 | } 67 | 68 | // buf := newInternalRWC() 69 | sc := bufio.NewScanner(r) 70 | if err := mds.Watch("json", w); err != nil { 71 | t.Fatal(err) 72 | } 73 | 74 | getText := func() string { 75 | var buf bytes.Buffer 76 | mds.Get("text", &buf) 77 | return buf.String() 78 | } 79 | 80 | // verify init data 81 | assertJSONScanLine(t, sc, 82 | `{"/meta/nouns":{"formats":["json","text"],"attrs":null}}`, 83 | "should get /meta/nouns initially") 84 | assert.Equal(t, getText(), "Data Sources:\n"+ 85 | "/meta/nouns formats: [json text]\n") 86 | 87 | // add a data source, observe it 88 | assert.NoError(t, dss.Add(marshaled.NewDataSource(&dummyDataSource{ 89 | name: "/foo", 90 | tmpl: nil, 91 | }, nil)), "no add error expected") 92 | assertJSONScanLine(t, sc, 93 | `{"name":"/foo","type":"add","info":{"formats":["json","text"],"attrs":null}}`, 94 | "should get an add event for /foo") 95 | assert.Equal(t, getText(), "Data Sources:\n"+ 96 | "/foo formats: [json text]\n"+ 97 | "/meta/nouns formats: [json text]\n") 98 | 99 | // add another data source, observe it 100 | assert.NoError(t, dss.Add(marshaled.NewDataSource(&dummyDataSource{ 101 | name: "/bar", 102 | tmpl: template.Must(template.New("bar_tmpl").Parse("")), 103 | }, nil)), "no add error expected") 104 | assertJSONScanLine(t, sc, 105 | `{"name":"/bar","type":"add","info":{"formats":["json","text"],"attrs":null}}`, 106 | "should get an add event for /bar") 107 | assert.Equal(t, getText(), "Data Sources:\n"+ 108 | "/bar formats: [json text]\n"+ 109 | "/foo formats: [json text]\n"+ 110 | "/meta/nouns formats: [json text]\n") 111 | 112 | // remove the /foo data source, observe it 113 | assert.NotNil(t, dss.Remove("/foo"), "expected a removed data source") 114 | assertJSONScanLine(t, sc, 115 | `{"name":"/foo","type":"remove"}`, 116 | "should get a remove event for /foo") 117 | assert.Equal(t, getText(), "Data Sources:\n"+ 118 | "/bar formats: [json text]\n"+ 119 | "/meta/nouns formats: [json text]\n") 120 | 121 | // remove the /bar data source, observe it 122 | assert.NotNil(t, dss.Remove("/bar"), "expected a removed data source") 123 | assertJSONScanLine(t, sc, 124 | `{"name":"/bar","type":"remove"}`, 125 | "should get a remove event for /bar") 126 | assert.Equal(t, getText(), "Data Sources:\n"+ 127 | "/meta/nouns formats: [json text]\n") 128 | 129 | // shutdown the watch stream 130 | assert.NoError(t, r.Close()) 131 | assert.False(t, sc.Scan(), "no more scan") 132 | } 133 | 134 | func assertJSONScanLine(t *testing.T, sc *bufio.Scanner, expected string, msgAndArgs ...interface{}) { 135 | if !sc.Scan() { 136 | assert.Fail(t, "expected to scan a JSON line", msgAndArgs...) 137 | } else { 138 | expected = strings.Join([]string{expected, "\n"}, "") 139 | assert.JSONEq(t, sc.Text(), expected, msgAndArgs...) 140 | } 141 | assert.NoError(t, sc.Err()) 142 | } 143 | -------------------------------------------------------------------------------- /source/source.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package source 22 | 23 | import ( 24 | "errors" 25 | "io" 26 | ) 27 | 28 | var ( 29 | // ErrUnsupportedFormat should be returned by DataSource.Get and 30 | // DataSource.Watch if the requested format is not supported. 31 | ErrUnsupportedFormat = errors.New("unsupported format") 32 | 33 | // ErrNotGetable should be returned by DataSource.Get if the data source 34 | // does not support get. 35 | ErrNotGetable = errors.New("get not supported, data source is watch-only") 36 | 37 | // ErrNotWatchable should be returned by DataSource.Get if the data source 38 | // does not support watch. 39 | ErrNotWatchable = errors.New("watch not supported, data source is get-only") 40 | ) 41 | 42 | // DataSource is the low-level interface implemented by all data sources. 43 | // 44 | // On formats, implementanions: 45 | // - must implement format == "json" 46 | // - should implement format == "text" 47 | // - may implement any other formats that make sense for them 48 | // 49 | // Further implementation requirements are listed within the interface 50 | // functions' documentation. 51 | type DataSource interface { 52 | // Name returns the unique identifier (GWR noun path) for this source. 53 | Name() string 54 | 55 | // Formats returns a list of supported format name strings. All 56 | // implemented formats must be listed. At least "json" must be supported. 57 | Formats() []string 58 | 59 | // Attrs returnts arbitrary descriptive data about the data source. This 60 | // data is exposed be the /meta/nouns data source. 61 | // 62 | // TODO: standardize and document common fields; current ideas include: 63 | // - affording get-only or watch-only 64 | // - affording sampling config (%-age, N-per-t, etc) 65 | // - whether this data source is lossy (likely the default) or whether 66 | // attempts should be taken to drop no item (opt-in edge case); this 67 | // could be used internally at least to switch between an implementation 68 | // that drops data when buffers fill, or blocks and provides back 69 | // pressure. 70 | Attrs() map[string]interface{} 71 | 72 | // Get implementations: 73 | // - may return ErrNotGetable if get is not supported by the data source 74 | // - if the format is not support then ErrUnsupportedFormat must be returned 75 | // - must format and write any available data to the supplied io.Writer 76 | // - should return any write error 77 | Get(format string, w io.Writer) error 78 | 79 | // Watch implementations: 80 | // - may return ErrNotWatchable if watch is not supported by the data 81 | // source 82 | // - if the format is not support then ErrUnsupportedFormat must be returned 83 | // - may format and write initial data to the supplied io.Writer; any 84 | // initial write error must be returned 85 | // - may retain and write to the supplied io.Writer indefinately until it 86 | // returns a write error 87 | // 88 | // Note that at this level, data sources are responsible for both item 89 | // marshalling and stream framing. 90 | // 91 | // Framing for the required "json" format is as follows: 92 | // - JSON must be encoded in compact (no intermediate whitespace) form 93 | // - each JSON record must be separated by a newline "\n" 94 | // 95 | // Framing for the required "text" format is as follows: 96 | // - any initial stream data should be followed by a blank line (double new 97 | // line "\n\n") 98 | // - items should be separated by newlines 99 | // - if an item's text form takes up multiple lines, it should either use 100 | // indentation or a double blank line to separate itself from siblings 101 | Watch(format string, w io.Writer) error 102 | } 103 | 104 | // DrainableSource is a DataSource that can be drained. Draining a source 105 | // should flush any unsent data, and then close any remaining Watch writers. 106 | type DrainableSource interface { 107 | DataSource 108 | Drain() 109 | } 110 | 111 | // TODO: should add a ClosableSource so that DataSources.Remove can close any 112 | // active watchers etc. 113 | -------------------------------------------------------------------------------- /source/generic.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package source 22 | 23 | import "text/template" 24 | 25 | // GenericDataWatcher is the interface for the watcher passed to 26 | // GenericDataSource.SetWatcher. Both single-item and batch methods are 27 | // provided. 28 | type GenericDataWatcher interface { 29 | Active() bool 30 | 31 | // HandleItem is called with a single item of generic unmarshaled data. 32 | HandleItem(item interface{}) bool 33 | 34 | // HandleItem is called with a batch of generic unmarshaled data. 35 | HandleItems(items []interface{}) bool 36 | } 37 | 38 | // GenericDataSource is a format-agnostic data source 39 | type GenericDataSource interface { 40 | // Name must return the name of the data source; see DataSource.Name. 41 | Name() string 42 | } 43 | 44 | // TextTemplatedSource is implemented by generic data sources to provide a 45 | // convenience template for the "text" format. 46 | type TextTemplatedSource interface { 47 | // TextTemplate returns the text/template that is used to construct a 48 | // TemplatedMarshal to implement the "text" format for this data source. 49 | TextTemplate() *template.Template 50 | } 51 | 52 | // GenericDataSourceFormats is implemented by generic data sources to define 53 | // additional formats beyond the default json and templated text ones. 54 | type GenericDataSourceFormats interface { 55 | Formats() map[string]GenericDataFormat 56 | } 57 | 58 | // GetableDataSource is the interface implemented by GenericDataSources that 59 | // support Get. If a GenericDataSource does not implement GetableDataSource, 60 | // then any gets for it return source.ErrNotGetable. 61 | type GetableDataSource interface { 62 | GenericDataSource 63 | 64 | // Get should return any data available for the data source. 65 | Get() interface{} 66 | } 67 | 68 | // WatchableDataSource is the interface implemented by GenericDataSources that 69 | // support Watch. If a GenericDataSource does not implement 70 | // WatchableDataSource, then any watches for it return source.ErrNotWatchable. 71 | type WatchableDataSource interface { 72 | GenericDataSource 73 | 74 | // SetWatcher sets the watcher. 75 | // 76 | // Implementations should retain a reference to the last passed watcher, 77 | // and need not retain multiple; in the usual case this method will only be 78 | // called once per data source lifecycle. 79 | // 80 | // Implementations should pass items to watcher.HandleItem and/or 81 | // watcher.HandleItems methods. 82 | // 83 | // Implementations may use watcher.Active to avoid building items which 84 | // would just be thrown out by a call to HandleItem(s). 85 | SetWatcher(watcher GenericDataWatcher) 86 | } 87 | 88 | // ActivateWatchableDataSource is an optional interface that 89 | // WatchableDataSources may implement to get notified about source activation. 90 | type ActivateWatchableDataSource interface { 91 | WatchableDataSource 92 | 93 | // Activate gets called when the GenericDataWatcher transitions from 94 | // inactive to active. It may be used by implementations to start or 95 | // trigger any resources needed to generate items to pass to the set 96 | // GenericDataWatcher. 97 | Activate() 98 | } 99 | 100 | // WatchInitableDataSource is the interface that a WatchableDataSource should 101 | // implement if it wants to provide an initial data item to all new watch 102 | // streams. 103 | type WatchInitableDataSource interface { 104 | WatchableDataSource 105 | 106 | // GetInit should returns initial data to send to new watch streams. 107 | WatchInit() interface{} 108 | } 109 | 110 | // GenericDataFormat provides both a data marshaling protocol and a framing 111 | // protocol for the watch stream. Any marshaling or framing error should cause 112 | // a break in any watch streams subscribed to this format. 113 | type GenericDataFormat interface { 114 | // MarshalGet serializes the passed data from GenericDataSource.Get. 115 | MarshalGet(interface{}) ([]byte, error) 116 | 117 | // MarshalInit serializes the passed data from GenericDataSource.GetInit. 118 | MarshalInit(interface{}) ([]byte, error) 119 | 120 | // MarshalItem serializes data passed to a GenericDataWatcher. 121 | MarshalItem(interface{}) ([]byte, error) 122 | 123 | // FrameItem wraps a MarshalItem-ed byte buffer for a watch stream. 124 | FrameItem([]byte) ([]byte, error) 125 | } 126 | 127 | // GenericDataFormatFunc is a convenience for implement simple single-function 128 | // formats with newline framing. 129 | type GenericDataFormatFunc func(interface{}) ([]byte, error) 130 | 131 | // MarshalGet calls the wrapped function. 132 | func (fn GenericDataFormatFunc) MarshalGet(item interface{}) ([]byte, error) { 133 | return fn(item) 134 | } 135 | 136 | // MarshalInit calls the wrapped function. 137 | func (fn GenericDataFormatFunc) MarshalInit(item interface{}) ([]byte, error) { 138 | return fn(item) 139 | } 140 | 141 | // MarshalItem calls the wrapped function. 142 | func (fn GenericDataFormatFunc) MarshalItem(item interface{}) ([]byte, error) { 143 | return fn(item) 144 | } 145 | 146 | // FrameItem wraps a MarshalItem-ed byte buffer for a watch stream. 147 | func (fn GenericDataFormatFunc) FrameItem(buf []byte) ([]byte, error) { 148 | n := len(buf) 149 | frame := make([]byte, n+1) 150 | copy(frame, buf) 151 | frame[n] = '\n' 152 | return frame, nil 153 | } 154 | -------------------------------------------------------------------------------- /report/formatted.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package report 22 | 23 | import ( 24 | "errors" 25 | 26 | "github.com/uber-go/gwr/source" 27 | ) 28 | 29 | var errReporterClosed = errors.New("reporter closed") 30 | var errRawSource = errors.New("raw sources unsupported, only item data sources") 31 | 32 | // FormattedReporter reports observed items from a data source to a formatting 33 | // function. Only works with sources that support the "text" format. 34 | // 35 | // For example to send a source to stdandard output: 36 | // rep := NewPrintfReporter(someSource, fmt.Printf) 37 | // if err := rep.Start(); err != nil { 38 | // panic(err) 39 | // } 40 | // defer rep.Stop() 41 | type FormattedReporter interface { 42 | source.ItemWatcher 43 | Source() source.DataSource 44 | Start() error 45 | Stop() 46 | } 47 | 48 | // logfReporter is a FormattedReporter that targets a log formatting function. 49 | type logfReporter struct { 50 | src source.DataSource 51 | logf func(format string, args ...interface{}) 52 | stopped bool 53 | } 54 | 55 | // NewLogfReporter creates a FormattedReporter around a log formatting 56 | // function. Log formatting functions are not expected to return an error, and 57 | // are expected to handle their own framing concerns (e.g. adding a trailing 58 | // newline). 59 | func NewLogfReporter( 60 | src source.DataSource, 61 | logf func(format string, args ...interface{}), 62 | ) FormattedReporter { 63 | return &logfReporter{ 64 | src: src, 65 | logf: logf, 66 | } 67 | } 68 | 69 | // Source returns the target source. 70 | func (rep *logfReporter) Source() source.DataSource { 71 | return rep.src 72 | } 73 | 74 | // Start clears any stop flag, and starts watching the data source. 75 | func (rep *logfReporter) Start() error { 76 | var err error 77 | rep.stopped = false 78 | if isrc, ok := rep.src.(source.ItemDataSource); ok { 79 | err = isrc.WatchItems("text", rep) 80 | } else { 81 | err = errRawSource 82 | } 83 | if err != nil { 84 | rep.stopped = true 85 | } 86 | return err 87 | } 88 | 89 | // Stop sets a flag internally so that the next HandleItem(s) will return an 90 | // error, removing the watcher resource. 91 | func (rep *logfReporter) Stop() { 92 | rep.stopped = true 93 | } 94 | 95 | // HandleItem outputs the item to the logging function with a source-name 96 | // prefix. 97 | func (rep *logfReporter) HandleItem(item []byte) error { 98 | if rep.stopped { 99 | return errReporterClosed 100 | } 101 | rep.logf("%s: %s", rep.src.Name(), item) 102 | return nil 103 | } 104 | 105 | // HandleItems outputs all items to the logging function with a source-name 106 | // prefix on each item. 107 | func (rep *logfReporter) HandleItems(items [][]byte) error { 108 | if rep.stopped { 109 | return errReporterClosed 110 | } 111 | name := rep.src.Name() 112 | for _, item := range items { 113 | rep.logf("%s: %s", name, item) 114 | } 115 | return nil 116 | } 117 | 118 | // printfReporter is a FormattedReporter that targets a log formatting function. 119 | type printfReporter struct { 120 | src source.DataSource 121 | printf func(format string, args ...interface{}) (int, error) 122 | stopped bool 123 | } 124 | 125 | // NewPrintfReporter creates a new FormattedReporter around a raw 126 | // fmt.Printf-family formatting function. The formatting function is expected 127 | // to return a number and error in package fmt style. NewPrintfReporter will 128 | // append a newline to passed format strings, since the print formatting 129 | // function is expected to not do so. 130 | func NewPrintfReporter( 131 | src source.DataSource, 132 | printf func(format string, args ...interface{}) (int, error), 133 | ) FormattedReporter { 134 | return &printfReporter{ 135 | src: src, 136 | printf: printf, 137 | } 138 | } 139 | 140 | // Source returns the target source. 141 | func (rep *printfReporter) Source() source.DataSource { 142 | return rep.src 143 | } 144 | 145 | // Start clears any stop flag, and starts watching the data source. 146 | func (rep *printfReporter) Start() error { 147 | var err error 148 | rep.stopped = false 149 | if isrc, ok := rep.src.(source.ItemDataSource); ok { 150 | err = isrc.WatchItems("text", rep) 151 | } else { 152 | err = errRawSource 153 | } 154 | if err != nil { 155 | rep.stopped = true 156 | } 157 | return err 158 | } 159 | 160 | // Stop sets a flag internally so that the next HandleItem(s) will return an 161 | // error, removing the watcher resource. 162 | func (rep *printfReporter) Stop() { 163 | rep.stopped = true 164 | } 165 | 166 | // HandleItem outputs the item to the printf function with a source-name 167 | // prefix and trailing newline. 168 | func (rep *printfReporter) HandleItem(item []byte) error { 169 | if rep.stopped { 170 | return errReporterClosed 171 | } 172 | if _, err := rep.printf("%s: %s\n", rep.src.Name(), item); err != nil { 173 | rep.stopped = true 174 | return err 175 | } 176 | return nil 177 | } 178 | 179 | // HandleItems outputs all items to the logging function with a source-name 180 | // prefix and trailing newline on each item. 181 | func (rep *printfReporter) HandleItems(items [][]byte) error { 182 | if rep.stopped { 183 | return errReporterClosed 184 | } 185 | name := rep.src.Name() 186 | for _, item := range items { 187 | if _, err := rep.printf("%s: %s\n", name, item); err != nil { 188 | rep.stopped = true 189 | return err 190 | } 191 | } 192 | return nil 193 | } 194 | -------------------------------------------------------------------------------- /internal/resp/handler.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package resp 22 | 23 | import ( 24 | "fmt" 25 | "io" 26 | "strings" 27 | ) 28 | 29 | // RedisHandler is the interface used by the connection protocol layer to 30 | // hand-off to application specific code 31 | type RedisHandler interface { 32 | HandleStart(*RedisConnection) error 33 | HandleEnd(*RedisConnection) error 34 | HandleNull(*RedisConnection) error 35 | HandleInteger(*RedisConnection, int) error 36 | HandleString(*RedisConnection, []byte) error 37 | HandleBulkString(*RedisConnection, int, io.Reader) error 38 | HandleArray(*RedisConnection, int) error 39 | HandleError(*RedisConnection, []byte) error 40 | } 41 | 42 | type ValueConsumer struct { 43 | rconn *RedisConnection 44 | seen int 45 | max int 46 | missingErrFormat string 47 | nextVal RedisValue 48 | } 49 | 50 | func NewValueConsumer(rconn *RedisConnection, max int, kind string) *ValueConsumer { 51 | return &ValueConsumer{ 52 | rconn: rconn, 53 | seen: 0, 54 | max: max, 55 | missingErrFormat: strings.Join([]string{"missing %s", kind}, " "), 56 | } 57 | } 58 | 59 | func (vc *ValueConsumer) NumRemaining() int { 60 | if vc.seen > vc.max { 61 | return 0 62 | } 63 | return vc.max - vc.seen 64 | } 65 | 66 | func (vc *ValueConsumer) NumValues() int { 67 | return vc.max 68 | } 69 | 70 | func (vc *ValueConsumer) Consume(name string) (RedisValue, error) { 71 | if vc.seen >= vc.max { 72 | return NilRedisValue, fmt.Errorf(vc.missingErrFormat, name) 73 | } 74 | if err := vc.rconn.Consume(vc); err != nil { 75 | return vc.nextVal, err 76 | } 77 | vc.seen += 1 78 | return vc.nextVal, nil 79 | } 80 | 81 | func (vc *ValueConsumer) HandleNull(_ *RedisConnection) error { 82 | vc.nextVal = NilRedisValue 83 | return nil 84 | } 85 | 86 | func (vc *ValueConsumer) HandleInteger(_ *RedisConnection, num int) error { 87 | vc.nextVal = NewIntRedisValue(num) 88 | return nil 89 | } 90 | 91 | func (vc *ValueConsumer) HandleString(_ *RedisConnection, buf []byte) error { 92 | vc.nextVal = NewBytesRedisValue(buf) 93 | return nil 94 | } 95 | 96 | func (vc *ValueConsumer) HandleBulkString(_ *RedisConnection, n int, r io.Reader) error { 97 | buf := make([]byte, n) 98 | if _, err := io.ReadFull(r, buf); err != nil { 99 | return err 100 | } 101 | vc.nextVal = NewBytesRedisValue(buf) 102 | return nil 103 | } 104 | 105 | func (vc *ValueConsumer) HandleArray(_ *RedisConnection, _ int) error { 106 | return fmt.Errorf("unexpected RESP array, expected a string or integer") 107 | } 108 | 109 | func (vc *ValueConsumer) HandleError(_ *RedisConnection, _ []byte) error { 110 | return fmt.Errorf("unexpected RESP error, expected a string or integer") 111 | } 112 | 113 | func (vc *ValueConsumer) HandleStart(_ *RedisConnection) error { 114 | return nil 115 | } 116 | 117 | func (vc *ValueConsumer) HandleEnd(_ *RedisConnection) error { 118 | return nil 119 | } 120 | 121 | // CmdHandler implements simple command-dispatch: 122 | // - only accepts arrays 123 | // - those arrays must have one or more elements 124 | // - the first element must be a string 125 | type CmdHandler func(*RedisConnection, []byte, *ValueConsumer) error 126 | 127 | func (_ CmdHandler) HandleNull(*RedisConnection) error { 128 | return fmt.Errorf("unexpected RESP null, expected an array") 129 | } 130 | 131 | func (_ CmdHandler) HandleInteger(*RedisConnection, int) error { 132 | return fmt.Errorf("unexpected RESP integer, expected an array") 133 | } 134 | 135 | func (_ CmdHandler) HandleString(*RedisConnection, []byte) error { 136 | return fmt.Errorf("unexpected RESP string, expected an array") 137 | } 138 | 139 | func (_ CmdHandler) HandleBulkString(*RedisConnection, int, io.Reader) error { 140 | return fmt.Errorf("unexpected RESP bulk string, expected an array") 141 | } 142 | 143 | func (f CmdHandler) HandleArray(rconn *RedisConnection, n int) error { 144 | vc := NewValueConsumer(rconn, n, "argument") 145 | 146 | cmdVal, err := vc.Consume("command") 147 | if err != nil { 148 | return err 149 | } 150 | cmd, ok := cmdVal.GetBytes() 151 | if !ok { 152 | return fmt.Errorf("expected command string") 153 | } 154 | 155 | if err := f(rconn, cmd, vc); err != nil { 156 | return err 157 | } 158 | 159 | if vc.NumRemaining() > 0 { 160 | return fmt.Errorf("too many arguments to %v command", string(cmd)) 161 | } 162 | 163 | return nil 164 | } 165 | 166 | func (_ CmdHandler) HandleError(*RedisConnection, []byte) error { 167 | return fmt.Errorf("unexpected RESP error, expected an array") 168 | } 169 | 170 | func (f CmdHandler) HandleStart(rconn *RedisConnection) error { 171 | return f(rconn, []byte("__start__"), nil) 172 | } 173 | 174 | func (f CmdHandler) HandleEnd(rconn *RedisConnection) error { 175 | return f(rconn, []byte("__end__"), nil) 176 | } 177 | 178 | type CmdFunc func(*RedisConnection, *ValueConsumer) error 179 | 180 | type cmdMap map[string]CmdFunc 181 | 182 | func (cmds cmdMap) handleCommand(rconn *RedisConnection, cmdBuf []byte, vc *ValueConsumer) error { 183 | cmd := string(cmdBuf) 184 | cmdFunc, ok := cmds[cmd] 185 | if !ok && !strings.HasPrefix(cmd, "_") { 186 | return rconn.WriteError(fmt.Errorf("unimplemented command %#v", cmd)) 187 | } 188 | if cmdFunc != nil { 189 | return cmdFunc(rconn, vc) 190 | } 191 | return nil 192 | } 193 | 194 | func CmdMapHandler(cmds map[string]CmdFunc) CmdHandler { 195 | return CmdHandler(cmdMap(cmds).handleCommand) 196 | } 197 | -------------------------------------------------------------------------------- /source/tap/tracer_example_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package tap_test 22 | 23 | import ( 24 | "fmt" 25 | "strings" 26 | 27 | "github.com/uber-go/gwr" 28 | "github.com/uber-go/gwr/report" 29 | "github.com/uber-go/gwr/source" 30 | "github.com/uber-go/gwr/source/tap" 31 | ) 32 | 33 | var fibTracer = tap.AddNewTracer("fib") 34 | 35 | // untracedFib is a classic recursive fibonacci computation 36 | func untracedFib(n int) int { 37 | if n < 0 { 38 | return 0 39 | } 40 | if n < 2 { 41 | return 1 42 | } 43 | return untracedFib(n-1) + untracedFib(n-2) 44 | } 45 | 46 | // tracedFib is a modified copy of untracedFib that uses and passes along a 47 | // tracing scope. 48 | // 49 | // For more complex functions, you can call other methods on scope like: 50 | // - Info(...) to emit intermediate data 51 | // - Error(err, ...) to emit any error about to be returned 52 | // - ErrorName("name", err, ...) to further specify a name describing the path 53 | // or cause of the error if it would otherwise be unclear 54 | func tracedFib(n int, scope *tap.TraceScope) (r int) { 55 | scope = scope.Sub("fib").OpenCall(n) 56 | defer func() { scope.CloseCall(r) }() 57 | 58 | if n < 0 { 59 | return 0 60 | } 61 | if n < 2 { 62 | return 1 63 | } 64 | return tracedFib(n-1, scope) + tracedFib(n-2, scope) 65 | } 66 | 67 | // fib is a wrapper that checks if the fibTracer is active (has any watchers) 68 | // and calls either tracedFib or untracedFib accordingly. 69 | // 70 | // This dual-implementation approach is only one option, you could also: 71 | // - choose to pass along a nil scope, and nil-check it throughout a single 72 | // implementation path 73 | // - use Tracer.Scope to always create a scope object, all of its record emits 74 | // will simply go nowhere 75 | func fib(n int) int { 76 | if scope := fibTracer.MaybeScope("wrapper"); scope != nil { 77 | scope.Open(n) 78 | r := tracedFib(n, scope) 79 | scope.CloseCall(r) 80 | return r 81 | } 82 | return untracedFib(n) 83 | } 84 | 85 | func ExampleTracer() { 86 | // this just makes trace ids stable for the test 87 | tap.ResetTraceID() 88 | 89 | // this one won't be traced since there is no watcher yet 90 | fmt.Printf("the 4th fib is %v\n", fib(4)) 91 | 92 | // this causes fibTracer's output to get printed to stdout; the use here is 93 | // more complicated than you'd have in a real program to get stable test 94 | // output. 95 | rep := report.NewPrintfReporter( 96 | gwr.DefaultDataSources.Get("/tap/trace/fib"), 97 | (&timeElider{}).printf) 98 | if err := rep.Start(); err != nil { 99 | panic(err) 100 | } 101 | defer rep.Stop() 102 | 103 | // this one will be traced since there's now a watcher 104 | fmt.Printf("the 5th fib is %v\n", fib(5)) 105 | 106 | // this flushes and stops all watchers on the reported source; otherwise 107 | // this function returns too quickly for even one of the emitted trace 108 | // items to have been printed. 109 | rep.Source().(source.DrainableSource).Drain() 110 | 111 | // this one won't be traced since Drain deactivated the tracer 112 | fmt.Printf("the 6th fib is %v\n", fib(6)) 113 | 114 | // Output: 115 | // the 4th fib is 5 116 | // the 5th fib is 8 117 | // /tap/trace/fib: --> DATE TIME_0 [1::1] wrapper: 5 118 | // /tap/trace/fib: --> DATE TIME_1 [1:1:2] fib(5) 119 | // /tap/trace/fib: --> DATE TIME_2 [1:2:3] fib(4) 120 | // /tap/trace/fib: --> DATE TIME_3 [1:3:4] fib(3) 121 | // /tap/trace/fib: --> DATE TIME_4 [1:4:5] fib(2) 122 | // /tap/trace/fib: --> DATE TIME_5 [1:5:6] fib(1) 123 | // /tap/trace/fib: <-- DATE TIME_6 [1:5:6] return 1 124 | // /tap/trace/fib: --> DATE TIME_7 [1:5:7] fib(0) 125 | // /tap/trace/fib: <-- DATE TIME_8 [1:5:7] return 1 126 | // /tap/trace/fib: <-- DATE TIME_9 [1:4:5] return 2 127 | // /tap/trace/fib: --> DATE TIME_10 [1:4:8] fib(1) 128 | // /tap/trace/fib: <-- DATE TIME_11 [1:4:8] return 1 129 | // /tap/trace/fib: <-- DATE TIME_12 [1:3:4] return 3 130 | // /tap/trace/fib: --> DATE TIME_13 [1:3:9] fib(2) 131 | // /tap/trace/fib: --> DATE TIME_14 [1:9:10] fib(1) 132 | // /tap/trace/fib: <-- DATE TIME_15 [1:9:10] return 1 133 | // /tap/trace/fib: --> DATE TIME_16 [1:9:11] fib(0) 134 | // /tap/trace/fib: <-- DATE TIME_17 [1:9:11] return 1 135 | // /tap/trace/fib: <-- DATE TIME_18 [1:3:9] return 2 136 | // /tap/trace/fib: <-- DATE TIME_19 [1:2:3] return 5 137 | // /tap/trace/fib: --> DATE TIME_20 [1:2:12] fib(3) 138 | // /tap/trace/fib: --> DATE TIME_21 [1:12:13] fib(2) 139 | // /tap/trace/fib: --> DATE TIME_22 [1:13:14] fib(1) 140 | // /tap/trace/fib: <-- DATE TIME_23 [1:13:14] return 1 141 | // /tap/trace/fib: --> DATE TIME_24 [1:13:15] fib(0) 142 | // /tap/trace/fib: <-- DATE TIME_25 [1:13:15] return 1 143 | // /tap/trace/fib: <-- DATE TIME_26 [1:12:13] return 2 144 | // /tap/trace/fib: --> DATE TIME_27 [1:12:16] fib(1) 145 | // /tap/trace/fib: <-- DATE TIME_28 [1:12:16] return 1 146 | // /tap/trace/fib: <-- DATE TIME_29 [1:2:12] return 3 147 | // /tap/trace/fib: <-- DATE TIME_30 [1:1:2] return 8 148 | // /tap/trace/fib: <-- DATE TIME_31 [1::1] return 8 149 | // the 6th fib is 13 150 | } 151 | 152 | // timeElider is here just to normalize output for the test 153 | type timeElider struct { 154 | n int 155 | } 156 | 157 | func (te *timeElider) printf(format string, args ...interface{}) (int, error) { 158 | s := fmt.Sprintf(format, args...) 159 | if fields := strings.Split(s, " "); len(fields) > 6 { 160 | fields[2] = fmt.Sprintf("DATE TIME_%d", te.n) 161 | te.n++ 162 | copy(fields[3:], fields[6:]) 163 | fields = fields[:len(fields)-3] 164 | s = strings.Join(fields, " ") 165 | } 166 | return fmt.Printf(s) 167 | } 168 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package gwr 22 | 23 | import ( 24 | "errors" 25 | "net" 26 | "os" 27 | "sync/atomic" 28 | 29 | "github.com/uber-common/stacked" 30 | ) 31 | 32 | var ( 33 | // ErrAlreadyConfigured is returned by gwr.Configure when called more than 34 | // once. 35 | ErrAlreadyConfigured = errors.New("gwr already configured") 36 | 37 | // ErrAlreadyStarted is returned by ConfiguredServer.Start if the server is 38 | // already listening. 39 | ErrAlreadyStarted = errors.New("gwr server already started") 40 | ) 41 | 42 | // Config defines configuration for GWR. For now this only defines server 43 | // configuration; however once we have reporting support we'll add something 44 | // ReportingCofiguration here. 45 | type Config struct { 46 | // Enabled controls whether GWR is enabled or not, it defaults true. 47 | // Currently this only controls whether ConfiguredServer starting. 48 | Enabled *bool `yaml:"enabled"` 49 | 50 | // ListenAddr controls what address ConfiguredServer will listen on. It is 51 | // superceded by the $GWR_LISTEN environment variable. 52 | // 53 | // If no listen address is set, then GWR does not start its own listening 54 | // server; however GWR can still be accessed under "/gwr/..." from any 55 | // default http servers. 56 | ListenAddr string `yaml:"listen"` 57 | } 58 | 59 | var theServer *ConfiguredServer 60 | 61 | // Configure sets up the gwr library and starts any resources (like a listening 62 | // server) if enabled. 63 | // - if nil config is passed, it's a convenience for &gwr.Config{} 64 | // - if called more than once, ErrAlreadyConfigured is returned 65 | // - otherwise any ConfiguredServer.Start error is returned. 66 | func Configure(config *Config) error { 67 | if theServer != nil { 68 | return ErrAlreadyConfigured 69 | } 70 | if config == nil { 71 | config = &Config{} 72 | } 73 | theServer = NewConfiguredServer(*config) 74 | return theServer.Start() 75 | } 76 | 77 | // Enabled returns true if the gwr library is configured and enabled. 78 | func Enabled() bool { 79 | if theServer == nil { 80 | return false 81 | } 82 | return theServer.Enabled() 83 | } 84 | 85 | // DefaultServer returns the configured gwr server, or nil if Configure 86 | // hasn't been called yet. 87 | func DefaultServer() *ConfiguredServer { 88 | return theServer 89 | } 90 | 91 | type serverConfig struct { 92 | enabled bool 93 | listenAddr string 94 | } 95 | 96 | var defaultServerConfig = serverConfig{ 97 | enabled: true, 98 | listenAddr: "", 99 | } 100 | 101 | // ConfiguredServer manages the lifecycle of a configured GWR server, as 102 | // created by gwr.NewServer. 103 | type ConfiguredServer struct { 104 | config serverConfig 105 | stacked stacked.Server 106 | ln net.Listener 107 | stopping uint32 108 | done chan error 109 | } 110 | 111 | // NewConfiguredServer creates a new ConfiguredServer for a given config. 112 | func NewConfiguredServer(cfg Config) *ConfiguredServer { 113 | srv := &ConfiguredServer{ 114 | config: defaultServerConfig, 115 | stacked: NewServer(DefaultDataSources), 116 | } 117 | 118 | if cfg.Enabled != nil { 119 | srv.config.enabled = *cfg.Enabled 120 | } 121 | 122 | if envListen := os.Getenv("GWR_LISTEN"); envListen != "" { 123 | srv.config.listenAddr = envListen 124 | } else if cfg.ListenAddr != "" { 125 | srv.config.listenAddr = cfg.ListenAddr 126 | } 127 | 128 | return srv 129 | } 130 | 131 | // Enabled return true if the server is enabled. 132 | func (srv *ConfiguredServer) Enabled() bool { 133 | return srv.config.enabled 134 | } 135 | 136 | // ListenAddr returns the configured listen address string. 137 | func (srv *ConfiguredServer) ListenAddr() string { 138 | return srv.config.listenAddr 139 | } 140 | 141 | // Addr returns the current listening address, if any. 142 | func (srv *ConfiguredServer) Addr() net.Addr { 143 | if srv.ln == nil { 144 | return nil 145 | } 146 | return srv.ln.Addr() 147 | } 148 | 149 | // Start starts the server by creating the listener and a server goroutine to 150 | // accept connections. 151 | // - if not enabled, or if no listen address is configured, noops and returns 152 | // nil 153 | // - if already listening, returns ErrAlreadyStarted 154 | // - otherwise any net.Listen error is returned. 155 | func (srv *ConfiguredServer) Start() error { 156 | if !srv.config.enabled { 157 | return nil 158 | } 159 | 160 | if srv.config.listenAddr == "" { 161 | return nil 162 | } 163 | 164 | if srv.ln != nil { 165 | return ErrAlreadyStarted 166 | } 167 | 168 | ln, err := net.Listen("tcp", srv.config.listenAddr) 169 | if err != nil { 170 | return err 171 | } 172 | 173 | srv.ln = ln 174 | srv.done = make(chan error, 1) 175 | go func(ln net.Listener, done chan<- error) { 176 | err := srv.stacked.Serve(ln) 177 | if atomic.LoadUint32(&srv.stopping) == 0 { 178 | done <- err 179 | } else { 180 | done <- nil 181 | } 182 | }(srv.ln, srv.done) 183 | return nil 184 | } 185 | 186 | // StartOn starts the server on a given listening address. If the start 187 | // succeeds, it also updates the configured listening address for later 188 | // reference. It has all the same error cases as ConfiguredServer.Start. 189 | func (srv *ConfiguredServer) StartOn(laddr string) error { 190 | if !srv.config.enabled { 191 | return nil 192 | } 193 | 194 | if srv.ln != nil { 195 | return ErrAlreadyStarted 196 | } 197 | 198 | oldLaddr := srv.config.listenAddr 199 | srv.config.listenAddr = laddr 200 | err := srv.Start() 201 | if err != nil { 202 | srv.config.listenAddr = oldLaddr 203 | } 204 | return err 205 | } 206 | 207 | // Stop closes the current listener and shuts down the server goroutine started 208 | // by Start (if any). 209 | func (srv *ConfiguredServer) Stop() error { 210 | if srv.ln == nil { 211 | return nil 212 | } 213 | if !atomic.CompareAndSwapUint32(&srv.stopping, 0, 1) { 214 | return nil 215 | } 216 | ln, done := srv.ln, srv.done 217 | srv.ln, srv.done = nil, nil 218 | err := ln.Close() 219 | if serveErr := <-done; err == nil && serveErr != nil { 220 | err = serveErr 221 | } 222 | atomic.CompareAndSwapUint32(&srv.stopping, 1, 0) 223 | return err 224 | } 225 | -------------------------------------------------------------------------------- /internal/protocol/http_rest.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package protocol 22 | 23 | import ( 24 | "bytes" 25 | "fmt" 26 | "io" 27 | "log" 28 | "net" 29 | "net/http" 30 | "strings" 31 | 32 | "github.com/uber-go/gwr/internal/meta" 33 | "github.com/uber-go/gwr/source" 34 | ) 35 | 36 | // Servable is a minimal server interface supported by the "/listen" facility. 37 | type Servable interface { 38 | Addr() net.Addr 39 | StartOn(string) error 40 | Stop() error 41 | } 42 | 43 | var formatContetTypes = map[string]string{ 44 | "json": "application/json", 45 | "text": "text/plain", 46 | "html": "text/html", 47 | } 48 | 49 | func contentTypeFor(formatName string) string { 50 | if contetType, ok := formatContetTypes[formatName]; ok { 51 | return contetType 52 | } 53 | return "application/octet" 54 | } 55 | 56 | // HTTPRest implements http.Handler to host a collection of data sources 57 | // REST-fully. 58 | type HTTPRest struct { 59 | defaultFormats []string 60 | prefix string 61 | dss *source.DataSources 62 | srv Servable 63 | } 64 | 65 | // NewHTTPRest returns an http.Handler to host the data sources REST-fully at a 66 | // given prefix. 67 | // 68 | // If a non-nil servable is passed, then a /listen convenience endpoint will be 69 | // provided to afford server discovery and lifecycle management. 70 | func NewHTTPRest(dss *source.DataSources, prefix string, srv Servable) *HTTPRest { 71 | return &HTTPRest{ 72 | defaultFormats: []string{"text", "json"}, 73 | prefix: prefix, 74 | dss: dss, 75 | srv: srv, 76 | } 77 | } 78 | 79 | func (hndl *HTTPRest) ServeHTTP(w http.ResponseWriter, r *http.Request) { 80 | if err := hndl.routeSource(w, r); err != nil { 81 | http.Error(w, "500 Internal Server Error", http.StatusInternalServerError) 82 | log.Printf("data source serve failed: %v\n", err) 83 | // XXX log 84 | return 85 | } 86 | } 87 | 88 | func (hndl *HTTPRest) doListen(w http.ResponseWriter, r *http.Request) error { 89 | // TODO: this could be "just" another meta source, if sources had a way to 90 | // define custom actions, e.g. to tell it to go listen 91 | switch strings.ToLower(r.Method) { 92 | case "get": 93 | addr := hndl.srv.Addr() 94 | if addr == nil { 95 | http.Error(w, 96 | "503 Not Listening\nServer not started, POST an address to start it.", 97 | http.StatusServiceUnavailable) 98 | return nil 99 | } 100 | io.WriteString(w, fmt.Sprintf("%v\n", addr)) 101 | 102 | case "post": 103 | if strings.HasPrefix(r.Header.Get("Content-Type"), "multipart/form-data") { 104 | if err := r.ParseMultipartForm(1024); err != nil { 105 | return err 106 | } 107 | } else if err := r.ParseForm(); err != nil { 108 | return err 109 | } 110 | 111 | if r.Form.Get("stop") != "" { 112 | if hndl.srv.Addr() == nil { 113 | io.WriteString(w, "not running\n") 114 | } else if err := hndl.srv.Stop(); err != nil { 115 | io.WriteString(w, err.Error()) 116 | } else { 117 | io.WriteString(w, "stopped\n") 118 | } 119 | return nil 120 | } 121 | 122 | if laddr := r.Form.Get("address"); laddr == "" { 123 | http.Error(w, "400 Missing \"address\" form value.", http.StatusBadRequest) 124 | return nil 125 | } else if err := hndl.srv.StartOn(laddr); err != nil { 126 | http.Error(w, 127 | fmt.Sprintf("503 Unable to start server\nstart failed: %s", err.Error()), 128 | http.StatusServiceUnavailable) 129 | return nil 130 | } 131 | 132 | w.WriteHeader(http.StatusCreated) 133 | io.WriteString(w, fmt.Sprintf("%v\n", hndl.srv.Addr())) 134 | 135 | default: 136 | w.Header().Set("Allow", "GET, POST") 137 | w.WriteHeader(http.StatusMethodNotAllowed) 138 | io.WriteString(w, "405 Invalid Method\n") 139 | } 140 | 141 | return nil 142 | } 143 | 144 | func (hndl *HTTPRest) routeSource(w http.ResponseWriter, r *http.Request) error { 145 | path := r.URL.Path[len(hndl.prefix):] 146 | if hndl.srv != nil && path == "/listen" { 147 | return hndl.doListen(w, r) 148 | } 149 | 150 | var src source.DataSource 151 | if len(path) == 0 || path == "/" { 152 | src = hndl.dss.Get(meta.NounsName) 153 | } else { 154 | src = hndl.dss.Get(path) 155 | } 156 | if src == nil { 157 | http.NotFound(w, r) 158 | return nil 159 | } 160 | return hndl.routeVerb(src, w, r) 161 | } 162 | 163 | func (hndl *HTTPRest) routeVerb( 164 | src source.DataSource, 165 | w http.ResponseWriter, 166 | r *http.Request, 167 | ) error { 168 | if err := r.ParseForm(); err != nil { 169 | return err 170 | } 171 | 172 | switch strings.ToLower(r.Method) { 173 | case "get": 174 | if r.Form.Get("watch") != "" { 175 | // convenience for http clients that don't easily support custom 176 | // method strings 177 | return hndl.doWatch(src, w, r) 178 | } 179 | return hndl.doGet(src, w, r) 180 | 181 | case "watch": 182 | return hndl.doWatch(src, w, r) 183 | 184 | default: 185 | w.Header().Set("Allow", "GET, WATCH") 186 | w.WriteHeader(http.StatusMethodNotAllowed) 187 | io.WriteString(w, "405 Invalid Method\n") 188 | } 189 | return nil 190 | } 191 | 192 | func (hndl *HTTPRest) doGet( 193 | src source.DataSource, 194 | w http.ResponseWriter, 195 | r *http.Request, 196 | ) error { 197 | formatName, err := hndl.determineFormat(src, w, r) 198 | if len(formatName) == 0 || err != nil { 199 | return err 200 | } 201 | 202 | var buf bytes.Buffer 203 | if err := src.Get(formatName, &buf); err == source.ErrNotGetable { 204 | http.Error(w, "501 source does not support Get", http.StatusNotImplemented) 205 | return nil 206 | } else if err != nil { 207 | return err 208 | } 209 | 210 | if contetType, ok := formatContetTypes[formatName]; ok { 211 | w.Header().Set("Content-Type", contetType) 212 | } else { 213 | w.Header().Set("Content-Type", "application/octet") 214 | } 215 | 216 | w.WriteHeader(http.StatusOK) 217 | _, err = buf.WriteTo(w) 218 | return err 219 | } 220 | 221 | type flushWriter struct { 222 | w io.Writer 223 | f http.Flusher 224 | } 225 | 226 | func (fw *flushWriter) Write(p []byte) (int, error) { 227 | n, err := fw.w.Write(p) 228 | fw.f.Flush() 229 | return n, err 230 | } 231 | 232 | func (hndl *HTTPRest) doWatch( 233 | src source.DataSource, 234 | w http.ResponseWriter, 235 | r *http.Request, 236 | ) error { 237 | formatName, err := hndl.determineFormat(src, w, r) 238 | if len(formatName) == 0 || err != nil { 239 | return err 240 | } 241 | 242 | ready := make(chan *chanBuf, 1) 243 | var buf = chanBuf{ready: ready} 244 | defer buf.Close() 245 | 246 | if err := src.Watch(formatName, &buf); err == source.ErrNotWatchable { 247 | http.Error(w, "501 source does not support Watch", http.StatusNotImplemented) 248 | return nil 249 | } else if err != nil { 250 | return err 251 | } 252 | 253 | w.Header().Set("Content-Type", contentTypeFor(formatName)) 254 | w.Header().Set("Transfer-Encoding", "chunked") 255 | 256 | w.WriteHeader(http.StatusOK) 257 | 258 | var fw io.Writer = w 259 | 260 | if f, _ := w.(http.Flusher); f != nil { 261 | f.Flush() 262 | fw = &flushWriter{w, f} 263 | } 264 | 265 | var cn <-chan bool 266 | if cnr, ok := w.(http.CloseNotifier); ok { 267 | cn = cnr.CloseNotify() 268 | } 269 | 270 | for { 271 | select { 272 | case <-ready: 273 | if _, err := buf.writeTo(fw); err != nil { 274 | return err 275 | } 276 | case <-cn: 277 | // TODO: don't get this, why 278 | return nil 279 | } 280 | } 281 | } 282 | 283 | func (hndl *HTTPRest) determineFormat( 284 | src source.DataSource, 285 | w http.ResponseWriter, 286 | r *http.Request, 287 | ) (string, error) { 288 | // TODO: some people like Accepts negotiation 289 | 290 | formats := src.Formats() 291 | 292 | formatName := r.Form.Get("format") 293 | if len(formatName) != 0 { 294 | for _, availFormat := range formats { 295 | if strings.EqualFold(formatName, availFormat) { 296 | return availFormat, nil 297 | } 298 | } 299 | w.WriteHeader(http.StatusBadRequest) 300 | io.WriteString(w, "400 Bad Request\nUnsupported Format\n") 301 | return "", nil 302 | } 303 | 304 | for _, defaultFormat := range hndl.defaultFormats { 305 | for _, availFormat := range formats { 306 | if strings.EqualFold(availFormat, defaultFormat) { 307 | return availFormat, nil 308 | } 309 | } 310 | } 311 | 312 | return formats[0], nil 313 | } 314 | -------------------------------------------------------------------------------- /internal/marshaled/watcher.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package marshaled 22 | 23 | import ( 24 | "errors" 25 | "io" 26 | "log" 27 | "sync" 28 | 29 | "github.com/uber-go/gwr/internal" 30 | "github.com/uber-go/gwr/source" 31 | ) 32 | 33 | var errDefaultFrameWatcherDone = errors.New("all defaultFrameWatcher writers done") 34 | 35 | // marshaledWatcher manages all of the low level io.Writers for a given format. 36 | // Instances are created once for each DataSource. 37 | // 38 | // DataSource then manages calling marshaledWatcher.emit for each data item as 39 | // long as there is one valid io.Writer for a given format. Once the last 40 | // marshaledWatcher goes idle, the underlying GenericDataSource watch is ended. 41 | type marshaledWatcher struct { 42 | source *DataSource 43 | format source.GenericDataFormat 44 | dfw defaultFrameWatcher 45 | watchers []source.ItemWatcher 46 | } 47 | 48 | func newMarshaledWatcher(src *DataSource, format source.GenericDataFormat) *marshaledWatcher { 49 | mw := &marshaledWatcher{source: src, format: format} 50 | mw.dfw.format = format 51 | return mw 52 | } 53 | 54 | func (mw *marshaledWatcher) Close() error { 55 | var errs []error 56 | for _, watcher := range mw.watchers { 57 | if closer, ok := watcher.(io.Closer); ok { 58 | if err := closer.Close(); err != nil { 59 | if errs == nil { 60 | errs = make([]error, 0, len(mw.watchers)) 61 | } 62 | errs = append(errs, err) 63 | } 64 | } 65 | } 66 | mw.watchers = mw.watchers[:0] 67 | return internal.MultiErr(errs).AsError() 68 | } 69 | 70 | func (mw *marshaledWatcher) init(w io.Writer) error { 71 | if mw.source.watiSource != nil { 72 | initData := mw.source.watiSource.WatchInit() 73 | if err := mw.dfw.writeInitData(initData, w); err != nil { 74 | return err 75 | } 76 | } 77 | mw.dfw.writers = append(mw.dfw.writers, w) 78 | if len(mw.dfw.writers) == 1 { 79 | mw.watchers = append(mw.watchers, &mw.dfw) 80 | } 81 | return nil 82 | } 83 | 84 | func (mw *marshaledWatcher) initItems(iw source.ItemWatcher) error { 85 | if mw.source.watiSource != nil { 86 | initData := mw.source.watiSource.WatchInit() 87 | if buf, err := mw.format.MarshalInit(initData); err != nil { 88 | log.Printf("initial marshaling error %v", err) 89 | return err 90 | } else if err := iw.HandleItem(buf); err != nil { 91 | return err 92 | } 93 | } 94 | mw.watchers = append(mw.watchers, iw) 95 | return nil 96 | } 97 | 98 | func (mw *marshaledWatcher) emit(item interface{}) bool { 99 | if len(mw.watchers) == 0 { 100 | return false 101 | } 102 | data, err := mw.format.MarshalItem(item) 103 | if err != nil { 104 | log.Printf("item marshaling error %v", err) 105 | return false 106 | } 107 | 108 | var failed []int // TODO: could carry this rather than allocate on failure 109 | for i, iw := range mw.watchers { 110 | if err := iw.HandleItem(data); err != nil { 111 | if failed == nil { 112 | failed = make([]int, 0, len(mw.watchers)) 113 | } 114 | failed = append(failed, i) 115 | } 116 | } 117 | if len(failed) == 0 { 118 | return true 119 | } 120 | 121 | var ( 122 | okay []source.ItemWatcher 123 | remain = len(mw.watchers) - len(failed) 124 | ) 125 | if remain > 0 { 126 | okay = make([]source.ItemWatcher, 0, remain) 127 | } 128 | for i, iw := range mw.watchers { 129 | if i != failed[0] { 130 | okay = append(okay, iw) 131 | } 132 | if i >= failed[0] { 133 | failed = failed[1:] 134 | if len(failed) == 0 { 135 | if j := i + 1; j < len(mw.watchers) { 136 | okay = append(okay, mw.watchers[j:]...) 137 | } 138 | break 139 | } 140 | } 141 | } 142 | mw.watchers = okay 143 | 144 | return len(mw.watchers) != 0 145 | } 146 | 147 | func (mw *marshaledWatcher) emitBatch(items []interface{}) bool { 148 | if len(mw.watchers) == 0 { 149 | return false 150 | } 151 | 152 | data := make([][]byte, len(items)) 153 | for i, item := range items { 154 | buf, err := mw.format.MarshalItem(item) 155 | if err != nil { 156 | log.Printf("item marshaling error %v", err) 157 | return false 158 | } 159 | data[i] = buf 160 | } 161 | 162 | var failed []int // TODO: could carry this rather than allocate on failure 163 | for i, iw := range mw.watchers { 164 | if err := iw.HandleItems(data); err != nil { 165 | if failed == nil { 166 | failed = make([]int, 0, len(mw.watchers)) 167 | } 168 | failed = append(failed, i) 169 | } 170 | } 171 | if len(failed) == 0 { 172 | return true 173 | } 174 | 175 | var ( 176 | okay []source.ItemWatcher 177 | remain = len(mw.watchers) - len(failed) 178 | ) 179 | if remain > 0 { 180 | okay = make([]source.ItemWatcher, 0, remain) 181 | } 182 | for i, iw := range mw.watchers { 183 | if i != failed[0] { 184 | okay = append(okay, iw) 185 | } 186 | if i >= failed[0] { 187 | failed = failed[1:] 188 | if len(failed) == 0 { 189 | if j := i + 1; j < len(mw.watchers) { 190 | okay = append(okay, mw.watchers[j:]...) 191 | } 192 | break 193 | } 194 | } 195 | } 196 | mw.watchers = okay 197 | 198 | return len(mw.watchers) != 0 199 | } 200 | 201 | type defaultFrameWatcher struct { 202 | sync.Mutex 203 | format source.GenericDataFormat 204 | writers []io.Writer 205 | } 206 | 207 | func (dfw *defaultFrameWatcher) writeInitData(data interface{}, w io.Writer) error { 208 | buf, err := dfw.format.MarshalInit(data) 209 | if err != nil { 210 | log.Printf("initial marshaling error %v", err) 211 | return err 212 | } 213 | buf, err = dfw.format.FrameItem(buf) 214 | if err != nil { 215 | log.Printf("initial framing error %v", err) 216 | return err 217 | } 218 | if _, err := w.Write(buf); err != nil { 219 | return err 220 | } 221 | return nil 222 | } 223 | 224 | func (dfw *defaultFrameWatcher) HandleItem(item []byte) error { 225 | if len(dfw.writers) == 0 { 226 | return errDefaultFrameWatcherDone 227 | } 228 | buf, err := dfw.format.FrameItem(item) 229 | if err != nil { 230 | log.Printf("item framing error %v", err) 231 | return err 232 | } 233 | if err := dfw.writeToAll(buf); err != nil { 234 | return err 235 | } 236 | return nil 237 | } 238 | 239 | func (dfw *defaultFrameWatcher) HandleItems(items [][]byte) error { 240 | if len(dfw.writers) == 0 { 241 | return errDefaultFrameWatcherDone 242 | } 243 | for _, item := range items { 244 | buf, err := dfw.format.FrameItem(item) 245 | if err != nil { 246 | log.Printf("item framing error %v", err) 247 | return err 248 | } 249 | if err := dfw.writeToAll(buf); err != nil { 250 | return err 251 | } 252 | } 253 | return nil 254 | } 255 | 256 | func (dfw *defaultFrameWatcher) Close() error { 257 | dfw.Lock() 258 | writers := dfw.writers 259 | dfw.writers = nil 260 | dfw.Unlock() 261 | 262 | var errs []error 263 | for _, writer := range writers { 264 | if closer, ok := writer.(io.Closer); ok { 265 | if err := closer.Close(); err != nil { 266 | if errs == nil { 267 | errs = make([]error, 0, len(writers)) 268 | } 269 | errs = append(errs, err) 270 | } 271 | } 272 | } 273 | 274 | return internal.MultiErr(errs).AsError() 275 | } 276 | 277 | func (dfw *defaultFrameWatcher) writeToAll(buf []byte) error { 278 | // TODO: avoid blocking fan out, parallelize; error back-propagation then 279 | // needs to happen over another channel 280 | 281 | dfw.Lock() 282 | defer dfw.Unlock() 283 | 284 | var failed []int // TODO: could carry this rather than allocate on failure 285 | for i, w := range dfw.writers { 286 | if _, err := w.Write(buf); err != nil { 287 | if failed == nil { 288 | failed = make([]int, 0, len(dfw.writers)) 289 | } 290 | failed = append(failed, i) 291 | } 292 | } 293 | if len(failed) == 0 { 294 | return nil 295 | } 296 | 297 | var ( 298 | okay []io.Writer 299 | remain = len(dfw.writers) - len(failed) 300 | ) 301 | if remain > 0 { 302 | okay = make([]io.Writer, 0, remain) 303 | } 304 | for i, w := range dfw.writers { 305 | if i != failed[0] { 306 | okay = append(okay, w) 307 | } 308 | if i >= failed[0] { 309 | failed = failed[1:] 310 | if len(failed) == 0 { 311 | if j := i + 1; j < len(dfw.writers) { 312 | okay = append(okay, dfw.writers[j:]...) 313 | } 314 | break 315 | } 316 | } 317 | } 318 | dfw.writers = okay 319 | 320 | if len(dfw.writers) == 0 { 321 | return errDefaultFrameWatcherDone 322 | } 323 | return nil 324 | } 325 | -------------------------------------------------------------------------------- /example_access_log_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package gwr_test 22 | 23 | import ( 24 | "bufio" 25 | "fmt" 26 | "io" 27 | "log" 28 | "net" 29 | "net/http" 30 | "net/http/httptest" 31 | "os" 32 | "text/template" 33 | 34 | gwr "github.com/uber-go/gwr" 35 | "github.com/uber-go/gwr/source" 36 | ) 37 | 38 | type accessLogger struct { 39 | handler http.Handler 40 | watcher source.GenericDataWatcher 41 | } 42 | 43 | func logged(handler http.Handler) *accessLogger { 44 | if handler == nil { 45 | handler = http.DefaultServeMux 46 | } 47 | return &accessLogger{ 48 | handler: handler, 49 | } 50 | } 51 | 52 | type accessEntry struct { 53 | Method string `json:"method"` 54 | Path string `json:"path"` 55 | Query string `json:"query"` 56 | Code int `json:"code"` 57 | Bytes int `json:"bytes"` 58 | ContentType string `json:"content_type"` 59 | } 60 | 61 | var accessLogTextTemplate = template.Must(template.New("req_logger_text").Parse(` 62 | {{ define "item" }}{{ .Method }} {{ .Path }}{{ if .Query }}?{{ .Query }}{{ end }} {{ .Code }} {{ .Bytes }} {{ .ContentType }}{{ end }} 63 | `)) 64 | 65 | func (al *accessLogger) ServeHTTP(w http.ResponseWriter, r *http.Request) { 66 | if !al.watcher.Active() { 67 | al.handler.ServeHTTP(w, r) 68 | return 69 | } 70 | 71 | rec := httptest.NewRecorder() 72 | al.handler.ServeHTTP(rec, r) 73 | bytes := rec.Body.Len() 74 | 75 | // finishing work first... 76 | hdr := w.Header() 77 | for key, vals := range rec.HeaderMap { 78 | hdr[key] = vals 79 | } 80 | w.WriteHeader(rec.Code) 81 | if _, err := rec.Body.WriteTo(w); err != nil { 82 | http.Error(w, err.Error(), http.StatusInternalServerError) 83 | } 84 | 85 | // ...then emitting the entry may afford slightly less overhead; most 86 | // overhead, like marshalling, should be deferred by the gwr library, 87 | // watcher.HandleItem is supposed to be fast enough to not need a channel 88 | // indirection within each source. 89 | al.watcher.HandleItem(accessEntry{ 90 | Method: r.Method, 91 | Path: r.URL.Path, 92 | Query: r.URL.RawQuery, 93 | Code: rec.Code, 94 | Bytes: bytes, 95 | ContentType: rec.HeaderMap.Get("Content-Type"), 96 | }) 97 | } 98 | 99 | func (al *accessLogger) Name() string { 100 | return "/access_log" 101 | } 102 | 103 | func (al *accessLogger) TextTemplate() *template.Template { 104 | return accessLogTextTemplate 105 | } 106 | 107 | func (al *accessLogger) SetWatcher(watcher source.GenericDataWatcher) { 108 | al.watcher = watcher 109 | } 110 | 111 | // TODO: this has become more test than example; maybe just make it a test? 112 | 113 | func Example_httpserver_accesslog() { 114 | // Uses :0 for no conflict in test. 115 | if err := gwr.Configure(&gwr.Config{ListenAddr: ":0"}); err != nil { 116 | log.Fatal(err) 117 | } 118 | defer gwr.DefaultServer().Stop() 119 | gwrAddr := gwr.DefaultServer().Addr() 120 | 121 | // a handler so we get more than just 404s 122 | http.HandleFunc("/foo", func(w http.ResponseWriter, r *http.Request) { 123 | w.Header().Set("Content-Type", "text/plain; charset=utf-8") 124 | if _, err := io.WriteString(w, "Ok ;-)\n"); err != nil { 125 | panic(err.Error()) 126 | } 127 | }) 128 | 129 | // wraps the default serve mux in an access logging gwr source... 130 | loggedHTTPHandler := logged(nil) 131 | 132 | // ...which we then register with gwr 133 | gwr.AddGenericDataSource(loggedHTTPHandler) 134 | 135 | // Again note the :0 pattern complicates things more than normal; this is 136 | // just the default http server 137 | ln, err := net.Listen("tcp", ":0") 138 | if err != nil { 139 | log.Fatal(err) 140 | } 141 | svcAddr := ln.Addr() 142 | go http.Serve(ln, loggedHTTPHandler) 143 | 144 | // make two requests, one should get a 200, the other a 404; we should see 145 | // only their output since no watchers are going yet 146 | fmt.Println("start") 147 | httpGetStdout("http://%s/foo?num=1", svcAddr) 148 | httpGetStdout("http://%s/bar?num=2", svcAddr) 149 | 150 | // start a json watch on the access log; NOTE we don't care about any copy 151 | // error because normal termination here is closed-mid-read; TODO we could 152 | // tighten this to log fatal any non "closed" error 153 | jsonLines := newHTTPGetChan("JSON", "http://%s/access_log?format=json&watch=1", gwrAddr) 154 | fmt.Println("\nwatching json") 155 | 156 | // make two requests, now with json watcher 157 | httpGetStdout("http://%s/foo?num=3", svcAddr) 158 | httpGetStdout("http://%s/bar?num=4", svcAddr) 159 | jsonLines.printOne() 160 | jsonLines.printOne() 161 | 162 | // start a text watch on the access log 163 | textLines := newHTTPGetChan("TEXT", "http://%s/access_log?format=text&watch=1", gwrAddr) 164 | fmt.Println("\nwatching text & json") 165 | 166 | // make two requests, now with both watchers 167 | httpGetStdout("http://%s/foo?num=5", svcAddr) 168 | httpGetStdout("http://%s/bar?num=6", svcAddr) 169 | jsonLines.printOne() 170 | jsonLines.printOne() 171 | textLines.printOne() 172 | textLines.printOne() 173 | 174 | // shutdown the json watch 175 | jsonLines.close() 176 | fmt.Println("\njust text") 177 | 178 | // make two requests; we should still see the body copies, but only get the text watch data 179 | httpGetStdout("http://%s/foo?num=7", svcAddr) 180 | httpGetStdout("http://%s/bar?num=8", svcAddr) 181 | textLines.printOne() 182 | textLines.printOne() 183 | 184 | // shutdown the json watch 185 | textLines.close() 186 | fmt.Println("\nno watchers") 187 | 188 | // make two requests; we should still see the body copies, but get no watch data for them 189 | httpGetStdout("http://%s/foo?num=9", svcAddr) 190 | httpGetStdout("http://%s/bar?num=10", svcAddr) 191 | 192 | // output: start 193 | // Ok ;-) 194 | // 404 page not found 195 | // 196 | // watching json 197 | // Ok ;-) 198 | // 404 page not found 199 | // JSON: {"method":"GET","path":"/foo","query":"num=3","code":200,"bytes":7,"content_type":"text/plain; charset=utf-8"} 200 | // JSON: {"method":"GET","path":"/bar","query":"num=4","code":404,"bytes":19,"content_type":"text/plain; charset=utf-8"} 201 | // 202 | // watching text & json 203 | // Ok ;-) 204 | // 404 page not found 205 | // JSON: {"method":"GET","path":"/foo","query":"num=5","code":200,"bytes":7,"content_type":"text/plain; charset=utf-8"} 206 | // JSON: {"method":"GET","path":"/bar","query":"num=6","code":404,"bytes":19,"content_type":"text/plain; charset=utf-8"} 207 | // TEXT: GET /foo?num=5 200 7 text/plain; charset=utf-8 208 | // TEXT: GET /bar?num=6 404 19 text/plain; charset=utf-8 209 | // JSON: CLOSE 210 | // 211 | // just text 212 | // Ok ;-) 213 | // 404 page not found 214 | // TEXT: GET /foo?num=7 200 7 text/plain; charset=utf-8 215 | // TEXT: GET /bar?num=8 404 19 text/plain; charset=utf-8 216 | // TEXT: CLOSE 217 | // 218 | // no watchers 219 | // Ok ;-) 220 | // 404 page not found 221 | } 222 | 223 | // test brevity conveniences 224 | 225 | func httpGetStdout(format string, args ...interface{}) { 226 | url := fmt.Sprintf(format, args...) 227 | resp, err := http.Get(url) 228 | if err != nil { 229 | log.Fatal(err) 230 | } 231 | if _, err := io.Copy(os.Stdout, resp.Body); err != nil { 232 | log.Fatal(err) 233 | } 234 | if err := resp.Body.Close(); err != nil { 235 | log.Fatal(err) 236 | } 237 | } 238 | 239 | type httpGetChan struct { 240 | tag string 241 | c chan string 242 | r *http.Response 243 | } 244 | 245 | func newHTTPGetChan(tag string, format string, args ...interface{}) *httpGetChan { 246 | url := fmt.Sprintf(format, args...) 247 | resp, err := http.Get(url) 248 | if err != nil { 249 | log.Fatal(err) 250 | } 251 | hc := &httpGetChan{ 252 | tag: tag, 253 | c: make(chan string), 254 | r: resp, 255 | } 256 | go hc.scanLines() 257 | return hc 258 | } 259 | 260 | func (hc *httpGetChan) printOne() { 261 | if _, err := fmt.Printf("%s: %s\n", hc.tag, <-hc.c); err != nil { 262 | log.Fatal(err) 263 | } 264 | } 265 | 266 | func (hc *httpGetChan) close() { 267 | if err := hc.r.Body.Close(); err != nil { 268 | log.Fatal(err) 269 | } 270 | hc.printOne() 271 | } 272 | 273 | func (hc *httpGetChan) scanLines() { 274 | s := bufio.NewScanner(hc.r.Body) 275 | for s.Scan() { 276 | hc.c <- s.Text() 277 | } 278 | hc.c <- "CLOSE" 279 | close(hc.c) 280 | // NOTE: s.Err() intentionally not checking since this is a "just" 281 | // function; in particular, we expect to get a closed-during-read error 282 | } 283 | -------------------------------------------------------------------------------- /internal/resp/connection.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package resp 22 | 23 | import ( 24 | "bufio" 25 | "fmt" 26 | "io" 27 | "net" 28 | ) 29 | 30 | // RedisConnection is the protocol reading and writing layer 31 | type RedisConnection struct { 32 | Conn net.Conn 33 | reader *bufio.Reader 34 | // TODO: use a bufio.Writer too 35 | } 36 | 37 | // NewRedisConnection creates a redis connection around an existing net.Conn 38 | // and an optional reader (e.g. a bufio.Reader that's already been created 39 | // around the net.Conn). If r is nill, conn is used instead. 40 | func NewRedisConnection(conn net.Conn, r io.Reader) *RedisConnection { 41 | if r == nil { 42 | r = conn 43 | } 44 | return &RedisConnection{ 45 | Conn: conn, 46 | reader: bufio.NewReader(r), 47 | } 48 | } 49 | 50 | // Close closes the underlying connection. 51 | func (rconn *RedisConnection) Close() error { 52 | return rconn.Conn.Close() 53 | } 54 | 55 | // Handle runs the passed handler until the connection ends or errors. 56 | func (rconn *RedisConnection) Handle(handler RedisHandler) { 57 | if err := rconn.handle(handler); err != nil { 58 | fmt.Printf("redis handler error from %v: %v\n", rconn.Conn.RemoteAddr(), err) 59 | } 60 | if err := rconn.Close(); err != nil { 61 | fmt.Printf("error closing connection from %v: %v\n", rconn.Conn.RemoteAddr(), err) 62 | } 63 | } 64 | 65 | func (rconn *RedisConnection) handle(handler RedisHandler) error { 66 | if err := handler.HandleStart(rconn); err != nil { 67 | return err 68 | } 69 | 70 | for { 71 | err := rconn.Consume(handler) 72 | if err != nil { 73 | if err != io.EOF { 74 | rconn.WriteError(err) 75 | } 76 | break 77 | } 78 | } 79 | 80 | return handler.HandleEnd(rconn) 81 | } 82 | 83 | // Consume reads one element from the connection and passes it to the given handler. 84 | func (rconn *RedisConnection) Consume(handler RedisHandler) error { 85 | c, err := rconn.reader.ReadByte() 86 | if err != nil { 87 | return err 88 | } 89 | 90 | switch c { 91 | case '-': 92 | return rconn.consumeError(handler) 93 | 94 | case ':': 95 | return rconn.consumeInteger(handler) 96 | 97 | case '+': 98 | return rconn.consumeShortString(handler) 99 | 100 | case '$': 101 | return rconn.consumeBulkString(handler) 102 | 103 | case '*': 104 | return rconn.consumeArray(handler) 105 | 106 | default: 107 | return fmt.Errorf("unknown RESP type %#v", string(c)) 108 | } 109 | } 110 | 111 | func (rconn *RedisConnection) consumeError(handler RedisHandler) error { 112 | buf, err := rconn.scanLine() 113 | if err != nil { 114 | return err 115 | } 116 | return handler.HandleError(rconn, buf) 117 | } 118 | 119 | func (rconn *RedisConnection) consumeInteger(handler RedisHandler) error { 120 | n, err := rconn.readInteger() 121 | if err != nil { 122 | return err 123 | } 124 | return handler.HandleInteger(rconn, n) 125 | } 126 | 127 | func (rconn *RedisConnection) consumeShortString(handler RedisHandler) error { 128 | buf, err := rconn.scanLine() 129 | if err != nil { 130 | return err 131 | } 132 | return handler.HandleString(rconn, buf) 133 | } 134 | 135 | func (rconn *RedisConnection) consumeBulkString(handler RedisHandler) error { 136 | n, err := rconn.readInteger() 137 | if err != nil { 138 | return err 139 | } 140 | 141 | if n < 0 { 142 | return handler.HandleNull(rconn) 143 | } 144 | 145 | strReader := io.LimitReader(rconn.reader, int64(n)) 146 | if err := handler.HandleBulkString(rconn, n, strReader); err != nil { 147 | return err 148 | } 149 | 150 | c, err := rconn.reader.ReadByte() 151 | if err != nil { 152 | return err 153 | } 154 | if c != '\r' { 155 | return fmt.Errorf("missing CR") 156 | } 157 | 158 | c, err = rconn.reader.ReadByte() 159 | if err != nil { 160 | return err 161 | } 162 | if c != '\n' { 163 | return fmt.Errorf("missing LF after CR") 164 | } 165 | 166 | return nil 167 | } 168 | 169 | func (rconn *RedisConnection) consumeArray(handler RedisHandler) error { 170 | n, err := rconn.readInteger() 171 | if err != nil { 172 | return err 173 | } 174 | 175 | if n < 0 { 176 | return handler.HandleNull(rconn) 177 | } 178 | 179 | return handler.HandleArray(rconn, n) 180 | } 181 | 182 | func (rconn *RedisConnection) readUInteger() (uint, error) { 183 | n, err := rconn.scanNumbers(0) 184 | if err != nil { 185 | return n, err 186 | } 187 | 188 | c, err := rconn.reader.ReadByte() 189 | if err != nil { 190 | return n, err 191 | } 192 | if c != '\n' { 193 | return n, fmt.Errorf("missing LF after CR") 194 | } 195 | 196 | return n, nil 197 | } 198 | 199 | func (rconn *RedisConnection) readInteger() (int, error) { 200 | n := 0 201 | 202 | c, err := rconn.reader.ReadByte() 203 | if err != nil { 204 | return n, err 205 | } 206 | 207 | if c == '-' { 208 | nu, err := rconn.scanNumbers(0) 209 | n = -int(nu) 210 | if err != nil { 211 | return n, err 212 | } 213 | } else if c != '\r' { 214 | // if c > '9' { 215 | // return n, fmt.Errorf("unexpected byte %v while scanning integer, expected [0-9]", c) 216 | // } 217 | nu, err := rconn.scanNumbers(uint(c - '0')) 218 | n = int(nu) 219 | if err != nil { 220 | return n, err 221 | } 222 | } 223 | 224 | c, err = rconn.reader.ReadByte() 225 | if err != nil { 226 | return n, err 227 | } 228 | if c != '\n' { 229 | return n, fmt.Errorf("missing LF after CR") 230 | } 231 | 232 | return n, nil 233 | } 234 | 235 | func (rconn *RedisConnection) scanNumbers(n uint) (uint, error) { 236 | for { 237 | c, err := rconn.reader.ReadByte() 238 | if err != nil { 239 | return n, err 240 | } 241 | if c == '\r' { 242 | break 243 | } 244 | // if c > '9' { 245 | // return n, fmt.Errorf("unexpected byte %v while scanning integer, expected [0-9]", c) 246 | // } 247 | 248 | n = 10*n + uint(c-'0') 249 | } 250 | 251 | return n, nil 252 | } 253 | 254 | func (rconn *RedisConnection) scanLine() ([]byte, error) { 255 | buf, err := rconn.reader.ReadBytes('\r') 256 | if err != nil { 257 | return buf, err 258 | } 259 | 260 | c, err := rconn.reader.ReadByte() 261 | if err != nil { 262 | return buf, err 263 | } 264 | if c != '\n' { 265 | return buf, fmt.Errorf("missing LF after CR") 266 | } 267 | 268 | return buf, nil 269 | } 270 | 271 | // WriteArrayHeader writes a "*N\r\n" array header. 272 | func (rconn *RedisConnection) WriteArrayHeader(num int) error { 273 | return rconn.writef("*%v\r\n", num) 274 | } 275 | 276 | // WriteInteger writes a ":N\r\n" integer literal 277 | func (rconn *RedisConnection) WriteInteger(num int) error { 278 | return rconn.writef(":%v\r\n", num) 279 | } 280 | 281 | // WriteNull writes a "$-1\r\n" null string 282 | func (rconn *RedisConnection) WriteNull() error { 283 | return rconn.write([]byte("$-1\r\n")) 284 | } 285 | 286 | // WriteNullArray writes a "*-1\r\n" null array 287 | func (rconn *RedisConnection) WriteNullArray() error { 288 | return rconn.write([]byte("*-1\r\n")) 289 | } 290 | 291 | // WriteBulkBytes writes "$N\r\n...\r\n" bulk string from a byte slice. 292 | func (rconn *RedisConnection) WriteBulkBytes(buf []byte) error { 293 | n := len(buf) 294 | if n == 0 { 295 | return rconn.write([]byte("$0\r\n\r\n")) 296 | } 297 | 298 | if err := rconn.writef("$%v\r\n", n); err != nil { 299 | return err 300 | } 301 | 302 | if _, err := rconn.Conn.Write(buf); err != nil { 303 | return err 304 | } 305 | 306 | return rconn.write([]byte("\r\n")) 307 | } 308 | 309 | // WriteBulkStringHeader writes a "$N\r\n" bulk string header. 310 | func (rconn *RedisConnection) WriteBulkStringHeader(n int) error { 311 | return rconn.writef("$%v\r\n", n) 312 | } 313 | 314 | // WriteBulkStringFooter writes a "\r\n" bulk string footer. 315 | func (rconn *RedisConnection) WriteBulkStringFooter() error { 316 | return rconn.write([]byte("\r\n")) 317 | } 318 | 319 | // WriteBulkString writes a "$N\r\n...\r\n" bulk string. 320 | func (rconn *RedisConnection) WriteBulkString(str string) error { 321 | n := len(str) 322 | if n == 0 { 323 | return rconn.write([]byte("$0\r\n\r\n")) 324 | } 325 | return rconn.writef("$%v\r\n%v\r\n", n, str) 326 | } 327 | 328 | // WriteSimpleString writes a "+...\r\n" simple string. 329 | func (rconn *RedisConnection) WriteSimpleString(str string) error { 330 | return rconn.writef("+%v\r\n", str) 331 | } 332 | 333 | // WriteSimpleBytes writes a "+...\r\n" simple string froma byte slice. 334 | func (rconn *RedisConnection) WriteSimpleBytes(b []byte) error { 335 | return rconn.writef("+%s\r\n", b) 336 | } 337 | 338 | // WriteError writes a "-ERR...\r\n" error. 339 | func (rconn *RedisConnection) WriteError(err error) error { 340 | return rconn.writef("-ERR %v\r\n", err) 341 | } 342 | 343 | // WriteErrorBytes writes a "-...\r\n" error from a byte slice. 344 | func (rconn *RedisConnection) WriteErrorBytes(b []byte) error { 345 | if _, err := rconn.Conn.Write([]byte("-")); err != nil { 346 | return err 347 | } 348 | if _, err := rconn.Conn.Write(b); err != nil { 349 | return err 350 | } 351 | if _, err := rconn.Conn.Write([]byte("\r\n")); err != nil { 352 | return err 353 | } 354 | return nil 355 | } 356 | 357 | // WriteErrorString writes a "-TYPE ...\r\n" error from a string type and body. 358 | func (rconn *RedisConnection) WriteErrorString(errType, str string) error { 359 | return rconn.writef("-%v %v\r\n", errType, str) 360 | } 361 | 362 | func (rconn *RedisConnection) writef(format string, a ...interface{}) error { 363 | _, err := fmt.Fprintf(rconn.Conn, format, a...) 364 | return err 365 | } 366 | 367 | func (rconn *RedisConnection) write(buf []byte) error { 368 | if _, err := rconn.Conn.Write(buf); err != nil { 369 | return err 370 | } 371 | return nil 372 | } 373 | -------------------------------------------------------------------------------- /internal/marshaled/source.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package marshaled 22 | 23 | import ( 24 | "fmt" 25 | "io" 26 | "log" 27 | "sort" 28 | "strings" 29 | "sync" 30 | "time" 31 | 32 | "github.com/uber-go/gwr/source" 33 | ) 34 | 35 | // NOTE: This approach is perhaps overfit to the json module's marshalling 36 | // mindset. A better interface (for performance) would work by passing a 37 | // writer to the specific encoder, rather than a []byte-returning Marshal 38 | // function. This would be possible perhaps using something like 39 | // io.MultiWriter. 40 | 41 | // DataSource wraps a format-agnostic data source and provides one or 42 | // more formats for it. 43 | // 44 | // DataSource implements: 45 | // - DataSource to satisfy DataSources and low level protocols 46 | // - ItemDataSource so that higher level protocols may add their own framing 47 | // - GenericDataWatcher inwardly to the wrapped GenericDataSource 48 | type DataSource struct { 49 | // TODO: better to have alternate implementations for each combination 50 | // rather than one with these nil checks 51 | source source.GenericDataSource 52 | getSource source.GetableDataSource 53 | watchSource source.WatchableDataSource 54 | watiSource source.WatchInitableDataSource 55 | actiSource source.ActivateWatchableDataSource 56 | 57 | formats map[string]source.GenericDataFormat 58 | formatNames []string 59 | maxItems int 60 | maxBatches int 61 | maxWait time.Duration 62 | 63 | procs sync.WaitGroup 64 | watchLock sync.RWMutex 65 | watchers map[string]*marshaledWatcher 66 | active bool 67 | itemChan chan interface{} 68 | itemsChan chan []interface{} 69 | } 70 | 71 | func stringIt(item interface{}) ([]byte, error) { 72 | var s string 73 | if ss, ok := item.(fmt.Stringer); ok { 74 | s = ss.String() 75 | } else { 76 | s = fmt.Sprintf("%+v", item) 77 | } 78 | return []byte(s), nil 79 | } 80 | 81 | // NewDataSource creates a DataSource for a given format-agnostic data source 82 | // and a map of marshalers 83 | func NewDataSource( 84 | src source.GenericDataSource, 85 | formats map[string]source.GenericDataFormat, 86 | ) *DataSource { 87 | if formats == nil { 88 | formats = make(map[string]source.GenericDataFormat) 89 | } 90 | 91 | // source-defined formats 92 | if fmtsrc, ok := src.(source.GenericDataSourceFormats); ok { 93 | fmts := fmtsrc.Formats() 94 | for name, fmt := range fmts { 95 | formats[name] = fmt 96 | } 97 | } 98 | 99 | // standard json protocol 100 | if formats["json"] == nil { 101 | formats["json"] = LDJSONMarshal 102 | } 103 | 104 | // convenience templated text protocol 105 | if formats["text"] == nil { 106 | if txtsrc, ok := src.(source.TextTemplatedSource); ok { 107 | if tt := txtsrc.TextTemplate(); tt != nil { 108 | formats["text"] = NewTemplatedMarshal(tt) 109 | } 110 | } 111 | } 112 | 113 | // default to just string-ing it 114 | if formats["text"] == nil { 115 | formats["text"] = source.GenericDataFormatFunc(stringIt) 116 | } 117 | 118 | ds := &DataSource{ 119 | source: src, 120 | formats: formats, 121 | watchers: make(map[string]*marshaledWatcher, len(formats)), 122 | // TODO: tunable 123 | maxItems: 100, 124 | maxBatches: 100, 125 | maxWait: 100 * time.Microsecond, 126 | } 127 | ds.getSource, _ = src.(source.GetableDataSource) 128 | ds.watchSource, _ = src.(source.WatchableDataSource) 129 | ds.watiSource, _ = src.(source.WatchInitableDataSource) 130 | ds.actiSource, _ = src.(source.ActivateWatchableDataSource) 131 | for name, format := range formats { 132 | ds.formatNames = append(ds.formatNames, name) 133 | ds.watchers[name] = newMarshaledWatcher(ds, format) 134 | } 135 | sort.Strings(ds.formatNames) 136 | 137 | if ds.watchSource != nil { 138 | ds.watchSource.SetWatcher(ds) 139 | } 140 | 141 | return ds 142 | } 143 | 144 | // Active returns true if there are any active watchers, false otherwise. If 145 | // Active returns false, so will any calls to HandleItem and HandleItems. 146 | func (mds *DataSource) Active() bool { 147 | mds.watchLock.RLock() 148 | r := mds.active && mds.itemChan != nil && mds.itemsChan != nil 149 | mds.watchLock.RUnlock() 150 | return r 151 | } 152 | 153 | // Name passes through the GenericDataSource.Name() 154 | func (mds *DataSource) Name() string { 155 | return mds.source.Name() 156 | } 157 | 158 | // Formats returns the list of supported format names. 159 | func (mds *DataSource) Formats() []string { 160 | return mds.formatNames 161 | } 162 | 163 | // Attrs returns arbitrary description information about the data source. 164 | func (mds *DataSource) Attrs() map[string]interface{} { 165 | // TODO: support per-format Attrs? 166 | // TODO: any support for per-source Attrs? 167 | return nil 168 | } 169 | 170 | // Get marshals data source's Get data to the writer 171 | func (mds *DataSource) Get(formatName string, w io.Writer) error { 172 | if mds.getSource == nil { 173 | return source.ErrNotGetable 174 | } 175 | format, ok := mds.formats[strings.ToLower(formatName)] 176 | if !ok { 177 | return source.ErrUnsupportedFormat 178 | } 179 | data := mds.getSource.Get() 180 | buf, err := format.MarshalGet(data) 181 | if err != nil { 182 | log.Printf("get marshaling error %v", err) 183 | return err 184 | } 185 | _, err = w.Write(buf) 186 | return err 187 | } 188 | 189 | // Watch marshals any data source GetInit data to the writer, and then 190 | // retains a reference to the writer so that any future agnostic data source 191 | // Watch(emit)'ed data gets marshaled to it as well 192 | func (mds *DataSource) Watch(formatName string, w io.Writer) error { 193 | if mds.watchSource == nil { 194 | return source.ErrNotWatchable 195 | } 196 | 197 | mds.watchLock.Lock() 198 | acted := !mds.active 199 | err := func() error { 200 | defer mds.watchLock.Unlock() 201 | watcher, ok := mds.watchers[strings.ToLower(formatName)] 202 | if !ok { 203 | return source.ErrUnsupportedFormat 204 | } 205 | if err := watcher.init(w); err != nil { 206 | return err 207 | } 208 | if err := mds.startWatching(); err != nil { 209 | return err 210 | } 211 | return nil 212 | }() 213 | 214 | if err == nil && acted && mds.actiSource != nil { 215 | mds.actiSource.Activate() 216 | } 217 | return err 218 | } 219 | 220 | // WatchItems marshals any data source GetInit data as a single item to the 221 | // ItemWatcher's HandleItem method. The watcher is then retained and future 222 | // items are marshaled to its HandleItem method. 223 | func (mds *DataSource) WatchItems(formatName string, iw source.ItemWatcher) error { 224 | if mds.watchSource == nil { 225 | return source.ErrNotWatchable 226 | } 227 | 228 | mds.watchLock.Lock() 229 | acted := !mds.active 230 | err := func() error { 231 | defer mds.watchLock.Unlock() 232 | watcher, ok := mds.watchers[strings.ToLower(formatName)] 233 | if !ok { 234 | return source.ErrUnsupportedFormat 235 | } 236 | if err := watcher.initItems(iw); err != nil { 237 | return err 238 | } 239 | if err := mds.startWatching(); err != nil { 240 | return err 241 | } 242 | return nil 243 | }() 244 | 245 | if err == nil && acted && mds.actiSource != nil { 246 | mds.actiSource.Activate() 247 | } 248 | return err 249 | } 250 | 251 | // startWatching flips the active bit, creates new item channels, and starts a 252 | // processing go routine; it assumes that the watchLock is being held by the 253 | // caller. 254 | func (mds *DataSource) startWatching() error { 255 | // TODO: we could optimize the only-one-format-being-watched case 256 | if mds.active { 257 | return nil 258 | } 259 | mds.active = true 260 | mds.itemChan = make(chan interface{}, mds.maxItems) 261 | mds.itemsChan = make(chan []interface{}, mds.maxBatches) 262 | mds.procs.Add(1) 263 | go mds.processItemChan(mds.itemChan, mds.itemsChan) 264 | return nil 265 | } 266 | 267 | // Drain closes the item channels, and waits for the item processor to finish. 268 | // After drain, any remaining watchers are closed, and the source goes 269 | // inactive. 270 | func (mds *DataSource) Drain() { 271 | mds.watchLock.Lock() 272 | any := false 273 | if mds.itemChan != nil { 274 | close(mds.itemChan) 275 | any = true 276 | mds.itemChan = nil 277 | } 278 | if mds.itemsChan != nil { 279 | close(mds.itemsChan) 280 | any = true 281 | mds.itemsChan = nil 282 | } 283 | if any { 284 | mds.watchLock.Unlock() 285 | mds.procs.Wait() 286 | mds.watchLock.Lock() 287 | } 288 | stop := mds.active 289 | if stop { 290 | mds.active = false 291 | } 292 | mds.watchLock.Unlock() 293 | 294 | if stop { 295 | for _, watcher := range mds.watchers { 296 | watcher.Close() 297 | } 298 | } 299 | } 300 | 301 | func (mds *DataSource) processItemChan(itemChan chan interface{}, itemsChan chan []interface{}) { 302 | defer mds.procs.Done() 303 | 304 | stop := false 305 | 306 | for !stop && (itemChan != nil || itemsChan != nil) { 307 | mds.watchLock.RLock() 308 | active := mds.active 309 | watchers := mds.watchers 310 | mds.watchLock.RUnlock() 311 | if !active { 312 | break 313 | } 314 | select { 315 | case item, ok := <-itemChan: 316 | if !ok { 317 | itemChan = nil 318 | continue 319 | } 320 | any := false 321 | for _, watcher := range watchers { 322 | if watcher.emit(item) { 323 | any = true 324 | } 325 | } 326 | if !any { 327 | stop = true 328 | } 329 | 330 | case items, ok := <-itemsChan: 331 | if !ok { 332 | itemsChan = nil 333 | continue 334 | } 335 | any := false 336 | for _, watcher := range watchers { 337 | if watcher.emitBatch(items) { 338 | any = true 339 | } 340 | } 341 | if !any { 342 | stop = true 343 | } 344 | } 345 | } 346 | 347 | mds.watchLock.Lock() 348 | if mds.itemChan == itemChan { 349 | mds.itemChan = nil 350 | } 351 | if mds.itemsChan == itemsChan { 352 | mds.itemsChan = nil 353 | } 354 | if stop { 355 | mds.active = false 356 | } 357 | mds.watchLock.Unlock() 358 | 359 | if stop { 360 | for _, watcher := range mds.watchers { 361 | watcher.Close() 362 | } 363 | } 364 | } 365 | 366 | // HandleItem implements GenericDataWatcher.HandleItem by passing the item to 367 | // all current marshaledWatchers. 368 | func (mds *DataSource) HandleItem(item interface{}) bool { 369 | if !mds.Active() { 370 | return false 371 | } 372 | select { 373 | case mds.itemChan <- item: 374 | return true 375 | case <-time.After(mds.maxWait): 376 | mds.watchLock.Lock() 377 | if !mds.active { 378 | mds.watchLock.Unlock() 379 | return false 380 | } 381 | mds.active = false 382 | mds.watchLock.Unlock() 383 | for _, watcher := range mds.watchers { 384 | watcher.Close() 385 | } 386 | return false 387 | } 388 | } 389 | 390 | // HandleItems implements GenericDataWatcher.HandleItems by passing the batch 391 | // to all current marshaledWatchers. 392 | func (mds *DataSource) HandleItems(items []interface{}) bool { 393 | if !mds.Active() { 394 | return false 395 | } 396 | select { 397 | case mds.itemsChan <- items: 398 | return true 399 | case <-time.After(mds.maxWait): 400 | mds.watchLock.Lock() 401 | if !mds.active { 402 | mds.watchLock.Unlock() 403 | return false 404 | } 405 | mds.active = false 406 | mds.watchLock.Unlock() 407 | for _, watcher := range mds.watchers { 408 | watcher.Close() 409 | } 410 | return false 411 | } 412 | } 413 | -------------------------------------------------------------------------------- /internal/protocol/redis.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package protocol 22 | 23 | import ( 24 | "bytes" 25 | "encoding/json" 26 | "fmt" 27 | "strings" 28 | 29 | "github.com/uber-go/gwr/internal/resp" 30 | "github.com/uber-go/gwr/source" 31 | ) 32 | 33 | // NewRedisServer creates a new redis server to provide access to a collection 34 | // of gwr data sources. 35 | func NewRedisServer(sources *source.DataSources) *resp.RedisServer { 36 | handler := NewRedisHandler(sources) 37 | return resp.NewRedisServer(handler) 38 | } 39 | 40 | // NewRedisHandler creates a new redis handler for a given collection of gwr 41 | // data sources for use with the resp package. 42 | func NewRedisHandler(sources *source.DataSources) resp.RedisHandler { 43 | model := respModel{ 44 | sources: sources, 45 | sessions: make(map[*resp.RedisConnection]*respSession, 1), 46 | } 47 | return resp.CmdMapHandler(map[string]resp.CmdFunc{ 48 | "ls": model.handleLs, 49 | "get": model.handleGet, 50 | "watch": model.handleWatch, 51 | "monitor": model.handleMonitor, 52 | "__end__": model.handleEnd, 53 | }) 54 | } 55 | 56 | type respModel struct { 57 | sources *source.DataSources 58 | sessions map[*resp.RedisConnection]*respSession 59 | } 60 | 61 | type respSession struct { 62 | watches map[string]string 63 | stopMonitor chan struct{} 64 | } 65 | 66 | func (rm *respModel) session(rconn *resp.RedisConnection) *respSession { 67 | if session, ok := rm.sessions[rconn]; ok { 68 | return session 69 | } 70 | session := &respSession{ 71 | watches: make(map[string]string, 1), 72 | stopMonitor: make(chan struct{}, 1), 73 | } 74 | rm.sessions[rconn] = session 75 | return session 76 | } 77 | 78 | func (rm *respModel) handleLs(rconn *resp.RedisConnection, vc *resp.ValueConsumer) error { 79 | // TODO: implement optional path argument 80 | // TODO: maybe custom format 81 | 82 | if vc.NumRemaining() > 0 { 83 | return fmt.Errorf("too many arguments to ls") 84 | } 85 | 86 | return rm.doGet(rconn, rm.sources.Get("/meta/nouns"), "text") 87 | } 88 | 89 | func (rm *respModel) handleGet(rconn *resp.RedisConnection, vc *resp.ValueConsumer) error { 90 | source, err := rm.consumeSource(rconn, vc) 91 | if err != nil { 92 | return err 93 | } 94 | 95 | format, err := rm.consumeFormat(rconn, vc) 96 | if err != nil { 97 | return err 98 | } 99 | 100 | if vc.NumRemaining() > 0 { 101 | return fmt.Errorf("too many arguments to get") 102 | } 103 | 104 | return rm.doGet(rconn, source, format) 105 | } 106 | 107 | func (rm *respModel) doGet(rconn *resp.RedisConnection, source source.DataSource, format string) error { 108 | var buf bytes.Buffer 109 | if err := source.Get(format, &buf); err != nil { 110 | return err 111 | } 112 | 113 | switch format { 114 | case "text": 115 | lines := strings.Split(buf.String(), "\n") 116 | if i := len(lines) - 1; len(lines[i]) == 0 { 117 | lines = lines[:i] 118 | } 119 | if err := rconn.WriteArrayHeader(len(lines)); err != nil { 120 | return err 121 | } 122 | for _, line := range lines { 123 | if err := rconn.WriteSimpleString(line); err != nil { 124 | return err 125 | } 126 | } 127 | default: 128 | return rconn.WriteBulkBytes(buf.Bytes()) 129 | } 130 | 131 | return nil 132 | } 133 | 134 | func (rm *respModel) handleWatch(rconn *resp.RedisConnection, vc *resp.ValueConsumer) error { 135 | session := rm.session(rconn) 136 | 137 | source, err := rm.consumeSource(rconn, vc) 138 | if err != nil { 139 | return err 140 | } 141 | 142 | format, err := rm.consumeFormat(rconn, vc) 143 | if err != nil { 144 | return err 145 | } 146 | 147 | if vc.NumRemaining() > 0 { 148 | return fmt.Errorf("too many arguments to watch") 149 | } 150 | 151 | name := source.Name() 152 | session.watches[name] = format 153 | 154 | return rconn.WriteSimpleString("OK") 155 | } 156 | 157 | func (rm *respModel) handleMonitor(rconn *resp.RedisConnection, vc *resp.ValueConsumer) error { 158 | session := rm.session(rconn) 159 | 160 | for vc.NumRemaining() > 0 { 161 | source, err := rm.consumeSource(rconn, vc) 162 | if err != nil { 163 | return err 164 | } 165 | 166 | format, err := rm.consumeFormat(rconn, vc) 167 | if err != nil { 168 | return err 169 | } 170 | 171 | name := source.Name() 172 | session.watches[name] = format 173 | } 174 | 175 | if len(session.watches) == 0 { 176 | return fmt.Errorf("no watches set, monitor likely to be uninteresting") 177 | } 178 | 179 | go rm.doWatch(rconn) 180 | 181 | return nil 182 | } 183 | 184 | func (rm *respModel) doWatch(rconn *resp.RedisConnection) error { 185 | type bufInfoEntry struct { 186 | name, format string 187 | } 188 | 189 | session := rm.session(rconn) 190 | bufs := make([]*chanBuf, 0, len(session.watches)) 191 | itemBufs := make([]*itemBuf, 0, len(session.watches)) 192 | bufInfo := make(map[*chanBuf]bufInfoEntry, len(session.watches)) 193 | itemBufInfo := make(map[*itemBuf]bufInfoEntry, len(session.watches)) 194 | bufReady := make(chan *chanBuf, len(session.watches)) 195 | itemBufReady := make(chan *itemBuf, len(session.watches)) 196 | defer func() { 197 | for _, buf := range bufs { 198 | buf.Close() 199 | } 200 | for _, itemBuf := range itemBufs { 201 | itemBuf.Close() 202 | } 203 | }() 204 | 205 | for name, format := range session.watches { 206 | src := rm.sources.Get(name) 207 | if src == nil { 208 | continue 209 | } 210 | if itemSource, ok := src.(source.ItemDataSource); ok { 211 | itemBuf := newItemBuf(itemBufReady) 212 | itemBufs = append(itemBufs, itemBuf) 213 | itemBufInfo[itemBuf] = bufInfoEntry{ 214 | name: name, 215 | format: strings.ToLower(format), 216 | } 217 | itemSource.WatchItems(format, itemBuf) 218 | } else { 219 | buf := &chanBuf{ready: bufReady} 220 | bufs = append(bufs, buf) 221 | bufInfo[buf] = bufInfoEntry{ 222 | name: name, 223 | format: strings.ToLower(format), 224 | } 225 | src.Watch(format, buf) 226 | } 227 | } 228 | 229 | var write func(*resp.RedisConnection, *chanBuf, string, string) error 230 | var writeItems func(*resp.RedisConnection, *itemBuf, string, string) error 231 | 232 | if len(session.watches) == 1 { 233 | write = rm.writeSingleWatchData 234 | writeItems = rm.writeSingleWatchItem 235 | } else { 236 | write = rm.writeMultiWatchData 237 | writeItems = rm.writeMultiWatchItem 238 | } 239 | 240 | for { 241 | select { 242 | case <-session.stopMonitor: 243 | return nil 244 | case buf := <-bufReady: 245 | info := bufInfo[buf] 246 | if err := write(rconn, buf, info.name, info.format); err != nil { 247 | return err 248 | } 249 | case itemBuf := <-itemBufReady: 250 | info := itemBufInfo[itemBuf] 251 | if err := writeItems(rconn, itemBuf, info.name, info.format); err != nil { 252 | return err 253 | } 254 | } 255 | } 256 | } 257 | 258 | type multiJSONMessage struct { 259 | Name string `json:"name"` 260 | Data *json.RawMessage `json:"data"` 261 | } 262 | 263 | func (rm *respModel) writeSingleWatchItem(rconn *resp.RedisConnection, itemBuf *itemBuf, name, format string) error { 264 | switch format { 265 | case "text": 266 | for _, line := range itemBuf.drain() { 267 | // TODO: still need to split and send individual lines? 268 | if err := rconn.WriteSimpleBytes(line); err != nil { 269 | return err 270 | } 271 | } 272 | 273 | default: 274 | for _, buf := range itemBuf.drain() { 275 | if err := rconn.WriteBulkBytes(buf); err != nil { 276 | return err 277 | } 278 | } 279 | } 280 | return nil 281 | } 282 | 283 | func (rm *respModel) writeMultiWatchItem(rconn *resp.RedisConnection, itemBuf *itemBuf, name, format string) error { 284 | switch format { 285 | case "text": 286 | for _, buf := range itemBuf.drain() { 287 | // TODO: still need to split and send individual lines? 288 | line := fmt.Sprintf("%s> %s", name, buf) 289 | if err := rconn.WriteSimpleString(line); err != nil { 290 | return err 291 | } 292 | } 293 | 294 | case "json": 295 | for _, buf := range itemBuf.drain() { 296 | if buf, err := json.Marshal(multiJSONMessage{ 297 | Name: name, 298 | Data: (*json.RawMessage)(&buf), 299 | }); err != nil { 300 | return err 301 | } else if err := rconn.WriteBulkBytes(buf); err != nil { 302 | return err 303 | } 304 | } 305 | 306 | default: 307 | for _, buf := range itemBuf.drain() { 308 | if err := rconn.WriteArrayHeader(2); err != nil { 309 | return err 310 | } 311 | if err := rconn.WriteSimpleString(name); err != nil { 312 | return err 313 | } 314 | if err := rconn.WriteBulkBytes(buf); err != nil { 315 | return err 316 | } 317 | } 318 | } 319 | return nil 320 | } 321 | 322 | // TODO: can we re-use code b/w *WatchData and *WatchItems? 323 | 324 | func (rm *respModel) writeSingleWatchData(rconn *resp.RedisConnection, buf *chanBuf, name, format string) error { 325 | switch format { 326 | case "text": 327 | buf.Lock() 328 | for { 329 | line, doneErr := buf.ReadString('\n') 330 | if doneErr == nil { 331 | line = line[:len(line)-1] 332 | } 333 | if len(line) > 0 || doneErr == nil { 334 | if err := rconn.WriteSimpleString(line); err != nil { 335 | return err 336 | } 337 | } 338 | if doneErr != nil { 339 | break 340 | } 341 | } 342 | buf.Reset() 343 | buf.Unlock() 344 | 345 | case "json": 346 | buf.Lock() 347 | for { 348 | line, doneErr := buf.ReadString('\n') 349 | if len(line) > 0 { 350 | line = line[:len(line)-1] 351 | } 352 | if len(line) > 0 { 353 | if err := rconn.WriteBulkString(line); err != nil { 354 | return err 355 | } 356 | } 357 | if doneErr != nil { 358 | break 359 | } 360 | } 361 | buf.Reset() 362 | buf.Unlock() 363 | 364 | default: 365 | b := buf.drain() 366 | if err := rconn.WriteBulkBytes(b); err != nil { 367 | return err 368 | } 369 | } 370 | 371 | return nil 372 | } 373 | 374 | func (rm *respModel) writeMultiWatchData(rconn *resp.RedisConnection, buf *chanBuf, name, format string) error { 375 | switch format { 376 | case "text": 377 | buf.Lock() 378 | for { 379 | line, doneErr := buf.ReadString('\n') 380 | if doneErr == nil { 381 | line = line[:len(line)-1] 382 | } 383 | if len(line) > 0 || doneErr == nil { 384 | line = fmt.Sprintf("%s> %s", name, line) 385 | if err := rconn.WriteSimpleString(line); err != nil { 386 | return err 387 | } 388 | } 389 | if doneErr != nil { 390 | break 391 | } 392 | } 393 | buf.Reset() 394 | buf.Unlock() 395 | 396 | case "json": 397 | buf.Lock() 398 | for { 399 | line, doneErr := buf.ReadString('\n') 400 | if doneErr == nil { 401 | line = line[:len(line)-1] 402 | } 403 | if len(line) > 0 { 404 | data := []byte(line) 405 | if buf, err := json.Marshal(multiJSONMessage{ 406 | Name: name, 407 | Data: (*json.RawMessage)(&data), 408 | }); err != nil { 409 | return err 410 | } else if err := rconn.WriteBulkBytes(buf); err != nil { 411 | return err 412 | } 413 | } 414 | if doneErr != nil { 415 | break 416 | } 417 | } 418 | buf.Reset() 419 | buf.Unlock() 420 | 421 | default: 422 | b := buf.drain() 423 | if err := rconn.WriteArrayHeader(2); err != nil { 424 | return err 425 | } 426 | if err := rconn.WriteSimpleString(name); err != nil { 427 | return err 428 | } 429 | if err := rconn.WriteBulkBytes(b); err != nil { 430 | return err 431 | } 432 | } 433 | 434 | return nil 435 | } 436 | 437 | func (rm *respModel) handleEnd(rconn *resp.RedisConnection, vc *resp.ValueConsumer) error { 438 | session, ok := rm.sessions[rconn] 439 | if !ok { 440 | return nil 441 | } 442 | 443 | session.stopMonitor <- struct{}{} 444 | 445 | delete(rm.sessions, rconn) 446 | return nil 447 | } 448 | 449 | func (rm *respModel) consumeSource(rconn *resp.RedisConnection, vc *resp.ValueConsumer) (source.DataSource, error) { 450 | nameRV, err := vc.Consume("name") 451 | if err != nil { 452 | return nil, err 453 | } 454 | name, ok := nameRV.GetString() 455 | if !ok { 456 | return nil, fmt.Errorf("name argument not a string") 457 | } 458 | source := rm.sources.Get(name) 459 | if source == nil { 460 | return nil, fmt.Errorf("no such data source") 461 | } 462 | return source, nil 463 | } 464 | 465 | func (rm *respModel) consumeFormat(rconn *resp.RedisConnection, vc *resp.ValueConsumer) (string, error) { 466 | if vc.NumRemaining() == 0 { 467 | return "text", nil // XXX default 468 | } 469 | rv, err := vc.Consume("format") 470 | if err != nil { 471 | return "", err 472 | } 473 | format, ok := rv.GetString() 474 | if !ok { 475 | return "", fmt.Errorf("format argument not a string") 476 | } 477 | return format, nil 478 | } 479 | -------------------------------------------------------------------------------- /source/tap/tracer.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package tap 22 | 23 | import ( 24 | "fmt" 25 | "strings" 26 | "sync/atomic" 27 | "time" 28 | 29 | "github.com/uber-go/gwr" 30 | "github.com/uber-go/gwr/source" 31 | ) 32 | 33 | const ( 34 | defaultName = "/tap/trace" 35 | namePattern = "/tap/trace/%s" 36 | ) 37 | 38 | // Tracer implements a gwr data source that allows easy tracing of scope data, 39 | // such as function calls, or rounds of a worker goroutine's loop. 40 | // 41 | // Tracers should be created for each area of the application that can be 42 | // traced. This could be as simple as creating a package-level tracer: 43 | // 44 | // package foo 45 | // 46 | // import "github.com/uber-go/gwr/source" 47 | // 48 | // tracer := source.AddNewTracer("foo") 49 | // 50 | // Tracers can also be attached to parts of the application: 51 | // 52 | // type Thing struct { 53 | // t *Tracer 54 | // } 55 | // 56 | // func NewThing() *Thing { 57 | // // ... 58 | // t.tracer = source.AddNewTracer(fmt.Sprintf("foo/%v", someThingIdentifier)) 59 | // // ... 60 | // } 61 | // 62 | // If Things are not the same life-cycle as the application, then they should 63 | // have teardown code to remove their tracer data sources: 64 | // 65 | // gwr.DefaultDataSources.Remove(t.tracer.Name()) 66 | // 67 | // You can then proceed to trace your functions and methods. First decide 68 | // where/what you want to start tracing. this will probably be one or more 69 | // exported functions or methods called by your user. 70 | // 71 | // Within these entry points use Tracer.Scope to create a root scope. Pass 72 | // this scope along to any called functions that you want to trace. Functions 73 | // that get passed a scope should start off by calling scope.Sub to create 74 | // their own scope. 75 | // 76 | // Within a traced function, start off by calling scope.OpenCall to note the 77 | // start of function. While doing normal work within a traced function, you 78 | // may call scope.Info to note log any additional data. Finally once done the 79 | // traced function should call one of scope.CloseCall, scope.Error or 80 | // scope.ErrorName. 81 | // 82 | // If a traced function has more than one possible way to error, it should use 83 | // scope.ErrorName to describe what failed. Furthermore, a traced function may 84 | // call scope.ErrorName more than once if it has recoverable errors, or 85 | // otherwise makes progress around errors. 86 | // 87 | // You can similarly trace a worker goroutine: 88 | // 89 | // ch := make(chan int) 90 | // go func() { 91 | // for n := range ch { 92 | // scope := tracer.Scope("n <- workerChan").Open(n) 93 | // // do something... 94 | // scope.Close() 95 | // } 96 | // }() 97 | type Tracer struct { 98 | name string 99 | watcher source.GenericDataWatcher 100 | } 101 | 102 | // NewTracer creates a Tracer with a given name. 103 | func NewTracer(name string) *Tracer { 104 | name = fmt.Sprintf(namePattern, name) 105 | return &Tracer{ 106 | name: name, 107 | } 108 | } 109 | 110 | // AddNewTracer creates a new tracer and adds it to the default gwr sources. 111 | // It panics if the given name is already defined. 112 | func AddNewTracer(name string) *Tracer { 113 | src := NewTracer(name) 114 | if err := gwr.AddGenericDataSource(src); err != nil { 115 | panic(err.Error()) 116 | } 117 | return src 118 | } 119 | 120 | func (src *Tracer) emit(item interface{}) bool { 121 | if src.watcher == nil { 122 | return false 123 | } 124 | return src.watcher.HandleItem(item) 125 | } 126 | 127 | // Active returns true if there any watchers; when not active, all emitted data 128 | // is dropped. This should be used by call sites to control scope creation. 129 | func (src *Tracer) Active() bool { 130 | return src.watcher != nil && src.watcher.Active() 131 | } 132 | 133 | // Name returns the gwr source name of the tracer. 134 | func (src *Tracer) Name() string { 135 | return src.name 136 | } 137 | 138 | // Formats returns tracer-specific formats. 139 | func (src *Tracer) Formats() map[string]source.GenericDataFormat { 140 | return map[string]source.GenericDataFormat{ 141 | "text": defaultTextFormat, 142 | } 143 | } 144 | 145 | // SetWatcher sets the current watcher. 146 | func (src *Tracer) SetWatcher(watcher source.GenericDataWatcher) { 147 | src.watcher = watcher 148 | } 149 | 150 | // Scope creates a new named trace scope 151 | func (src *Tracer) Scope(name string) *TraceScope { 152 | return newScope(src, nil, name) 153 | } 154 | 155 | // MaybeScope creates a new named scope if the tracer is active; otherwise nil 156 | // is returned. 157 | func (src *Tracer) MaybeScope(name string) *TraceScope { 158 | if !src.Active() { 159 | return nil 160 | } 161 | return newScope(src, nil, name) 162 | } 163 | 164 | // DefaultTracer is available for easy scope logging without needing to create 165 | // a separate tracer. 166 | var DefaultTracer = Tracer{ 167 | name: "/tap/trace", 168 | } 169 | 170 | // Active returns whether the default tracer is active. 171 | func Active() bool { 172 | return DefaultTracer.Active() 173 | } 174 | 175 | // Scope creates a new scope on the default tracer. 176 | func Scope(name string) *TraceScope { 177 | return DefaultTracer.Scope(name) 178 | } 179 | 180 | // MaybeScope creates a new scope on the default tracer, if it is active; 181 | // otherwise nil is returned. 182 | func MaybeScope(name string) *TraceScope { 183 | return DefaultTracer.MaybeScope(name) 184 | } 185 | 186 | // TODO: better do this 187 | var lastTraceId uint64 188 | 189 | // ResetTraceID resets the last trace id; this is intended to be used only for 190 | // test stability. 191 | func ResetTraceID() { 192 | atomic.StoreUint64(&lastTraceId, 0) 193 | } 194 | 195 | // TraceScope represents a traced scope, such as a function call, or an 196 | // iteration of a worker goroutine loop. 197 | type TraceScope struct { 198 | trc *Tracer 199 | top *TraceScope 200 | parent *TraceScope 201 | id uint64 202 | name string 203 | begin time.Time 204 | end time.Time 205 | } 206 | 207 | func newScope(trc *Tracer, parent *TraceScope, name string) *TraceScope { 208 | sc := &TraceScope{ 209 | trc: trc, 210 | parent: parent, 211 | id: atomic.AddUint64(&lastTraceId, 1), 212 | name: name, 213 | } 214 | if parent != nil { 215 | sc.top = parent.top 216 | } else { 217 | sc.top = sc 218 | } 219 | return sc 220 | } 221 | 222 | // Active returns true if the tracer is active, false otherwise. 223 | func (sc *TraceScope) Active() bool { 224 | return sc.trc.Active() 225 | } 226 | 227 | // Root returns the root scope. 228 | func (sc *TraceScope) Root() *TraceScope { 229 | return sc.top 230 | } 231 | 232 | // Parent returns the parent scope; this is nil for root scopes.. 233 | func (sc *TraceScope) Parent() *TraceScope { 234 | return sc.parent 235 | } 236 | 237 | // BeginTime returns the time of the first scope.Open or scope.OpenCall (only 238 | // one of these should be called, but the first one wins for begin time 239 | // anyhow). Sub-scope times do not affect their parent scope's begin/end. 240 | func (sc *TraceScope) BeginTime() time.Time { 241 | return sc.begin 242 | } 243 | 244 | // EndTime returns the time of the last scope.Close, scope.CloseCall, 245 | // scope.Error, or scope.ErrorName. Sub-scope times do not affect their parent 246 | // scope's begin/end. 247 | func (sc *TraceScope) EndTime() time.Time { 248 | return sc.end 249 | } 250 | 251 | // Sub opens and returns a new sub-scope. 252 | func (sc *TraceScope) Sub(name string) *TraceScope { 253 | return newScope(sc.trc, sc, name) 254 | } 255 | 256 | // Info emits an info record with the passed arguments 257 | func (sc *TraceScope) Info(args ...interface{}) *TraceScope { 258 | return sc.emitRecord(infoRecord, genericArgs(args)) 259 | } 260 | 261 | // Open emits a begin record with the given arguments. 262 | func (sc *TraceScope) Open(args ...interface{}) *TraceScope { 263 | return sc.emitRecord(beginRecord, genericArgs(args)) 264 | } 265 | 266 | // Error emits an error record with the given error and arguments. 267 | func (sc *TraceScope) Error(err error, args ...interface{}) *TraceScope { 268 | return sc.ErrorName("", err, args...) 269 | } 270 | 271 | // ErrorName emits an error record with the given error and arguments. 272 | func (sc *TraceScope) ErrorName(name string, err error, args ...interface{}) *TraceScope { 273 | return sc.emitRecord(errRecord, errArgs{name, err, genericArgs(args)}) 274 | } 275 | 276 | // Close emits a end record with the given arguments. 277 | func (sc *TraceScope) Close(args ...interface{}) *TraceScope { 278 | return sc.emitRecord(endRecord, genericArgs(args)) 279 | } 280 | 281 | // OpenCall emits a begin record for a function call with the given arguments. 282 | func (sc *TraceScope) OpenCall(args ...interface{}) *TraceScope { 283 | return sc.emitRecord(beginRecord, callArgs(args)) 284 | } 285 | 286 | // CloseCall emits an end record for a function call with the return values. 287 | func (sc *TraceScope) CloseCall(rets ...interface{}) *TraceScope { 288 | return sc.emitRecord(endRecord, callRets(rets)) 289 | } 290 | 291 | func (sc *TraceScope) emitRecord(t recordType, args interface{}) *TraceScope { 292 | now := time.Now() 293 | switch t { 294 | case beginRecord: 295 | if sc.begin.IsZero() { 296 | sc.begin = now 297 | } 298 | case endRecord: 299 | fallthrough 300 | case errRecord: 301 | if sc.end.IsZero() || now.After(sc.end) { 302 | sc.end = now 303 | } 304 | } 305 | rec := record{ 306 | Time: now, 307 | Type: t, 308 | ScopeId: sc.top.id, 309 | SpanId: sc.id, 310 | Name: sc.name, 311 | Args: args, 312 | } 313 | if sc.parent != nil { 314 | rec.ParentId = &sc.parent.id 315 | } 316 | sc.trc.emit(&rec) 317 | return sc 318 | } 319 | 320 | func dumpArgs(args []interface{}) string { 321 | // TODO: replace / make better; consider using go-spew 322 | parts := make([]string, len(args)) 323 | for i, arg := range args { 324 | parts[i] = fmt.Sprintf("%v", arg) 325 | } 326 | return strings.Join(parts, ", ") 327 | } 328 | 329 | type recordType uint 330 | 331 | const ( 332 | beginRecord recordType = iota 333 | infoRecord 334 | endRecord 335 | errRecord 336 | ) 337 | 338 | func (t recordType) String() string { 339 | switch t { 340 | case beginRecord: 341 | return "begin" 342 | case infoRecord: 343 | return "info" 344 | case endRecord: 345 | return "end" 346 | case errRecord: 347 | return "error" 348 | default: 349 | return fmt.Sprintf("UNK(%d)", int(t)) 350 | } 351 | } 352 | 353 | func (t recordType) MarkString() string { 354 | switch t { 355 | case beginRecord: 356 | return "-->" 357 | case infoRecord: 358 | return "..." 359 | case endRecord: 360 | return "<--" 361 | case errRecord: 362 | return "!!!" 363 | default: 364 | return fmt.Sprintf("UNK(%d)", int(t)) 365 | } 366 | } 367 | 368 | type genericArgs []interface{} 369 | 370 | func (args genericArgs) String() string { 371 | return dumpArgs(args) 372 | } 373 | 374 | type callArgs []interface{} 375 | 376 | func (args callArgs) String() string { 377 | return dumpArgs(args) 378 | } 379 | 380 | type callRets []interface{} 381 | 382 | func (args callRets) String() string { 383 | return dumpArgs(args) 384 | } 385 | 386 | type errArgs struct { 387 | name string 388 | err error 389 | extra genericArgs 390 | } 391 | 392 | func (args errArgs) String() string { 393 | var s string 394 | if args.name != "" { 395 | s = fmt.Sprintf("%s Error(%s)", args.name, args.err) 396 | } else { 397 | s = fmt.Sprintf("Error(%s)", args.err) 398 | } 399 | if len(args.extra) > 0 { 400 | s = fmt.Sprintf("%s %s", s, args.extra) 401 | } 402 | return s 403 | } 404 | 405 | type record struct { 406 | Time time.Time `json:"time"` 407 | Type recordType `json:"type"` 408 | ScopeId uint64 `json:"scope_id"` 409 | SpanId uint64 `json:"span_id"` 410 | ParentId *uint64 `json:"parent_id"` 411 | Name string `json:"name"` 412 | Args interface{} `json:"args"` 413 | } 414 | 415 | func (rec record) IDString() string { 416 | if rec.ParentId == nil { 417 | return fmt.Sprintf("%v::%v", rec.ScopeId, rec.SpanId) 418 | } 419 | return fmt.Sprintf("%v:%v:%v", rec.ScopeId, *rec.ParentId, rec.SpanId) 420 | } 421 | 422 | func (rec record) String() string { 423 | switch rec.Args.(type) { 424 | case callArgs: 425 | return fmt.Sprintf("%s %s [%s] %s(%s)", 426 | rec.Type.MarkString(), rec.Time, rec.IDString(), 427 | rec.Name, rec.Args) 428 | case callRets: 429 | return fmt.Sprintf("%s %s [%s] return %s", 430 | rec.Type.MarkString(), rec.Time, rec.IDString(), 431 | rec.Args) 432 | default: 433 | switch rec.Type { 434 | case beginRecord: 435 | return fmt.Sprintf("%s %s [%s] %s: %s", 436 | rec.Type.MarkString(), rec.Time, rec.IDString(), 437 | rec.Name, rec.Args) 438 | default: 439 | return fmt.Sprintf("%s %s [%s] %s", 440 | rec.Type.MarkString(), rec.Time, rec.IDString(), 441 | rec.Args) 442 | } 443 | } 444 | } 445 | --------------------------------------------------------------------------------