├── .gitignore ├── .travis.yml ├── LICENSE ├── Makefile ├── README.md ├── benchmarks └── bench_test.go ├── closers.go ├── config.go ├── config_test.go ├── getter.go ├── getter_test.go ├── go.mod ├── go.sum ├── group.go ├── group_test.go ├── konfig.go ├── loader.go ├── loader ├── klconsul │ ├── README.md │ ├── consulloader.go │ ├── consulloader_integration_test.go │ └── consulloader_test.go ├── klenv │ ├── README.md │ ├── envloader.go │ └── envloader_test.go ├── kletcd │ ├── README.md │ ├── etcd_integration_test.go │ ├── etcdloader.go │ └── etcdloader_test.go ├── klfile │ ├── README.md │ ├── fileloader.go │ └── fileloader_test.go ├── klflag │ ├── README.md │ ├── flagloader.go │ └── flagloader_test.go ├── klhttp │ ├── README.md │ ├── httploader.go │ ├── httploader_test.go │ └── source.go ├── klreader │ ├── README.md │ ├── klreader.go │ └── reader_test.go └── klvault │ ├── README.md │ ├── auth │ ├── k8s │ │ ├── k8s.go │ │ └── k8s_test.go │ └── token │ │ └── token.go │ ├── authprovider.go │ ├── vaultloader.go │ └── vaultloader_test.go ├── loader_mock_test.go ├── loader_test.go ├── loaderwatcher.go ├── metrics.go ├── metrics_test.go ├── mocks ├── authprovider_mock.go ├── client_mock.go ├── consulkv_mock.go ├── contexter_mock.go ├── kv_mock.go ├── loader_mock.go ├── logicalclient_mock.go ├── parser_mock.go └── watcher_mock.go ├── parser ├── kpjson │ ├── README.md │ ├── jsonparser.go │ └── jsonparser_test.go ├── kpkeyval │ ├── README.md │ ├── kvparser.go │ └── kvparser_test.go ├── kpmap │ ├── README.md │ ├── mapparser.go │ └── mapparser_test.go ├── kptoml │ ├── README.md │ ├── tomlparser.go │ └── tomlparser_test.go ├── kpyaml │ ├── README.md │ ├── yamlparser.go │ └── yamlparser_test.go ├── parser.go └── parser_test.go ├── test.sh ├── test ├── configfile_test.go └── data │ ├── cfg │ └── cfg.yml ├── util.go ├── util_test.go ├── value.go ├── value_test.go ├── values.go ├── watcher.go ├── watcher ├── kwfile │ ├── README.md │ ├── filewatcher.go │ └── filewatcher_test.go └── kwpoll │ ├── README.md │ ├── pollwatcher.go │ └── pollwatcher_test.go ├── watcher_mock_test.go └── watcher_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | debug.test 2 | coverage.out 3 | vendor/* 4 | **/**/*.out 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - "1.13.x" 5 | - "1.14.x" 6 | 7 | script: 8 | - go get -u golang.org/x/lint/golint 9 | - make lint 10 | - go get github.com/stretchr/testify 11 | - ./test.sh 12 | 13 | after_success: 14 | - bash <(curl -s https://codecov.io/bash) 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2019 Lalamove 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | test: 2 | go test $(shell go list ./... | grep -v /examples/ ) -covermode=count 3 | 4 | test-race: 5 | go test -race $(shell go list ./... | grep -v /examples/ ) 6 | 7 | coverage: 8 | GO111MODULE=off go test ./... -cover -covermode=count -coverprofile=cover.out; GO111MODULE=off go tool cover -func cover.out; 9 | 10 | coverage-html: 11 | GO111MODULE=off go test ./... -cover -covermode=count -coverprofile=cover.out; GO111MODULE=off go tool cover -html=cover.out; 12 | 13 | benchmarks: 14 | cd benchmarks && go test -bench . && cd ../ 15 | 16 | lint: 17 | golint -set_exit_status $(shell (go list ./... | grep -v /vendor/)) 18 | 19 | mocks: 20 | mockgen -source ./loader.go -package mocks > ./mocks/loader_mock.go 21 | mockgen -source ./watcher.go -package mocks > ./mocks/watcher_mock.go 22 | mockgen -source ./loader.go -package konfig > ./loader_mock_test.go 23 | mockgen -source ./watcher.go -package konfig > ./watcher_mock_test.go 24 | mockgen -source ./loader/klvault/authprovider.go -package mocks > ./mocks/authprovider_mock.go 25 | mockgen -source ./loader/klvault/vaultloader.go -package mocks LogicalClient > ./mocks/logicalclient_mock.go 26 | mockgen -source ./parser/parser.go -package mocks Parser > ./mocks/parser_mock.go 27 | mockgen -source ./loader/klhttp/httploader.go -package mocks Client > ./mocks/client_mock.go 28 | mockgen -package mocks go.etcd.io/etcd/clientv3 KV > ./mocks/kv_mock.go 29 | mockgen -package mocks github.com/lalamove/nui/ncontext Contexter > ./mocks/contexter_mock.go 30 | mockgen -source ./parser/parser.go -package mocks Parser > ./mocks/parser_mock.go 31 | mockgen -source ./loader/klconsul/consulloader.go -package mocks ConsulKV > ./mocks/consulkv_mock.go 32 | 33 | .PHONY: test test-race coverage coverage-html lint benchmarks mocks 34 | -------------------------------------------------------------------------------- /benchmarks/bench_test.go: -------------------------------------------------------------------------------- 1 | package benchmarks 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/lalamove/konfig" 7 | config "github.com/micro/go-micro/config" 8 | "github.com/micro/go-micro/config/source/memory" 9 | "github.com/spf13/viper" 10 | ) 11 | 12 | func BenchmarkGetKonfig(b *testing.B) { 13 | var k = konfig.New(konfig.DefaultConfig()) 14 | k.Set("foo", "bar") 15 | 16 | b.ReportAllocs() 17 | b.ResetTimer() 18 | for i := 0; i < b.N; i++ { 19 | k.Get("foo") 20 | } 21 | } 22 | 23 | func BenchmarkStringKonfig(b *testing.B) { 24 | var k = konfig.New(konfig.DefaultConfig()) 25 | k.Set("foo", "bar") 26 | 27 | b.ReportAllocs() 28 | b.ResetTimer() 29 | for i := 0; i < b.N; i++ { 30 | k.String("foo") 31 | } 32 | } 33 | 34 | func BenchmarkGetViper(b *testing.B) { 35 | var v = viper.New() 36 | v.Set("foo", "bar") 37 | 38 | b.ReportAllocs() 39 | b.ResetTimer() 40 | for i := 0; i < b.N; i++ { 41 | v.Get("foo") 42 | } 43 | 44 | } 45 | 46 | func BenchmarkStringViper(b *testing.B) { 47 | var v = viper.New() 48 | v.Set("foo", "bar") 49 | 50 | b.ReportAllocs() 51 | b.ResetTimer() 52 | for i := 0; i < b.N; i++ { 53 | v.GetString("foo") 54 | } 55 | } 56 | 57 | var data = []byte(`{ 58 | "foo": "bar" 59 | }`) 60 | 61 | func newGoConfig() config.Config { 62 | memorySource := memory.NewSource( 63 | memory.WithJSON(data), 64 | ) 65 | // Create new config 66 | conf := config.NewConfig() 67 | // Load file source 68 | conf.Load(memorySource) 69 | 70 | return conf 71 | } 72 | 73 | func BenchmarkGetGoConfig(b *testing.B) { 74 | conf := newGoConfig() 75 | 76 | b.ReportAllocs() 77 | b.ResetTimer() 78 | for i := 0; i < b.N; i++ { 79 | conf.Get("foo") 80 | } 81 | } 82 | 83 | func BenchmarkStringGoConfig(b *testing.B) { 84 | conf := newGoConfig() 85 | 86 | b.ReportAllocs() 87 | b.ResetTimer() 88 | for i := 0; i < b.N; i++ { 89 | conf.Get("foo").String("bar") 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /closers.go: -------------------------------------------------------------------------------- 1 | package konfig 2 | 3 | import ( 4 | "io" 5 | 6 | multierror "github.com/hashicorp/go-multierror" 7 | ) 8 | 9 | // Closers is a multi closer 10 | type Closers []io.Closer 11 | 12 | // Close closes all closers in the multi closer and returns an error if an error was encountered. 13 | // Error returned is multierror.Error. https://github.com/hashicorp/go-multierror 14 | func (cs Closers) Close() error { 15 | var multiErr error 16 | for _, closer := range cs { 17 | if err := closer.Close(); err != nil { 18 | multierror.Append(multiErr, err) 19 | } 20 | } 21 | return multiErr 22 | } 23 | -------------------------------------------------------------------------------- /getter.go: -------------------------------------------------------------------------------- 1 | package konfig 2 | 3 | import "github.com/lalamove/nui/ngetter" 4 | 5 | // Getter returns a mgetter.Getter for the key k 6 | func Getter(k string) ngetter.GetterTyped { 7 | return instance().Getter(k) 8 | } 9 | 10 | // Getter returns a mgetter.Getter for the key k 11 | func (c *S) Getter(k string) ngetter.GetterTyped { 12 | return ngetter.GetterTypedFunc(func() interface{} { 13 | return c.Get(k) 14 | }) 15 | } 16 | -------------------------------------------------------------------------------- /getter_test.go: -------------------------------------------------------------------------------- 1 | package konfig 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestGetter(t *testing.T) { 10 | t.Run( 11 | "test new getter", 12 | func(t *testing.T) { 13 | Init(DefaultConfig()) 14 | 15 | var c = instance() 16 | c.Set("int", 1) 17 | 18 | var g = Getter("int") 19 | 20 | require.Equal(t, "1", g.String()) 21 | }, 22 | ) 23 | } 24 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/lalamove/konfig 2 | 3 | require ( 4 | github.com/BurntSushi/toml v0.3.1 5 | github.com/armon/go-metrics v0.3.1 // indirect 6 | github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect 7 | github.com/coreos/bbolt v1.3.2 // indirect 8 | github.com/coreos/etcd v3.3.10+incompatible 9 | github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d // indirect 10 | github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect 11 | github.com/davecgh/go-spew v1.1.1 12 | github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect 13 | github.com/francoispqt/gojay v0.0.0-20181220093123-f2cc13a668ca 14 | github.com/frankban/quicktest v1.4.1 // indirect 15 | github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 // indirect 16 | github.com/go-test/deep v1.0.2 // indirect 17 | github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect 18 | github.com/golang/mock v1.4.3 19 | github.com/gorilla/websocket v1.4.1 // indirect 20 | github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 // indirect 21 | github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect 22 | github.com/grpc-ecosystem/grpc-gateway v1.9.5 // indirect 23 | github.com/hashicorp/consul/api v1.4.0 24 | github.com/hashicorp/consul/sdk v0.4.0 25 | github.com/hashicorp/go-multierror v1.1.0 26 | github.com/hashicorp/serf v0.9.2 // indirect 27 | github.com/hashicorp/vault/api v1.0.5-0.20200317185738-82f498082f02 28 | github.com/hashicorp/vault/sdk v0.1.14-0.20200429182704-29fce8f27ce4 // indirect 29 | github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3 30 | github.com/jonboulle/clockwork v0.1.0 // indirect 31 | github.com/lalamove/nui v0.2.0 32 | github.com/micro/go-micro v1.10.0 33 | github.com/mitchellh/go-testing-interface v1.0.3 // indirect 34 | github.com/mitchellh/mapstructure v1.2.3 // indirect 35 | github.com/pierrec/lz4 v2.2.6+incompatible // indirect 36 | github.com/pkg/errors v0.8.1 37 | github.com/prometheus/client_golang v1.4.0 38 | github.com/radovskyb/watcher v1.0.5 39 | github.com/soheilhy/cmux v0.1.4 // indirect 40 | github.com/spf13/cast v1.3.0 41 | github.com/spf13/viper v1.3.1 42 | github.com/stretchr/testify v1.4.0 43 | github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect 44 | github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43 // indirect 45 | github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect 46 | go.etcd.io/bbolt v1.3.3 // indirect 47 | go.etcd.io/etcd v3.3.10+incompatible 48 | go.uber.org/atomic v1.4.0 // indirect 49 | go.uber.org/multierr v1.1.0 // indirect 50 | go.uber.org/zap v1.10.0 // indirect 51 | golang.org/x/crypto v0.0.0-20200117160349-530e935923ad // indirect 52 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b // indirect 53 | google.golang.org/grpc v1.23.1 // indirect 54 | gopkg.in/square/go-jose.v2 v2.4.1 // indirect 55 | gopkg.in/yaml.v2 v2.2.8 56 | ) 57 | 58 | go 1.14 59 | -------------------------------------------------------------------------------- /group.go: -------------------------------------------------------------------------------- 1 | package konfig 2 | 3 | // Group gets a group of configs 4 | func Group(groupName string) Store { 5 | return instance().Group(groupName) 6 | } 7 | 8 | // Group gets a group of configs 9 | func (c *S) Group(groupName string) Store { 10 | return c.lazyGroup(groupName) 11 | } 12 | 13 | func (c *S) lazyGroup(groupName string) Store { 14 | c.mut.Lock() 15 | defer c.mut.Unlock() 16 | if v, ok := c.groups[groupName]; ok { 17 | return v 18 | } 19 | c.groups[groupName] = newStore(c.cfg) 20 | c.groups[groupName].name = groupName 21 | 22 | return c.groups[groupName] 23 | } 24 | -------------------------------------------------------------------------------- /group_test.go: -------------------------------------------------------------------------------- 1 | package konfig 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestGroup(t *testing.T) { 10 | t.Run( 11 | "test basic group", 12 | func(t *testing.T) { 13 | var c = newStore(DefaultConfig()) 14 | var g = c.Group("test") 15 | 16 | c.Set("foo", "bar") 17 | require.Equal(t, nil, g.Get("foo")) 18 | 19 | g.Set("foo", "bar") 20 | var gg = c.Group("test") 21 | require.Equal(t, "bar", gg.Get("foo")) 22 | }, 23 | ) 24 | } 25 | -------------------------------------------------------------------------------- /konfig.go: -------------------------------------------------------------------------------- 1 | // Package konfig provides a composable, observable and performant 2 | // config handling for Go. 3 | // Written for larger distributed systems where you may have plenty of 4 | // configuration sources - it allows you to compose configurations from 5 | // multiple sources with reload hooks making it simple to build apps that live 6 | // in a highly dynamic environment. 7 | // 8 | // Konfig is built around 4 small interfaces: Loader, Watcher, Parser, Closer 9 | // 10 | // Get started: 11 | // var configFiles = []klfile.File{ 12 | // { 13 | // Path: "./config.json", 14 | // Parser: kpjson.Parser, 15 | // }, 16 | // } 17 | // 18 | // func init() { 19 | // konfig.Init(konfig.DefaultConfig()) 20 | // } 21 | // 22 | // func main() { 23 | // // load from json file with a file wacher 24 | // konfig.RegisterLoaderWatcher( 25 | // klfile.New(&klfile.Config{ 26 | // Files: configFiles, 27 | // Watch: true, 28 | // }), 29 | // // optionally you can pass config hooks to run when a file is changed 30 | // func(c konfig.Store) error { 31 | // return nil 32 | // }, 33 | // ) 34 | // 35 | // // Load and start watching 36 | // if err := konfig.LoadWatch(); err != nil { 37 | // log.Fatal(err) 38 | // } 39 | // 40 | // // retrieve value from config file 41 | // konfig.Bool("debug") 42 | // } 43 | // 44 | // 45 | package konfig 46 | -------------------------------------------------------------------------------- /loader.go: -------------------------------------------------------------------------------- 1 | package konfig 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "sync" 7 | "time" 8 | 9 | "github.com/prometheus/client_golang/prometheus" 10 | ) 11 | 12 | // ErrNoLoaders is the error returned when no loaders are set in the config and Load is called 13 | var ErrNoLoaders = errors.New("No loaders in config") 14 | 15 | // Loader is the interface a config loader must implement to be used withint the package 16 | type Loader interface { 17 | // StopOnFailure tells whether a loader failure should lead to closing config and the registered closers. 18 | StopOnFailure() bool 19 | // Name returns the name of the loader 20 | Name() string 21 | // Load loads config values in a Values 22 | Load(Values) error 23 | // MaxRetry returns the max number of times to retry when Load fails 24 | MaxRetry() int 25 | // RetryDelay returns the delay between each retry 26 | RetryDelay() time.Duration 27 | } 28 | 29 | // LoaderHooks are functions ran when a config load has been performed 30 | type LoaderHooks []func(Store) error 31 | 32 | // Run runs all loader and stops when it encounters an error 33 | func (l LoaderHooks) Run(cfg Store) error { 34 | for _, h := range l { 35 | if err := h(cfg); err != nil { 36 | return err 37 | } 38 | } 39 | return nil 40 | } 41 | 42 | // LoadWatch loads the config then starts watching it 43 | func LoadWatch() error { 44 | return instance().LoadWatch() 45 | } 46 | 47 | // LoadWatch loads the config then starts watching it 48 | func (c *S) LoadWatch() error { 49 | if err := c.Load(); err != nil { 50 | return err 51 | } else if err := c.Watch(); err != nil { 52 | return err 53 | } 54 | return nil 55 | } 56 | 57 | // Load is a function running load on the global config instance 58 | func Load() error { 59 | return instance().Load() 60 | } 61 | 62 | // Load is a function running load on the global config instance 63 | func (c *S) Load() error { 64 | if len(c.WatcherLoaders) == 0 { 65 | panic(ErrNoLoaders) 66 | } 67 | for _, l := range c.WatcherLoaders { 68 | // we load the loader once, then we start the reload worker with the watcher 69 | if err := c.loaderLoadRetry(l, 0); err != nil { 70 | 71 | // if loader says we should stop in failure, stop the world 72 | // else just return the error 73 | if l.StopOnFailure() { 74 | c.stop() 75 | } 76 | 77 | return err 78 | } 79 | } 80 | 81 | // now that we've loaded everything, let's check strict keys 82 | if err := c.checkStrictKeys(); err != nil { 83 | c.cfg.Logger.Get().Error("Error while checking strict keys: " + err.Error()) 84 | return err 85 | } 86 | c.loaded = true 87 | 88 | return nil 89 | } 90 | 91 | // ConfigLoader is a wrapper of Loader with methods to add hooks 92 | type ConfigLoader struct { 93 | *loaderWatcher 94 | mut *sync.Mutex 95 | } 96 | 97 | func (c *S) newConfigLoader(lw *loaderWatcher) *ConfigLoader { 98 | var cl = &ConfigLoader{ 99 | loaderWatcher: lw, 100 | mut: &sync.Mutex{}, 101 | } 102 | 103 | return cl 104 | } 105 | 106 | // AddHooks adds hooks to the loader 107 | func (cl *ConfigLoader) AddHooks(loaderHooks ...func(Store) error) *ConfigLoader { 108 | cl.mut.Lock() 109 | defer cl.mut.Unlock() 110 | 111 | if cl.loaderWatcher.loaderHooks == nil { 112 | cl.loaderWatcher.loaderHooks = make(LoaderHooks, 0) 113 | } 114 | 115 | cl.loaderWatcher.loaderHooks = append( 116 | cl.loaderWatcher.loaderHooks, 117 | loaderHooks..., 118 | ) 119 | 120 | return cl 121 | } 122 | 123 | // We don't look for Done on the watcher here as the NopWatcher needs to run load at least once 124 | func (c *S) loaderLoadRetry(wl *loaderWatcher, retry int) error { 125 | // we create a new Values 126 | var v = make(Values, len(wl.values)) 127 | 128 | // we call the loader 129 | if err := wl.Load(v); err != nil { 130 | 131 | c.cfg.Logger.Get().Error(fmt.Sprintf( 132 | "Error %d in loader %s: %s", 133 | retry, 134 | wl.Name(), 135 | err.Error(), 136 | )) 137 | 138 | if retry >= wl.MaxRetry() { 139 | return err 140 | } 141 | 142 | // wait before retrying 143 | time.Sleep(wl.RetryDelay()) 144 | 145 | return c.loaderLoadRetry(wl, retry+1) 146 | } 147 | 148 | // we add the values to the store. 149 | var updatedKeys, err = v.load(wl.values, c) 150 | if err != nil { 151 | return err 152 | } 153 | 154 | wl.values = v 155 | 156 | // run key hooks 157 | if len(updatedKeys) != 0 && c.keyHooks != nil { 158 | return c.keyHooks.runForKeys(updatedKeys, c) 159 | } 160 | 161 | // run the loader hooks 162 | if wl.loaderHooks != nil { 163 | c.mut.Lock() 164 | if err := wl.loaderHooks.Run(c); err != nil { 165 | c.cfg.Logger.Get().Error("Error while running loader hooks: " + err.Error()) 166 | c.mut.Unlock() 167 | return err 168 | } 169 | c.mut.Unlock() 170 | } 171 | 172 | return nil 173 | } 174 | 175 | func (c *S) watchLoader(wl *loaderWatcher, panics int) { 176 | // if a panic occurs we log it 177 | // then, if the current loader requires a to stop on failure, we stop everything, 178 | // else, we restart the watcher. 179 | defer func() { 180 | if r := recover(); r != nil { 181 | c.cfg.Logger.Get().Error( 182 | fmt.Sprintf( 183 | "Panic %d in loader %s: %v", 184 | panics, 185 | wl.Name(), 186 | r, 187 | ), 188 | ) 189 | if wl.StopOnFailure() || panics >= c.cfg.MaxWatcherPanics { 190 | c.stop() 191 | return 192 | } 193 | go c.watchLoader(wl, panics+1) 194 | } 195 | }() 196 | 197 | for { 198 | select { 199 | case <-wl.Done(): 200 | if err := wl.Err(); err != nil { 201 | c.cfg.Logger.Get().Error(err.Error()) 202 | } 203 | // the watcher is closed 204 | return 205 | case <-wl.Watch(): 206 | // we got an event 207 | // do a loaderLoadRetry 208 | select { 209 | case <-wl.Done(): 210 | if err := wl.Err(); err != nil { 211 | c.cfg.Logger.Get().Error(err.Error()) 212 | } 213 | return 214 | default: 215 | 216 | var t *prometheus.Timer 217 | if c.cfg.Metrics { 218 | t = prometheus.NewTimer(wl.metrics.configReloadDuration) 219 | } 220 | 221 | if err := c.loaderLoadRetry(wl, 0); err != nil { 222 | // if metrics is enabled we record a load failure 223 | if c.cfg.Metrics { 224 | wl.metrics.configReloadFailure.Inc() 225 | t.ObserveDuration() 226 | } 227 | if !wl.StopOnFailure() { 228 | continue 229 | } 230 | c.stop() 231 | return 232 | } 233 | 234 | if c.cfg.Metrics { 235 | t.ObserveDuration() 236 | wl.metrics.configReloadSuccess.Inc() 237 | } 238 | } 239 | } 240 | } 241 | } 242 | -------------------------------------------------------------------------------- /loader/klconsul/README.md: -------------------------------------------------------------------------------- 1 | # Consul loader 2 | Loads config from consul KV. 3 | 4 | 5 | # Usage 6 | 7 | Basic usage loading keys and using result as string with watcher 8 | ```go 9 | etcdLoader := klconsul.New(&klconsul.Config{ 10 | Client: consulClient, // from github.com/hashicorp/consul/api package 11 | Keys: []Key{ 12 | { 13 | Key: "foo", 14 | }, 15 | }, 16 | Watch: true, 17 | }) 18 | ``` 19 | 20 | Loading keys and JSON parser 21 | ```go 22 | consulLoader := klconsul.New(&klconsul.Config{ 23 | Client: consulClient, // from github.com/hashicorp/consul/api package 24 | Keys: []Key{ 25 | { 26 | Key: "foo", 27 | Parser: kpjson.Parser, 28 | }, 29 | }, 30 | Watch: true, 31 | }) 32 | ``` 33 | 34 | # Strict mode 35 | If strict mode is enabled, a key defined in the config but missing in consul will trigger an error. 36 | -------------------------------------------------------------------------------- /loader/klconsul/consulloader.go: -------------------------------------------------------------------------------- 1 | package klconsul 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "fmt" 7 | "os" 8 | "time" 9 | 10 | "github.com/hashicorp/consul/api" 11 | "github.com/lalamove/konfig" 12 | "github.com/lalamove/konfig/parser" 13 | "github.com/lalamove/konfig/watcher/kwpoll" 14 | "github.com/lalamove/nui/nlogger" 15 | "github.com/lalamove/nui/nstrings" 16 | ) 17 | 18 | var ( 19 | defaultTimeout = 5 * time.Second 20 | _ konfig.Loader = (*Loader)(nil) 21 | ) 22 | 23 | const ( 24 | defaultName = "consul" 25 | ) 26 | 27 | // Key is an Consul Key to load 28 | type Key struct { 29 | // Key is the consul key 30 | Key string 31 | // Parser is the parser for the key 32 | // If nil, the value is casted to a string before adding to the config.Store 33 | Parser parser.Parser 34 | // QueryOptions is the query options to pass when retrieving the key from consul 35 | QueryOptions *api.QueryOptions 36 | } 37 | 38 | // ConsulKV is an interface that consul client.KV implements. It is used to retrieve keys. 39 | type ConsulKV interface { 40 | Get(key string, q *api.QueryOptions) (*api.KVPair, *api.QueryMeta, error) 41 | } 42 | 43 | // Config is the structure representing the config of a Loader 44 | type Config struct { 45 | // Name is the name of the loader 46 | Name string 47 | // Client is the consul KV client 48 | Client *api.Client 49 | // StopOnFailure tells whether a load failure(after the retries) leads to closing the config and all registered closers 50 | StopOnFailure bool 51 | // Keys is the list of keys to fetch 52 | Keys []Key 53 | // Timeout is the timeout duration when fetching a key 54 | Timeout time.Duration 55 | // Prefix is a prefix to prepend keys when adding into the konfig.Store 56 | Prefix string 57 | // Replacer is a Replacer for the key before adding to the konfig.Store 58 | Replacer nstrings.Replacer 59 | // Watch tells if there should be a watcher with the loader 60 | Watch bool 61 | // Rater is the rater to pass to the poll watcher 62 | Rater kwpoll.Rater 63 | // MaxRetry is the maximum number of times we can retry to load if it fails 64 | MaxRetry int 65 | // RetryDelay is the time between each retry when a load fails 66 | RetryDelay time.Duration 67 | // Debug sets debug mode on the consulloader 68 | Debug bool 69 | // Logger is used across this package to produce logs 70 | Logger nlogger.Provider 71 | // StrictMode will raise error if key was not found 72 | // In false state, konfig will try to reload desired key(s) 73 | // up until they are not found 74 | StrictMode bool 75 | 76 | kvClient ConsulKV 77 | } 78 | 79 | // Loader is the structure of a loader 80 | type Loader struct { 81 | *kwpoll.PollWatcher 82 | cfg *Config 83 | } 84 | 85 | // New returns a new loader with the given config 86 | func New(cfg *Config) *Loader { 87 | if cfg.Timeout == 0 { 88 | cfg.Timeout = defaultTimeout 89 | } 90 | 91 | if cfg.Client == nil { 92 | panic(errors.New("no consul client was provided")) 93 | } 94 | 95 | if cfg.Logger == nil { 96 | cfg.Logger = defaultLogger() 97 | } 98 | 99 | if cfg.Name == "" { 100 | cfg.Name = defaultName 101 | } 102 | 103 | if cfg.kvClient == nil { 104 | cfg.kvClient = cfg.Client.KV() 105 | } 106 | 107 | var l = &Loader{ 108 | cfg: cfg, 109 | } 110 | 111 | if cfg.Watch { 112 | var v = konfig.Values{} 113 | // we don't want to kill the process if there is an error 114 | // in the config 115 | if err := l.Load(v); err != nil { 116 | cfg.Logger.Get().Error(fmt.Sprintf("Can't read provided config: %v", err)) 117 | } 118 | 119 | l.PollWatcher = kwpoll.New(&kwpoll.Config{ 120 | Loader: l, 121 | Rater: cfg.Rater, 122 | InitValue: v, 123 | Diff: true, 124 | Debug: cfg.Debug, 125 | }) 126 | } 127 | 128 | return l 129 | } 130 | 131 | // Name returns the name of the loader 132 | func (l *Loader) Name() string { return l.cfg.Name } 133 | 134 | // Load implements konfig.Loader, 135 | // it loads environment variables into the konfig.Store 136 | // based on config passed to the loader 137 | func (l *Loader) Load(s konfig.Values) error { 138 | for _, k := range l.cfg.Keys { 139 | kp, _, err := l.keyValue(k.Key) 140 | if err != nil { 141 | return err 142 | } 143 | if kp == nil && l.cfg.StrictMode { 144 | return fmt.Errorf("provided key \"%v\" was not found", k.Key) 145 | } else if kp == nil { 146 | l.cfg.Logger.Get().Warn(fmt.Sprintf("provided key \"%v\" was not found", k.Key)) 147 | return nil 148 | } 149 | 150 | var configKey = l.cfg.Prefix + string(kp.Key) 151 | if l.cfg.Replacer != nil { 152 | configKey = l.cfg.Replacer.Replace(configKey) 153 | } 154 | 155 | // if the key has a parser, we parse the key value using the provided Parser 156 | // else we just convert the value to a string 157 | if k.Parser != nil { 158 | if err := k.Parser.Parse(bytes.NewReader(kp.Value), s); err != nil { 159 | return err 160 | } 161 | } else { 162 | s.Set(configKey, string(kp.Value)) 163 | } 164 | } 165 | 166 | return nil 167 | } 168 | 169 | // MaxRetry is the maximum number of time to retry when a load fails 170 | func (l *Loader) MaxRetry() int { 171 | return l.cfg.MaxRetry 172 | } 173 | 174 | // RetryDelay is the delay between each retry 175 | func (l *Loader) RetryDelay() time.Duration { 176 | return l.cfg.RetryDelay 177 | } 178 | 179 | // StopOnFailure returns whether a load failure should stop the config and the registered closers 180 | func (l *Loader) StopOnFailure() bool { 181 | return l.cfg.StopOnFailure 182 | } 183 | 184 | // keyValue is a quick helper to load KVPair from 185 | // the consul server 186 | func (l *Loader) keyValue(k string) (pair *api.KVPair, qm *api.QueryMeta, err error) { 187 | return l.cfg.kvClient.Get(k, nil) 188 | } 189 | 190 | func defaultLogger() nlogger.Provider { 191 | return nlogger.NewProvider(nlogger.New(os.Stdout, "CONSULLOADER | ")) 192 | } 193 | -------------------------------------------------------------------------------- /loader/klconsul/consulloader_integration_test.go: -------------------------------------------------------------------------------- 1 | // +build integration 2 | 3 | package klconsul 4 | 5 | import ( 6 | "testing" 7 | 8 | "github.com/golang/mock/gomock" 9 | "github.com/hashicorp/consul/api" 10 | "github.com/hashicorp/consul/sdk/testutil" 11 | "github.com/lalamove/konfig" 12 | "github.com/stretchr/testify/require" 13 | ) 14 | 15 | func TestIntegrationLoad(t *testing.T) { 16 | srv, err := testutil.NewTestServerConfigT(t, func(c *testutil.TestServerConfig) { 17 | c.LogLevel = "err" 18 | }) 19 | if err != nil { 20 | t.Fatal(err) 21 | } 22 | defer srv.Stop() 23 | 24 | var testCases = []struct { 25 | name string 26 | setUp func(ctrl *gomock.Controller) *Loader 27 | err bool 28 | }{ 29 | { 30 | name: "key exists, no panic, no errors", 31 | setUp: func(ctrl *gomock.Controller) *Loader { 32 | c, _ := api.NewClient(&api.Config{Address: srv.HTTPAddr}) 33 | 34 | var hl = New(&Config{ 35 | Client: c, 36 | Keys: []Key{{ 37 | Key: "foo", 38 | }}, 39 | }) 40 | 41 | srv.SetKV(t, "foo", []byte("bar")) 42 | 43 | kv, _, err := c.KV().Get("foo", nil) 44 | if err != nil { 45 | t.Fatal(err) 46 | } 47 | 48 | require.Equal(t, []byte("bar"), kv.Value) 49 | 50 | return hl 51 | }, 52 | err: false, 53 | }, 54 | { 55 | name: "strict mode on, key doesn't exist, no panic, error", 56 | setUp: func(ctrl *gomock.Controller) *Loader { 57 | c, _ := api.NewClient(&api.Config{Address: srv.HTTPAddr}) 58 | 59 | var hl = New(&Config{ 60 | Client: c, 61 | StrictMode: true, 62 | Keys: []Key{{ 63 | Key: "bar", 64 | }}, 65 | }) 66 | 67 | kv, _, err := c.KV().Get("bar", nil) 68 | if err != nil { 69 | t.Fatal(err) 70 | } 71 | 72 | require.Nil(t, kv) 73 | 74 | return hl 75 | }, 76 | err: true, 77 | }, 78 | { 79 | name: "strict mode off, key doesn't exist, no panic, error", 80 | setUp: func(ctrl *gomock.Controller) *Loader { 81 | c, _ := api.NewClient(&api.Config{Address: srv.HTTPAddr}) 82 | 83 | var hl = New(&Config{ 84 | Client: c, 85 | StrictMode: false, 86 | Keys: []Key{{ 87 | Key: "bar", 88 | }}, 89 | }) 90 | 91 | kv, _, err := c.KV().Get("bar", nil) 92 | if err != nil { 93 | t.Fatal(err) 94 | } 95 | 96 | require.Nil(t, kv) 97 | 98 | return hl 99 | }, 100 | err: false, 101 | }, 102 | { 103 | name: "multiple keys, no panic, no error", 104 | setUp: func(ctrl *gomock.Controller) *Loader { 105 | c, _ := api.NewClient(&api.Config{Address: srv.HTTPAddr}) 106 | 107 | var hl = New(&Config{ 108 | Client: c, 109 | StrictMode: false, 110 | Keys: []Key{ 111 | { 112 | Key: "key1", 113 | }, 114 | { 115 | Key: "key2", 116 | }}, 117 | }) 118 | 119 | srv.SetKV(t, "key1", []byte("test")) 120 | srv.SetKV(t, "key2", []byte("test")) 121 | 122 | kv1, _, err := c.KV().Get("key1", nil) 123 | if err != nil { 124 | t.Fatal(err) 125 | } 126 | 127 | kv2, _, err := c.KV().Get("key2", nil) 128 | if err != nil { 129 | t.Fatal(err) 130 | } 131 | 132 | require.Equal(t, []byte("test"), kv1.Value) 133 | require.Equal(t, []byte("test"), kv2.Value) 134 | 135 | return hl 136 | }, 137 | err: false, 138 | }, 139 | } 140 | 141 | for _, testCase := range testCases { 142 | t.Run( 143 | testCase.name, 144 | func(t *testing.T) { 145 | var ctrl = gomock.NewController(t) 146 | defer ctrl.Finish() 147 | 148 | konfig.Init(konfig.DefaultConfig()) 149 | var hl = testCase.setUp(ctrl) 150 | 151 | var err = hl.Load(konfig.Values{}) 152 | if testCase.err { 153 | require.NotNil(t, err, "err should not be nil") 154 | return 155 | } 156 | require.Nil(t, err, "err should be nil") 157 | }, 158 | ) 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /loader/klconsul/consulloader_test.go: -------------------------------------------------------------------------------- 1 | package klconsul 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | "time" 7 | 8 | "github.com/golang/mock/gomock" 9 | "github.com/hashicorp/consul/api" 10 | "github.com/lalamove/konfig" 11 | "github.com/lalamove/konfig/mocks" 12 | "github.com/lalamove/konfig/parser" 13 | "github.com/lalamove/konfig/watcher/kwpoll" 14 | "github.com/lalamove/nui/nstrings" 15 | "github.com/stretchr/testify/require" 16 | ) 17 | 18 | func TestLoad(t *testing.T) { 19 | var testCases = []struct { 20 | name string 21 | run func(t *testing.T, ctrl *gomock.Controller) *Loader 22 | err bool 23 | }{ 24 | { 25 | name: "key exists, no panic, no errors", 26 | run: func(t *testing.T, ctrl *gomock.Controller) *Loader { 27 | c, _ := api.NewClient(&api.Config{Address: "http://localhost"}) 28 | 29 | var hl = New(&Config{ 30 | Client: c, 31 | Keys: []Key{{ 32 | Key: "foo", 33 | }}, 34 | Replacer: nstrings.ReplacerToUpper, 35 | }) 36 | 37 | var kvClient = mocks.NewMockConsulKV(ctrl) 38 | kvClient.EXPECT().Get("foo", nil).Times(1).Return( 39 | &api.KVPair{ 40 | Key: "foo", 41 | Value: []byte(`bar`), 42 | }, 43 | &api.QueryMeta{}, 44 | nil, 45 | ) 46 | 47 | hl.cfg.kvClient = kvClient 48 | 49 | var v = konfig.Values{} 50 | var err = hl.Load(v) 51 | 52 | if err != nil { 53 | t.Fatal(err) 54 | } 55 | require.Equal(t, "bar", v["FOO"]) 56 | return hl 57 | }, 58 | err: false, 59 | }, 60 | { 61 | name: "strict mode off, key doesn't exist, no panic, no error", 62 | run: func(t *testing.T, ctrl *gomock.Controller) *Loader { 63 | c, _ := api.NewClient(&api.Config{Address: "http://localhost"}) 64 | 65 | var hl = New(&Config{ 66 | Client: c, 67 | StrictMode: false, 68 | Keys: []Key{{ 69 | Key: "bar", 70 | }}, 71 | }) 72 | 73 | var kvClient = mocks.NewMockConsulKV(ctrl) 74 | kvClient.EXPECT().Get("bar", nil).Return( 75 | nil, 76 | nil, 77 | nil, 78 | ) 79 | hl.cfg.kvClient = kvClient 80 | 81 | var v = konfig.Values{} 82 | var err = hl.Load(v) 83 | 84 | if err != nil { 85 | t.Fatal(err) 86 | } 87 | 88 | var _, ok = v["bar"] 89 | 90 | require.False(t, ok) 91 | 92 | return hl 93 | }, 94 | err: true, 95 | }, 96 | { 97 | name: "strict mode on, key doesn't exist, no panic, error", 98 | run: func(t *testing.T, ctrl *gomock.Controller) *Loader { 99 | c, _ := api.NewClient(&api.Config{Address: "http://localhost"}) 100 | 101 | var hl = New(&Config{ 102 | Client: c, 103 | StrictMode: true, 104 | Keys: []Key{{ 105 | Key: "bar", 106 | }}, 107 | }) 108 | 109 | var kvClient = mocks.NewMockConsulKV(ctrl) 110 | kvClient.EXPECT().Get("bar", nil).Return( 111 | nil, 112 | nil, 113 | nil, 114 | ) 115 | hl.cfg.kvClient = kvClient 116 | 117 | var v = konfig.Values{} 118 | var err = hl.Load(v) 119 | 120 | require.NotNil(t, err, "err should not be nil as key was not found and strict mode is off") 121 | 122 | return hl 123 | }, 124 | err: false, 125 | }, 126 | { 127 | name: "multiple keys, no panic, no error", 128 | run: func(t *testing.T, ctrl *gomock.Controller) *Loader { 129 | c, _ := api.NewClient(&api.Config{Address: "http://localhost"}) 130 | 131 | var hl = New(&Config{ 132 | Client: c, 133 | StrictMode: false, 134 | Keys: []Key{ 135 | { 136 | Key: "key1", 137 | }, 138 | { 139 | Key: "key2", 140 | }}, 141 | }) 142 | 143 | var kvClient = mocks.NewMockConsulKV(ctrl) 144 | kvClient.EXPECT().Get("key1", nil).Return( 145 | &api.KVPair{ 146 | Key: "key1", 147 | Value: []byte(`test1`), 148 | }, 149 | &api.QueryMeta{}, 150 | nil, 151 | ) 152 | 153 | kvClient.EXPECT().Get("key2", nil).Return( 154 | &api.KVPair{ 155 | Key: "key2", 156 | Value: []byte(`test2`), 157 | }, 158 | &api.QueryMeta{}, 159 | nil, 160 | ) 161 | 162 | hl.cfg.kvClient = kvClient 163 | 164 | var v = konfig.Values{} 165 | var err = hl.Load(v) 166 | 167 | if err != nil { 168 | t.Fatal(err) 169 | } 170 | require.Equal(t, "test1", v["key1"]) 171 | require.Equal(t, "test2", v["key2"]) 172 | 173 | return hl 174 | }, 175 | err: false, 176 | }, 177 | { 178 | name: "with watcher no error multiple keys", 179 | run: func(t *testing.T, ctrl *gomock.Controller) *Loader { 180 | 181 | c, _ := api.NewClient(&api.Config{Address: "http://localhost"}) 182 | 183 | var kvClient = mocks.NewMockConsulKV(ctrl) 184 | 185 | gomock.InOrder( 186 | kvClient.EXPECT().Get("key1", nil).Return( 187 | &api.KVPair{ 188 | Key: "key1", 189 | Value: []byte(`test1`), 190 | }, 191 | &api.QueryMeta{}, 192 | nil, 193 | ), 194 | kvClient.EXPECT().Get("key2", nil).Return( 195 | &api.KVPair{ 196 | Key: "key2", 197 | Value: []byte(`test2`), 198 | }, 199 | &api.QueryMeta{}, 200 | nil, 201 | ), 202 | kvClient.EXPECT().Get("key1", nil).Return( 203 | &api.KVPair{ 204 | Key: "key1", 205 | Value: []byte(`test11`), 206 | }, 207 | &api.QueryMeta{}, 208 | nil, 209 | ), 210 | kvClient.EXPECT().Get("key2", nil).Return( 211 | &api.KVPair{ 212 | Key: "key2", 213 | Value: []byte(`test22`), 214 | }, 215 | &api.QueryMeta{}, 216 | nil, 217 | ), 218 | ) 219 | 220 | var hl = New(&Config{ 221 | Client: c, 222 | kvClient: kvClient, 223 | StrictMode: false, 224 | Watch: true, 225 | Rater: kwpoll.Time(100 * time.Millisecond), 226 | Keys: []Key{ 227 | { 228 | Key: "key1", 229 | }, 230 | { 231 | Key: "key2", 232 | }}, 233 | }) 234 | 235 | err := hl.Start() 236 | require.Nil(t, err) 237 | 238 | var timer = time.NewTimer(150 * time.Millisecond) 239 | var watched bool 240 | outer: 241 | for { 242 | select { 243 | case <-hl.Watch(): 244 | watched = true 245 | hl.Close() 246 | case <-timer.C: 247 | hl.Close() 248 | require.True(t, watched) 249 | break outer 250 | } 251 | } 252 | 253 | return hl 254 | }, 255 | }, 256 | { 257 | name: "parse value fail", 258 | run: func(t *testing.T, ctrl *gomock.Controller) *Loader { 259 | c, _ := api.NewClient(&api.Config{Address: "http://localhost"}) 260 | 261 | var hl = New(&Config{ 262 | Client: c, 263 | Keys: []Key{{ 264 | Key: "foo", 265 | Parser: parser.NopParser{Err: errors.New("parse fail")}, 266 | }}, 267 | }) 268 | 269 | var kvClient = mocks.NewMockConsulKV(ctrl) 270 | kvClient.EXPECT().Get("foo", nil).Times(1).Return( 271 | &api.KVPair{ 272 | Key: "foo", 273 | Value: []byte(`bar`), 274 | }, 275 | &api.QueryMeta{}, 276 | nil, 277 | ) 278 | 279 | hl.cfg.kvClient = kvClient 280 | 281 | var v = konfig.Values{} 282 | var err = hl.Load(v) 283 | require.Error(t, err) 284 | return hl 285 | }, 286 | err: true, 287 | }, 288 | } 289 | 290 | for _, testCase := range testCases { 291 | t.Run( 292 | testCase.name, 293 | func(t *testing.T) { 294 | var ctrl = gomock.NewController(t) 295 | defer ctrl.Finish() 296 | 297 | konfig.Init(konfig.DefaultConfig()) 298 | testCase.run(t, ctrl) 299 | }, 300 | ) 301 | } 302 | } 303 | 304 | func TestNew(t *testing.T) { 305 | t.Run( 306 | "no client panics", 307 | func(t *testing.T) { 308 | require.Panics( 309 | t, 310 | func() { 311 | New(&Config{}) 312 | }, 313 | ) 314 | }, 315 | ) 316 | } 317 | 318 | func TestLoaderMethods(t *testing.T) { 319 | konfig.Init(konfig.DefaultConfig()) 320 | var ctrl = gomock.NewController(t) 321 | defer ctrl.Finish() 322 | 323 | client, _ := api.NewClient(&api.Config{}) 324 | 325 | var l = New(&Config{ 326 | Name: "consulloader", 327 | MaxRetry: 3, 328 | RetryDelay: 10 * time.Second, 329 | StopOnFailure: true, 330 | Client: client, 331 | Keys: []Key{{Key: "key1"}}, 332 | }) 333 | 334 | require.True(t, l.StopOnFailure()) 335 | require.Equal(t, "consulloader", l.Name()) 336 | require.Equal(t, 3, l.MaxRetry()) 337 | require.Equal(t, 10*time.Second, l.RetryDelay()) 338 | } 339 | -------------------------------------------------------------------------------- /loader/klenv/README.md: -------------------------------------------------------------------------------- 1 | # Env Loader 2 | Env loader loads environment variables in a konfig.Store 3 | 4 | # Usage 5 | 6 | Basic usage loading all environment variables 7 | ```go 8 | envLoader := klenv.New(&klenv.Config{}) 9 | ``` 10 | 11 | Loading specific variables 12 | ```go 13 | envLoader := klenv.New(&klenv.Config{ 14 | Vars: []string{ 15 | "DEBUG", 16 | "PORT", 17 | }, 18 | }) 19 | ``` 20 | 21 | Loading specific variables if key matches regexp 22 | ```go 23 | envLoader := klenv.New(&klenv.Config{ 24 | Regexp: "^APP_.*" 25 | }) 26 | ``` 27 | 28 | With a replacer and a Prefix for keys 29 | ```go 30 | envLoader := klenv.New(&klenv.Config{ 31 | Prefix: "config.", 32 | Replacer: nstrings.ReplacerToLower, 33 | }) 34 | ``` 35 | 36 | Loading value as string slice 37 | ```go 38 | os.Setenv("APP_VAR1", "value1,value2") // will be loaded as string slice 39 | os.Setenv("APP_VAR2", "value") // will be loaded as string 40 | 41 | envLoader := klenv.New(&klenv.Config{ 42 | Regexp: "^APP_.*", 43 | SliceSeparator: ",", 44 | }) 45 | 46 | ... 47 | 48 | fmt.Printf("%+v\n", store.Get("VAR1")) // Output: []string{"value1","value2} 49 | fmt.Printf("%+v\n", store.Get("VAR2")) // Output: value 50 | ``` 51 | -------------------------------------------------------------------------------- /loader/klenv/envloader.go: -------------------------------------------------------------------------------- 1 | package klenv 2 | 3 | import ( 4 | "os" 5 | "regexp" 6 | "strings" 7 | "time" 8 | 9 | "github.com/lalamove/konfig" 10 | "github.com/lalamove/nui/nstrings" 11 | ) 12 | 13 | var ( 14 | _ konfig.Loader = (*Loader)(nil) 15 | ) 16 | 17 | const ( 18 | sepEnvVar = "=" 19 | defaultName = "env" 20 | ) 21 | 22 | // Config is the config a an EnvLoader 23 | type Config struct { 24 | // Name is the name of the loader 25 | Name string 26 | // StopOnFailure tells whether a failure to load configs should closed the config and all registered closers 27 | StopOnFailure bool 28 | // Regexp will load the environment variable if it matches the given regexp 29 | Regexp string 30 | // Vars will load vars only present in the vars slice 31 | Vars []string 32 | // Prefix will add a prefix to the environment variables when adding them in the config store 33 | Prefix string 34 | // Replacer is used to replace chars in env vars keys 35 | Replacer nstrings.Replacer 36 | // MaxRetry is the maximum number of time the load method can be retried when it fails 37 | MaxRetry int 38 | // RetryDelay is the time betweel each retry 39 | RetryDelay time.Duration 40 | // SliceSeparator contains separator for values like `item1,item2,item3`. 41 | // Such values will be loaded as string slice if separator is not empty. 42 | SliceSeparator string 43 | } 44 | 45 | // Loader is the structure representing the environment loader 46 | type Loader struct { 47 | cfg *Config 48 | r *regexp.Regexp 49 | } 50 | 51 | // New return a new environment loader with the given config 52 | func New(cfg *Config) *Loader { 53 | var r *regexp.Regexp 54 | if cfg.Regexp != "" { 55 | r = regexp.MustCompile(cfg.Regexp) 56 | } 57 | 58 | if cfg.Name == "" { 59 | cfg.Name = defaultName 60 | } 61 | 62 | return &Loader{ 63 | cfg, 64 | r, 65 | } 66 | } 67 | 68 | // Name returns the name of the loader 69 | func (l *Loader) Name() string { return l.cfg.Name } 70 | 71 | func (l *Loader) convertValue(v string) interface{} { 72 | if l.cfg.SliceSeparator != "" { 73 | // do not load value as slice if it contains only one item 74 | // to avoid situation when all string values will be string slice values 75 | // binding mechanism correctly works when we try to bind one string to string slice 76 | if strings.Contains(v, l.cfg.SliceSeparator) { 77 | return strings.Split(v, l.cfg.SliceSeparator) 78 | } 79 | } 80 | 81 | return v 82 | } 83 | 84 | // Load implements konfig.Loader, it loads environment variables into the konfig.Store 85 | // based on config passed to the loader 86 | func (l *Loader) Load(s konfig.Values) error { 87 | if l.cfg.Vars != nil && len(l.cfg.Vars) > 0 { 88 | return l.loadVars(s) 89 | } 90 | for _, v := range os.Environ() { 91 | var spl = strings.SplitN(v, sepEnvVar, 2) 92 | // if has regex and key does not macth regexp we continue 93 | if l.r != nil && !l.r.MatchString(spl[0]) { 94 | continue 95 | } 96 | var k = spl[0] 97 | if l.cfg.Replacer != nil { 98 | k = l.cfg.Replacer.Replace(k) 99 | } 100 | k = l.cfg.Prefix + k 101 | s.Set(k, l.convertValue(spl[1])) 102 | } 103 | 104 | return nil 105 | } 106 | 107 | // MaxRetry returns the maximum number to retry a load when an error occurs 108 | func (l *Loader) MaxRetry() int { 109 | return l.cfg.MaxRetry 110 | } 111 | 112 | // RetryDelay returns the delay between each load retry 113 | func (l *Loader) RetryDelay() time.Duration { 114 | return l.cfg.RetryDelay 115 | } 116 | 117 | func (l *Loader) loadVars(s konfig.Values) error { 118 | for _, k := range l.cfg.Vars { 119 | var v = os.Getenv(k) 120 | if l.cfg.Replacer != nil { 121 | k = l.cfg.Replacer.Replace(k) 122 | } 123 | k = l.cfg.Prefix + k 124 | s.Set(k, l.convertValue(v)) 125 | } 126 | return nil 127 | } 128 | 129 | // StopOnFailure returns whether a load failure should stop the config and the registered closers 130 | func (l *Loader) StopOnFailure() bool { 131 | return l.cfg.StopOnFailure 132 | } 133 | -------------------------------------------------------------------------------- /loader/klenv/envloader_test.go: -------------------------------------------------------------------------------- 1 | package klenv 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | "time" 7 | 8 | "github.com/lalamove/konfig" 9 | "github.com/lalamove/nui/nstrings" 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | func TestEnvLoader(t *testing.T) { 14 | t.Run( 15 | "load defined env vars", 16 | func(t *testing.T) { 17 | os.Setenv("FOO", "BAR") 18 | os.Setenv("BAR", "FOO") 19 | 20 | var l = New(&Config{ 21 | Vars: []string{ 22 | "FOO", 23 | "BAR", 24 | }, 25 | Replacer: nstrings.ReplacerToLower, 26 | }) 27 | 28 | var v = konfig.Values{} 29 | l.Load(v) 30 | 31 | require.Equal(t, "BAR", v["foo"]) 32 | require.Equal(t, "FOO", v["bar"]) 33 | require.Equal(t, defaultName, l.Name()) 34 | }, 35 | ) 36 | 37 | t.Run( 38 | "load string slice values from defined env vars", 39 | func(t *testing.T) { 40 | os.Setenv("FOO", "BAR1,BAR2,BAR3") 41 | os.Setenv("BAR", "FOO") // we should get "string" value in store 42 | 43 | var l = New(&Config{ 44 | Vars: []string{ 45 | "FOO", 46 | "BAR", 47 | }, 48 | SliceSeparator: ",", 49 | }) 50 | 51 | var v = konfig.Values{} 52 | l.Load(v) 53 | 54 | require.Equal(t, []string{"BAR1", "BAR2", "BAR3"}, v["FOO"]) 55 | require.Equal(t, "FOO", v["BAR"]) 56 | }, 57 | ) 58 | 59 | t.Run( 60 | "load env vars regexp", 61 | func(t *testing.T) { 62 | konfig.Init(konfig.DefaultConfig()) 63 | 64 | os.Setenv("FOO", "BAR") 65 | os.Setenv("BAR", "FOO") 66 | 67 | var l = New(&Config{ 68 | Regexp: "^F.*", 69 | }) 70 | 71 | var v = konfig.Values{} 72 | l.Load(v) 73 | 74 | require.Equal(t, "BAR", v["FOO"]) 75 | var _, ok = v["BAR"] 76 | require.False(t, ok) 77 | }, 78 | ) 79 | 80 | t.Run( 81 | "load string slice values from env vars regexp", 82 | func(t *testing.T) { 83 | konfig.Init(konfig.DefaultConfig()) 84 | 85 | os.Setenv("FOO", "BAR1,BAR2,BAR3") 86 | os.Setenv("FAA", "VAL") 87 | os.Setenv("BAR", "FOO") 88 | 89 | var l = New(&Config{ 90 | Regexp: "^F.*", 91 | SliceSeparator: ",", 92 | }) 93 | 94 | var v = konfig.Values{} 95 | l.Load(v) 96 | 97 | require.Equal(t, []string{"BAR1", "BAR2", "BAR3"}, v["FOO"]) 98 | require.Equal(t, "VAL", v["FAA"]) 99 | var _, ok = v["BAR"] 100 | require.False(t, ok) 101 | }, 102 | ) 103 | 104 | t.Run( 105 | "load env vars prefix regexp", 106 | func(t *testing.T) { 107 | konfig.Init(konfig.DefaultConfig()) 108 | 109 | os.Setenv("FOO", "BAR") 110 | os.Setenv("BAR", "FOO") 111 | 112 | var l = New(&Config{ 113 | Regexp: "^F.*", 114 | Prefix: "KONFIG_", 115 | }) 116 | 117 | var v = konfig.Values{} 118 | l.Load(v) 119 | 120 | require.Equal(t, "BAR", v["KONFIG_FOO"]) 121 | 122 | var _, ok = v["KONFIG_BAR"] 123 | require.False(t, ok) 124 | }, 125 | ) 126 | 127 | t.Run( 128 | "load env vars prefix regexp replacer", 129 | func(t *testing.T) { 130 | konfig.Init(konfig.DefaultConfig()) 131 | 132 | os.Setenv("FOO", "BAR") 133 | os.Setenv("BAR", "FOO") 134 | 135 | var l = New(&Config{ 136 | Regexp: "^F.*", 137 | Prefix: "KONFIG_", 138 | Replacer: nstrings.ReplacerToLower, 139 | }) 140 | 141 | var v = konfig.Values{} 142 | l.Load(v) 143 | 144 | require.Equal(t, "BAR", v["KONFIG_foo"]) 145 | 146 | var _, ok = v["KONFIG_bar"] 147 | require.False(t, ok) 148 | }, 149 | ) 150 | 151 | t.Run( 152 | "new loader invalid regexp", 153 | func(t *testing.T) { 154 | konfig.Init(konfig.DefaultConfig()) 155 | 156 | os.Setenv("FOO", "BAR") 157 | os.Setenv("BAR", "FOO") 158 | 159 | require.Panics(t, func() { 160 | New(&Config{ 161 | Regexp: "[", 162 | }) 163 | }) 164 | 165 | }, 166 | ) 167 | 168 | t.Run( 169 | "test max retry stop on failure", 170 | func(t *testing.T) { 171 | var l = New(&Config{ 172 | MaxRetry: 1, 173 | RetryDelay: 1 * time.Second, 174 | StopOnFailure: true, 175 | }) 176 | 177 | require.True(t, l.StopOnFailure()) 178 | require.Equal(t, 1, l.MaxRetry()) 179 | require.Equal(t, 1*time.Second, l.RetryDelay()) 180 | }, 181 | ) 182 | } 183 | -------------------------------------------------------------------------------- /loader/kletcd/README.md: -------------------------------------------------------------------------------- 1 | # Etcd Loader 2 | Loads configs from Etcd into konfig.Store 3 | 4 | # Usage 5 | 6 | Basic usage loading keys and using result as string with watcher 7 | ```go 8 | etcdLoader := kletcd.New(&kletc.Config{ 9 | Client: etcdClient, // from go.etcd.io/etcd/clientv3 package 10 | Keys: []Key{ 11 | { 12 | Key: "foo/bar", 13 | }, 14 | }, 15 | Watch: true, 16 | }) 17 | ``` 18 | 19 | Loading keys and JSON parser 20 | ```go 21 | etcdLoader := kletcd.New(&kletc.Config{ 22 | Client: etcdClient, // from go.etcd.io/etcd/clientv3 package 23 | Keys: []Key{ 24 | { 25 | Key: "foo/bar", 26 | Parser: kpjson.Parser, 27 | }, 28 | }, 29 | Watch: true, 30 | }) 31 | ``` 32 | -------------------------------------------------------------------------------- /loader/kletcd/etcd_integration_test.go: -------------------------------------------------------------------------------- 1 | // +build integration 2 | 3 | package kletcd 4 | 5 | import ( 6 | "context" 7 | "testing" 8 | "time" 9 | 10 | "github.com/lalamove/konfig" 11 | "github.com/stretchr/testify/require" 12 | "go.etcd.io/etcd/clientv3" 13 | ) 14 | 15 | func TestIntegrationLoad(t *testing.T) { 16 | c, _ := clientv3.New(clientv3.Config{ 17 | Endpoints: []string{"http://localhost:2379"}, 18 | DialTimeout: 2 * time.Second, 19 | }) 20 | 21 | l := New(&Config{ 22 | Client: c, 23 | Keys: []Key{ 24 | { 25 | Key: "foo", 26 | }, 27 | { 28 | Key: "bar", 29 | }, 30 | }, 31 | }) 32 | 33 | c.KV.Put(context.Background(), "foo", "bar") 34 | 35 | v := konfig.Values{} 36 | require.Nil(t, l.Load(v)) 37 | 38 | require.Equal(t, "bar", v["foo"]) 39 | require.Nil(t, v["bar"]) 40 | } 41 | -------------------------------------------------------------------------------- /loader/kletcd/etcdloader.go: -------------------------------------------------------------------------------- 1 | package kletcd 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "time" 7 | 8 | "github.com/coreos/etcd/mvcc/mvccpb" 9 | "github.com/lalamove/konfig" 10 | "github.com/lalamove/konfig/parser" 11 | "github.com/lalamove/konfig/watcher/kwpoll" 12 | "github.com/lalamove/nui/ncontext" 13 | "github.com/lalamove/nui/nstrings" 14 | "go.etcd.io/etcd/clientv3" 15 | ) 16 | 17 | var ( 18 | defaultTimeout = 5 * time.Second 19 | _ konfig.Loader = (*Loader)(nil) 20 | ) 21 | 22 | const ( 23 | defaultName = "etcd" 24 | ) 25 | 26 | // Key is an Etcd Key to load 27 | type Key struct { 28 | // Key is the etcd key 29 | Key string 30 | // Parser is the parser for the key 31 | // If nil, the value is casted to a string before adding to the config.Store 32 | Parser parser.Parser 33 | } 34 | 35 | // Config is the structure representing the config of a Loader 36 | type Config struct { 37 | // Name is the name of the loader 38 | Name string 39 | // StopOnFailure tells whether a failure to load configs should closed the config and all registered closers 40 | StopOnFailure bool 41 | // Client is the etcd client 42 | Client *clientv3.Client 43 | // Keys is the list of keys to fetch 44 | Keys []Key 45 | // Timeout is the timeout duration when fetching a key 46 | Timeout time.Duration 47 | // Prefix is a prefix to prepend keys when adding into the konfig.Store 48 | Prefix string 49 | // Replacer is a Replacer for the key before adding to the konfig.Store 50 | Replacer nstrings.Replacer 51 | // Watch tells whether there should be a watcher with the loader 52 | Watch bool 53 | // Rater is the rater to pass to the poll watcher 54 | Rater kwpoll.Rater 55 | // MaxRetry is the maximum number of times we can retry to load if it fails 56 | MaxRetry int 57 | // RetryDelay is the time between each retry when a load fails 58 | RetryDelay time.Duration 59 | // Debug sets debug mode on the etcdloader 60 | Debug bool 61 | // Contexter provides a context, default value is contexter wrapping context package. It is used mostly for testing. 62 | Contexter ncontext.Contexter 63 | 64 | kvClient clientv3.KV 65 | } 66 | 67 | // Loader is the structure of a loader 68 | type Loader struct { 69 | *kwpoll.PollWatcher 70 | cfg *Config 71 | } 72 | 73 | // New returns a new loader with the given config 74 | func New(cfg *Config) *Loader { 75 | if cfg.Timeout == 0 { 76 | cfg.Timeout = defaultTimeout 77 | } 78 | 79 | if cfg.Contexter == nil { 80 | cfg.Contexter = ncontext.DefaultContexter 81 | } 82 | 83 | if cfg.Name == "" { 84 | cfg.Name = defaultName 85 | } 86 | 87 | if cfg.kvClient == nil { 88 | cfg.kvClient = cfg.Client.KV 89 | } 90 | 91 | var l = &Loader{ 92 | cfg: cfg, 93 | } 94 | 95 | if cfg.Watch { 96 | var v = konfig.Values{} 97 | var err = l.Load(v) 98 | if err != nil { 99 | panic(err) 100 | } 101 | l.PollWatcher = kwpoll.New(&kwpoll.Config{ 102 | Loader: l, 103 | Rater: cfg.Rater, 104 | InitValue: v, 105 | Debug: cfg.Debug, 106 | Diff: true, 107 | }) 108 | } 109 | 110 | return l 111 | } 112 | 113 | // Name returns the name of the loader 114 | func (l *Loader) Name() string { return l.cfg.Name } 115 | 116 | // Load loads the values from the keys defined by the config in the konfig.Store 117 | func (l *Loader) Load(s konfig.Values) error { 118 | for _, k := range l.cfg.Keys { 119 | 120 | values, err := l.keyValue(k.Key) 121 | if err != nil { 122 | return err 123 | } 124 | 125 | for _, v := range values { 126 | var configKey = l.cfg.Prefix + string(v.Key) 127 | if l.cfg.Replacer != nil { 128 | configKey = l.cfg.Replacer.Replace(configKey) 129 | } 130 | 131 | // if the key has a parser, we parse the key value using the provided Parser 132 | // else we just convert the value to a string 133 | if k.Parser != nil { 134 | if err := k.Parser.Parse(bytes.NewReader(v.Value), s); err != nil { 135 | return err 136 | } 137 | } else { 138 | s.Set(configKey, string(v.Value)) 139 | } 140 | } 141 | } 142 | 143 | return nil 144 | } 145 | 146 | // MaxRetry is the maximum number of time to retry when a load fails 147 | func (l *Loader) MaxRetry() int { 148 | return l.cfg.MaxRetry 149 | } 150 | 151 | // RetryDelay is the delay between each retry 152 | func (l *Loader) RetryDelay() time.Duration { 153 | return l.cfg.RetryDelay 154 | } 155 | 156 | func (l *Loader) keyValue(k string) ([]*mvccpb.KeyValue, error) { 157 | var ctx, cancel = l.cfg.Contexter.WithTimeout( 158 | context.Background(), 159 | l.cfg.Timeout, 160 | ) 161 | defer cancel() 162 | 163 | values, err := l.cfg.kvClient.Get(ctx, k) 164 | if err != nil { 165 | return nil, err 166 | } 167 | 168 | return values.Kvs, nil 169 | } 170 | 171 | // StopOnFailure returns whether a load failure should stop the config and the registered closers 172 | func (l *Loader) StopOnFailure() bool { 173 | return l.cfg.StopOnFailure 174 | } 175 | -------------------------------------------------------------------------------- /loader/kletcd/etcdloader_test.go: -------------------------------------------------------------------------------- 1 | package kletcd 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "strings" 7 | "testing" 8 | "time" 9 | 10 | "github.com/coreos/etcd/mvcc/mvccpb" 11 | "github.com/golang/mock/gomock" 12 | "github.com/lalamove/konfig" 13 | "github.com/lalamove/konfig/mocks" 14 | "github.com/lalamove/konfig/parser" 15 | "github.com/lalamove/konfig/watcher/kwpoll" 16 | "github.com/stretchr/testify/require" 17 | "go.etcd.io/etcd/clientv3" 18 | ) 19 | 20 | func newClient() *clientv3.Client { 21 | c, _ := clientv3.New(clientv3.Config{ 22 | Endpoints: []string{"http://254.0.0.1:12345"}, 23 | DialTimeout: 2 * time.Second, 24 | }) 25 | return c 26 | } 27 | 28 | func TestEtcdLoader(t *testing.T) { 29 | t.Run( 30 | "basic no error", 31 | func(t *testing.T) { 32 | konfig.Init(konfig.DefaultConfig()) 33 | 34 | var ctrl = gomock.NewController(t) 35 | defer ctrl.Finish() 36 | 37 | var mockClient = mocks.NewMockKV(ctrl) 38 | var mockContexter = mocks.NewMockContexter(ctrl) 39 | 40 | var ctx, cancel = context.WithTimeout( 41 | context.Background(), 42 | 5*time.Second, 43 | ) 44 | defer cancel() 45 | 46 | mockContexter.EXPECT().WithTimeout( 47 | context.Background(), 48 | 5*time.Second, 49 | ).Times(2).Return(ctx, context.CancelFunc(func() {})) 50 | 51 | mockClient.EXPECT().Get(ctx, "key1").Return( 52 | &clientv3.GetResponse{ 53 | Kvs: []*mvccpb.KeyValue{ 54 | { 55 | Key: []byte(`key1`), 56 | Value: []byte(`bar`), 57 | }, 58 | }, 59 | }, 60 | nil, 61 | ) 62 | 63 | mockClient.EXPECT().Get(ctx, "key2").Return( 64 | &clientv3.GetResponse{ 65 | Kvs: []*mvccpb.KeyValue{ 66 | { 67 | Key: []byte(`key2`), 68 | Value: []byte(`foo`), 69 | }, 70 | }, 71 | }, 72 | nil, 73 | ) 74 | 75 | var l = New(&Config{ 76 | Client: newClient(), 77 | kvClient: mockClient, 78 | Keys: []Key{ 79 | {Key: "key1"}, 80 | {Key: "key2"}, 81 | }, 82 | Contexter: mockContexter, 83 | }) 84 | 85 | var v = konfig.Values{} 86 | 87 | l.Load(v) 88 | 89 | require.Equal(t, "bar", v["key1"]) 90 | require.Equal(t, "foo", v["key2"]) 91 | }, 92 | ) 93 | 94 | t.Run( 95 | "basic no error multiple result in a key", 96 | func(t *testing.T) { 97 | konfig.Init(konfig.DefaultConfig()) 98 | 99 | var ctrl = gomock.NewController(t) 100 | defer ctrl.Finish() 101 | 102 | var mockClient = mocks.NewMockKV(ctrl) 103 | var mockContexter = mocks.NewMockContexter(ctrl) 104 | 105 | var ctx, cancel = context.WithTimeout( 106 | context.Background(), 107 | 5*time.Second, 108 | ) 109 | defer cancel() 110 | 111 | mockContexter.EXPECT(). 112 | WithTimeout( 113 | context.Background(), 114 | 5*time.Second, 115 | ). 116 | Times(1). 117 | Return(ctx, context.CancelFunc(func() {})) 118 | 119 | mockClient.EXPECT().Get(ctx, "key1").Return( 120 | &clientv3.GetResponse{ 121 | Kvs: []*mvccpb.KeyValue{ 122 | { 123 | Key: []byte(`key1`), 124 | Value: []byte(`bar`), 125 | }, 126 | { 127 | Key: []byte(`key2`), 128 | Value: []byte(`foo`), 129 | }, 130 | }, 131 | }, 132 | nil, 133 | ) 134 | 135 | var l = New(&Config{ 136 | Client: newClient(), 137 | kvClient: mockClient, 138 | Keys: []Key{ 139 | {Key: "key1"}, 140 | }, 141 | Contexter: mockContexter, 142 | }) 143 | 144 | var v = konfig.Values{} 145 | 146 | l.Load(v) 147 | 148 | require.Equal(t, "bar", v["key1"]) 149 | require.Equal(t, "foo", v["key2"]) 150 | }, 151 | ) 152 | 153 | t.Run( 154 | "with watcher no error multiple result in a key", 155 | func(t *testing.T) { 156 | konfig.Init(konfig.DefaultConfig()) 157 | 158 | var ctrl = gomock.NewController(t) 159 | defer ctrl.Finish() 160 | 161 | var mockClient = mocks.NewMockKV(ctrl) 162 | var mockContexter = mocks.NewMockContexter(ctrl) 163 | 164 | var timeout = time.Second 165 | 166 | var ctx, cancel = context.WithTimeout( 167 | context.Background(), 168 | timeout, 169 | ) 170 | defer cancel() 171 | 172 | mockContexter.EXPECT(). 173 | WithTimeout( 174 | context.Background(), 175 | timeout, 176 | ). 177 | MinTimes(1). 178 | Return(ctx, context.CancelFunc(func() {})) 179 | 180 | mockClient.EXPECT().Get(ctx, "key1").Times(1).Return( 181 | &clientv3.GetResponse{ 182 | Kvs: []*mvccpb.KeyValue{ 183 | { 184 | Key: []byte(`key1`), 185 | Value: []byte(`bar`), 186 | }, 187 | { 188 | Key: []byte(`key2`), 189 | Value: []byte(`foo`), 190 | }, 191 | }, 192 | }, 193 | nil, 194 | ) 195 | 196 | var l = New(&Config{ 197 | Client: newClient(), 198 | kvClient: mockClient, 199 | Keys: []Key{ 200 | {Key: "key1"}, 201 | }, 202 | Watch: true, 203 | Rater: kwpoll.Time(100 * time.Millisecond), 204 | Contexter: mockContexter, 205 | Debug: true, 206 | Timeout: timeout, 207 | }) 208 | 209 | mockClient.EXPECT().Get(ctx, "key1").MinTimes(1).Return( 210 | &clientv3.GetResponse{ 211 | Kvs: []*mvccpb.KeyValue{ 212 | { 213 | Key: []byte(`key1`), 214 | Value: []byte(`bar`), 215 | }, 216 | { 217 | Key: []byte(`key`), 218 | Value: []byte(`foo`), 219 | }, 220 | }, 221 | }, 222 | nil, 223 | ) 224 | 225 | err := l.Start() 226 | require.Nil(t, err) 227 | 228 | var timer = time.NewTimer(300 * time.Millisecond) 229 | var watched bool 230 | for { 231 | select { 232 | case <-timer.C: 233 | l.Close() 234 | require.True(t, watched) 235 | return 236 | case <-l.Watch(): 237 | watched = true 238 | } 239 | } 240 | }, 241 | ) 242 | 243 | t.Run( 244 | "no error multiple result in a key replacer prefix", 245 | func(t *testing.T) { 246 | konfig.Init(konfig.DefaultConfig()) 247 | 248 | var ctrl = gomock.NewController(t) 249 | defer ctrl.Finish() 250 | 251 | var mockClient = mocks.NewMockKV(ctrl) 252 | var mockContexter = mocks.NewMockContexter(ctrl) 253 | 254 | var ctx, cancel = context.WithTimeout( 255 | context.Background(), 256 | 5*time.Second, 257 | ) 258 | defer cancel() 259 | 260 | mockContexter.EXPECT(). 261 | WithTimeout( 262 | context.Background(), 263 | 5*time.Second, 264 | ). 265 | Times(1). 266 | Return(ctx, context.CancelFunc(func() {})) 267 | 268 | mockClient.EXPECT().Get(ctx, "key1").Return( 269 | &clientv3.GetResponse{ 270 | Kvs: []*mvccpb.KeyValue{ 271 | { 272 | Key: []byte(`key1`), 273 | Value: []byte(`bar`), 274 | }, 275 | { 276 | Key: []byte(`key2`), 277 | Value: []byte(`foo`), 278 | }, 279 | }, 280 | }, 281 | nil, 282 | ) 283 | 284 | var l = New(&Config{ 285 | Client: newClient(), 286 | kvClient: mockClient, 287 | Keys: []Key{ 288 | {Key: "key1"}, 289 | }, 290 | Prefix: "pfx_", 291 | Replacer: strings.NewReplacer("key", "yek"), 292 | Contexter: mockContexter, 293 | }) 294 | 295 | var v = konfig.Values{} 296 | l.Load(v) 297 | 298 | require.Equal(t, "bar", v["pfx_yek1"]) 299 | require.Equal(t, "foo", v["pfx_yek2"]) 300 | }, 301 | ) 302 | 303 | t.Run( 304 | "no error multiple result in a key replacer prefix", 305 | func(t *testing.T) { 306 | konfig.Init(konfig.DefaultConfig()) 307 | 308 | var ctrl = gomock.NewController(t) 309 | defer ctrl.Finish() 310 | 311 | var mockClient = mocks.NewMockKV(ctrl) 312 | var mockContexter = mocks.NewMockContexter(ctrl) 313 | 314 | var ctx, cancel = context.WithTimeout( 315 | context.Background(), 316 | 5*time.Second, 317 | ) 318 | defer cancel() 319 | 320 | mockContexter.EXPECT(). 321 | WithTimeout( 322 | context.Background(), 323 | 5*time.Second, 324 | ). 325 | Times(1). 326 | Return(ctx, context.CancelFunc(func() {})) 327 | 328 | mockClient.EXPECT().Get(ctx, "key1").Return( 329 | nil, 330 | errors.New(""), 331 | ) 332 | 333 | var l = New(&Config{ 334 | Client: newClient(), 335 | kvClient: mockClient, 336 | Keys: []Key{{Key: "key1"}}, 337 | Prefix: "pfx_", 338 | Replacer: strings.NewReplacer("key", "yek"), 339 | Contexter: mockContexter, 340 | }) 341 | 342 | err := l.Load(konfig.Values{}) 343 | 344 | require.NotNil(t, err) 345 | }, 346 | ) 347 | 348 | t.Run( 349 | "parse value fail", 350 | func(t *testing.T) { 351 | konfig.Init(konfig.DefaultConfig()) 352 | 353 | var ctrl = gomock.NewController(t) 354 | defer ctrl.Finish() 355 | 356 | var mockClient = mocks.NewMockKV(ctrl) 357 | var mockContexter = mocks.NewMockContexter(ctrl) 358 | 359 | var ctx, cancel = context.WithTimeout( 360 | context.Background(), 361 | 5*time.Second, 362 | ) 363 | defer cancel() 364 | 365 | mockContexter.EXPECT().WithTimeout( 366 | context.Background(), 367 | 5*time.Second, 368 | ).Times(1).Return(ctx, context.CancelFunc(func() {})) 369 | 370 | mockClient.EXPECT().Get(ctx, "key1").Return( 371 | &clientv3.GetResponse{ 372 | Kvs: []*mvccpb.KeyValue{ 373 | { 374 | Key: []byte(`key1`), 375 | Value: []byte(`bar`), 376 | }, 377 | }, 378 | }, 379 | nil, 380 | ) 381 | 382 | var l = New(&Config{ 383 | Client: newClient(), 384 | kvClient: mockClient, 385 | Keys: []Key{ 386 | {Key: "key1", Parser: parser.NopParser{Err: errors.New("parse fail")}}, 387 | {Key: "key2"}, 388 | }, 389 | Contexter: mockContexter, 390 | }) 391 | 392 | var v = konfig.Values{} 393 | var err = l.Load(v) 394 | require.Error(t, err) 395 | }, 396 | ) 397 | } 398 | 399 | func TestLoaderMethods(t *testing.T) { 400 | konfig.Init(konfig.DefaultConfig()) 401 | var ctrl = gomock.NewController(t) 402 | defer ctrl.Finish() 403 | 404 | var mockClient = mocks.NewMockKV(ctrl) 405 | 406 | var l = New(&Config{ 407 | Name: "etcdloader", 408 | StopOnFailure: true, 409 | MaxRetry: 1, 410 | RetryDelay: 10 * time.Second, 411 | Client: newClient(), 412 | kvClient: mockClient, 413 | Keys: []Key{{Key: "key1"}}, 414 | }) 415 | 416 | require.True(t, l.StopOnFailure()) 417 | require.Equal(t, "etcdloader", l.Name()) 418 | require.Equal(t, 1, l.MaxRetry()) 419 | require.Equal(t, 10*time.Second, l.RetryDelay()) 420 | } 421 | -------------------------------------------------------------------------------- /loader/klfile/README.md: -------------------------------------------------------------------------------- 1 | # File Loader 2 | File loader loads config from files 3 | 4 | # Usage 5 | 6 | Basic usage with files and json parser and a watcher 7 | ```go 8 | fileLoader := klfile.New(&klfile.Config{ 9 | Files: []File{ 10 | { 11 | Path: "./config.json", 12 | Parser: kpjson.Parser, 13 | }, 14 | }, 15 | Watch: true, 16 | Rate: 1 * time.Second, // Rate for the polling watching the file changes 17 | }) 18 | ``` 19 | 20 | Simplified syntax: 21 | ```go 22 | fileLoader := klfile. 23 | NewFileLoader("config-files", kpjson.Parser, "file1.json", "file2.json"). 24 | WithWatcher() 25 | ``` 26 | -------------------------------------------------------------------------------- /loader/klfile/fileloader.go: -------------------------------------------------------------------------------- 1 | package klfile 2 | 3 | import ( 4 | "errors" 5 | "os" 6 | "time" 7 | 8 | "github.com/lalamove/konfig" 9 | "github.com/lalamove/konfig/parser" 10 | "github.com/lalamove/konfig/watcher/kwfile" 11 | "github.com/lalamove/nui/nfs" 12 | "github.com/lalamove/nui/nlogger" 13 | ) 14 | 15 | var ( 16 | _ konfig.Loader = (*Loader)(nil) 17 | // ErrNoFiles is the error thrown when trying to create a file loader with no files in config 18 | ErrNoFiles = errors.New("no files provided") 19 | // ErrNoParser is the error thrown when trying to create a file loader with no parser 20 | ErrNoParser = errors.New("no parser provided") 21 | // DefaultRate is the default polling rate to check files 22 | DefaultRate = 10 * time.Second 23 | ) 24 | 25 | const ( 26 | defaultName = "file" 27 | ) 28 | 29 | // File is a file to load from 30 | type File struct { 31 | // Path is the path to the file 32 | Path string 33 | // Parser is the parser used to parse file and add it to the config store 34 | Parser parser.Parser 35 | } 36 | 37 | // Config is the config for the file loader 38 | type Config struct { 39 | // Name is the name of the loader 40 | Name string 41 | // StopOnFailure tells whether a failure to load configs should closed the config and all registered closers 42 | StopOnFailure bool 43 | // Files is the path to the files to load 44 | Files []File 45 | // MaxRetry is the maximum number of times load can be retried in config 46 | MaxRetry int 47 | // RetryDelay is the delay between each retry 48 | RetryDelay time.Duration 49 | // Debug sets the debug mode on the file loader 50 | Debug bool 51 | // Logger is the logger used to print messages 52 | Logger nlogger.Provider 53 | // Watch sets whether the fileloader should also watch be a konfig.Watcher 54 | Watch bool 55 | // Rate is the kwfile polling rate 56 | // Default is 10 seconds 57 | Rate time.Duration 58 | } 59 | 60 | // Loader is the structure representring a file loader. 61 | // A file loader loads data from a file and stores it in the konfig.Store. 62 | type Loader struct { 63 | *kwfile.FileWatcher 64 | cfg *Config 65 | fs nfs.FileSystem 66 | } 67 | 68 | // New creates a new Loader fromt the Config cfg. 69 | func New(cfg *Config) *Loader { 70 | if cfg.Files == nil || len(cfg.Files) == 0 { 71 | panic(ErrNoFiles) 72 | } 73 | // make sure all files have a parser 74 | for _, f := range cfg.Files { 75 | if f.Parser == nil { 76 | panic(ErrNoParser) 77 | } 78 | } 79 | if cfg.Logger == nil { 80 | cfg.Logger = defaultLogger() 81 | } 82 | 83 | if cfg.Name == "" { 84 | cfg.Name = defaultName 85 | } 86 | 87 | // create the watcher 88 | var fw *kwfile.FileWatcher 89 | if cfg.Watch { 90 | var filePaths = make([]string, len(cfg.Files)) 91 | for i, f := range cfg.Files { 92 | filePaths[i] = f.Path 93 | } 94 | fw = kwfile.New( 95 | &kwfile.Config{ 96 | Files: filePaths, 97 | Rate: cfg.Rate, 98 | Debug: cfg.Debug, 99 | Logger: cfg.Logger, 100 | }, 101 | ) 102 | } 103 | 104 | return &Loader{ 105 | FileWatcher: fw, 106 | cfg: cfg, 107 | fs: nfs.OSFileSystem{}, 108 | } 109 | } 110 | 111 | // NewFileLoader returns a new file loader with the given name n, the parser p and the file paths filePaths 112 | func NewFileLoader(n string, p parser.Parser, filePaths ...string) *Loader { 113 | var files = make([]File, len(filePaths)) 114 | for i, fp := range filePaths { 115 | files[i] = File{ 116 | Path: fp, 117 | Parser: p, 118 | } 119 | } 120 | 121 | return New(&Config{ 122 | Name: n, 123 | Files: files, 124 | Rate: DefaultRate, 125 | }) 126 | } 127 | 128 | // WithWatcher adds a watcher to the Loader 129 | func (f *Loader) WithWatcher() *Loader { 130 | var filePaths = make([]string, len(f.cfg.Files)) 131 | for i, fi := range f.cfg.Files { 132 | filePaths[i] = fi.Path 133 | } 134 | var fw = kwfile.New( 135 | &kwfile.Config{ 136 | Files: filePaths, 137 | Rate: f.cfg.Rate, 138 | Debug: f.cfg.Debug, 139 | Logger: f.cfg.Logger, 140 | }, 141 | ) 142 | f.FileWatcher = fw 143 | 144 | return f 145 | } 146 | 147 | // Name returns the name of the loader 148 | func (f *Loader) Name() string { return f.cfg.Name } 149 | 150 | // MaxRetry implements konfig.Loader interface and returns the maximum number 151 | // of time Load method can be retried 152 | func (f *Loader) MaxRetry() int { 153 | return f.cfg.MaxRetry 154 | } 155 | 156 | // RetryDelay implements konfig.Loader interface and returns the delay between each retry 157 | func (f *Loader) RetryDelay() time.Duration { 158 | return f.cfg.RetryDelay 159 | } 160 | 161 | // Load implements the konfig.Loader interface. It reads from the file and adds the data to the konfig.Store. 162 | func (f *Loader) Load(cfg konfig.Values) error { 163 | for _, file := range f.cfg.Files { 164 | var fd, err = f.fs.Open(file.Path) 165 | if err != nil { 166 | return err 167 | } 168 | 169 | // we parse the file 170 | if err := file.Parser.Parse(fd, cfg); err != nil { 171 | fd.Close() 172 | return err 173 | } 174 | fd.Close() 175 | } 176 | return nil 177 | } 178 | 179 | // StopOnFailure returns whether a load failure should stop the config and the registered closers 180 | func (f *Loader) StopOnFailure() bool { 181 | return f.cfg.StopOnFailure 182 | } 183 | 184 | func defaultLogger() nlogger.Provider { 185 | return nlogger.NewProvider(nlogger.New(os.Stdout, "FILEWATCHER | ")) 186 | } 187 | -------------------------------------------------------------------------------- /loader/klfile/fileloader_test.go: -------------------------------------------------------------------------------- 1 | package klfile 2 | 3 | import ( 4 | "errors" 5 | "io/ioutil" 6 | "strings" 7 | "testing" 8 | "time" 9 | 10 | "github.com/golang/mock/gomock" 11 | "github.com/lalamove/konfig" 12 | "github.com/lalamove/konfig/mocks" 13 | "github.com/lalamove/konfig/parser/kpjson" 14 | "github.com/lalamove/nui/nfs" 15 | "github.com/stretchr/testify/require" 16 | ) 17 | 18 | func TestFileLoader(t *testing.T) { 19 | var testCases = []struct { 20 | name string 21 | fileName string 22 | setUp func(ctrl *gomock.Controller, fl *Loader) 23 | err bool 24 | }{ 25 | { 26 | name: "BasicNoErrorLoadOnce", 27 | fileName: "./test", 28 | setUp: func(ctrl *gomock.Controller, fl *Loader) { 29 | var fs = nfs.NewMockFileSystem(ctrl) 30 | var r = ioutil.NopCloser(strings.NewReader( 31 | "FOO=BAR\nBAR=FOO", 32 | )) 33 | fs.EXPECT().Open("./test").Return(r, nil) 34 | fl.fs = fs 35 | fl.cfg.Files[0].Parser.(*mocks.MockParser).EXPECT().Parse(r, konfig.Values{}).Return(nil) 36 | }, 37 | }, 38 | { 39 | name: "ErrorOnFile", 40 | fileName: "./test", 41 | setUp: func(ctrl *gomock.Controller, fl *Loader) { 42 | var fs = nfs.NewMockFileSystem(ctrl) 43 | fs.EXPECT().Open("./test").Return(nil, errors.New("")) 44 | fl.fs = fs 45 | }, 46 | err: true, 47 | }, 48 | { 49 | name: "ErrorInvalidFormat", 50 | fileName: "./test", 51 | setUp: func(ctrl *gomock.Controller, fl *Loader) { 52 | var fs = nfs.NewMockFileSystem(ctrl) 53 | var r = ioutil.NopCloser( 54 | strings.NewReader(`{"test":"test"`), 55 | ) 56 | fs.EXPECT().Open("./test").Return( 57 | r, 58 | nil, 59 | ) 60 | fl.fs = fs 61 | fl.cfg.Files[0].Parser.(*mocks.MockParser). 62 | EXPECT(). 63 | Parse(r, konfig.Values{}). 64 | Return(errors.New("")) 65 | }, 66 | err: true, 67 | }, 68 | } 69 | for _, testCase := range testCases { 70 | t.Run(testCase.name, func(t *testing.T) { 71 | konfig.Init(&konfig.Config{}) 72 | var ctrl = gomock.NewController(t) 73 | defer ctrl.Finish() 74 | 75 | var v = konfig.Values{} 76 | 77 | var fl = New(&Config{ 78 | Files: []File{ 79 | { 80 | Path: testCase.fileName, 81 | Parser: mocks.NewMockParser(ctrl), 82 | }, 83 | }, 84 | }) 85 | 86 | testCase.setUp(ctrl, fl) 87 | var err = fl.Load(v) 88 | if testCase.err { 89 | require.NotNil(t, err, "err should not be nil") 90 | return 91 | } 92 | require.Nil(t, err, "err should be nil") 93 | require.Equal(t, defaultName, fl.Name()) 94 | require.False(t, fl.StopOnFailure()) 95 | }) 96 | } 97 | 98 | } 99 | 100 | func TestMaxRetryRetryDelay(t *testing.T) { 101 | var ctrl = gomock.NewController(t) 102 | defer ctrl.Finish() 103 | 104 | var fl = New(&Config{ 105 | MaxRetry: 10, 106 | RetryDelay: 1 * time.Second, 107 | Files: []File{ 108 | { 109 | Path: "dummy", 110 | Parser: mocks.NewMockParser(ctrl), 111 | }, 112 | }, 113 | }) 114 | require.Equal(t, 10, fl.MaxRetry()) 115 | require.Equal(t, 1*time.Second, fl.RetryDelay()) 116 | } 117 | 118 | func TestNewLoader(t *testing.T) { 119 | t.Run( 120 | "No parser panics", 121 | func(t *testing.T) { 122 | require.Panics(t, func() { 123 | New(&Config{ 124 | Files: []File{ 125 | { 126 | Path: "dummy", 127 | Parser: nil, 128 | }, 129 | }, 130 | }) 131 | }) 132 | }, 133 | ) 134 | 135 | t.Run( 136 | "No files panics", 137 | func(t *testing.T) { 138 | require.Panics(t, func() { 139 | var ctrl = gomock.NewController(t) 140 | defer ctrl.Finish() 141 | New(&Config{ 142 | Files: []File{}, 143 | }) 144 | }) 145 | }, 146 | ) 147 | 148 | t.Run( 149 | "With watcher", 150 | func(t *testing.T) { 151 | var ctrl = gomock.NewController(t) 152 | defer ctrl.Finish() 153 | var wl = New(&Config{ 154 | Files: []File{ 155 | { 156 | Path: "fileloader_test.go", 157 | Parser: mocks.NewMockParser(ctrl), 158 | }, 159 | }, 160 | Watch: true, 161 | }) 162 | require.NotNil(t, wl.FileWatcher) 163 | }, 164 | ) 165 | } 166 | 167 | func TestNewFileLoader(t *testing.T) { 168 | t.Run( 169 | "new file loader without watcher", 170 | func(t *testing.T) { 171 | var fl = NewFileLoader("config-files", kpjson.Parser, "foo.json", "bar.json") 172 | 173 | require.Equal( 174 | t, 175 | "foo.json", 176 | fl.cfg.Files[0].Path, 177 | ) 178 | 179 | require.Equal( 180 | t, 181 | "bar.json", 182 | fl.cfg.Files[1].Path, 183 | ) 184 | }, 185 | ) 186 | 187 | t.Run( 188 | "new file loader with watcher", 189 | func(t *testing.T) { 190 | var fl = NewFileLoader("config-files", kpjson.Parser, "./fileloader.go", "./fileloader_test.go").WithWatcher() 191 | 192 | require.Equal( 193 | t, 194 | "./fileloader.go", 195 | fl.cfg.Files[0].Path, 196 | ) 197 | 198 | require.Equal( 199 | t, 200 | "./fileloader_test.go", 201 | fl.cfg.Files[1].Path, 202 | ) 203 | 204 | require.NotNil( 205 | t, 206 | fl.FileWatcher, 207 | ) 208 | }, 209 | ) 210 | } 211 | -------------------------------------------------------------------------------- /loader/klflag/README.md: -------------------------------------------------------------------------------- 1 | # Flag Loader 2 | Loads config values from command line flags 3 | 4 | # Usage 5 | 6 | Basic usage with command line FlagSet 7 | ```go 8 | flagLoader := klflag.New(&klflag.Config{}) 9 | ``` 10 | 11 | With a nstrings.Replacer for keys 12 | ```go 13 | flagLoader := klflag.New(&klflag.Config{ 14 | Replacer: strings.NewReplacer(".", "-") 15 | }) 16 | ``` 17 | -------------------------------------------------------------------------------- /loader/klflag/flagloader.go: -------------------------------------------------------------------------------- 1 | package klflag 2 | 3 | import ( 4 | "flag" 5 | "time" 6 | 7 | "github.com/lalamove/konfig" 8 | "github.com/lalamove/nui/nstrings" 9 | ) 10 | 11 | var _ konfig.Loader = (*Loader)(nil) 12 | 13 | const defaultName = "flag" 14 | 15 | // Config is the config for the Flag Loader 16 | type Config struct { 17 | // Name is the name of the loader 18 | Name string 19 | // StopOnFailure tells whether a failure to load configs should closed the config and all registered closers 20 | StopOnFailure bool 21 | // FlagSet is the flag set from which to load flags in config 22 | // default value is flag.CommandLine 23 | FlagSet *flag.FlagSet 24 | // Prefix is the prefix to append before each flag to be added in the konfig.Store 25 | Prefix string 26 | // Replacer is a replacer to apply on flags to be added in the konfig.Store 27 | Replacer nstrings.Replacer 28 | // MaxRetry is the maximum number of times to retry 29 | MaxRetry int 30 | // RetryDelay is the delay between each retry 31 | RetryDelay time.Duration 32 | } 33 | 34 | // Loader is a loader for command line flags 35 | type Loader struct { 36 | cfg *Config 37 | } 38 | 39 | // New creates a new Loader with the given Config cfg 40 | func New(cfg *Config) *Loader { 41 | if cfg.FlagSet == nil { 42 | cfg.FlagSet = flag.CommandLine 43 | } 44 | 45 | if cfg.Name == "" { 46 | cfg.Name = defaultName 47 | } 48 | 49 | return &Loader{ 50 | cfg: cfg, 51 | } 52 | } 53 | 54 | // Name returns the name of the loader 55 | func (l *Loader) Name() string { return l.cfg.Name } 56 | 57 | // Load implements konfig.Loader interface, it loads flags from the FlagSet given in config 58 | // into the konfig.Store 59 | func (l *Loader) Load(s konfig.Values) error { 60 | l.cfg.FlagSet.VisitAll(func(f *flag.Flag) { 61 | var n = f.Name 62 | if l.cfg.Replacer != nil { 63 | n = l.cfg.Replacer.Replace(n) 64 | } 65 | s.Set(l.cfg.Prefix+n, f.Value.String()) 66 | }) 67 | return nil 68 | } 69 | 70 | // MaxRetry implements the konfig.Loader interface, it returns the max number of times a Load can be retried 71 | // if it fails 72 | func (l *Loader) MaxRetry() int { 73 | return l.cfg.MaxRetry 74 | } 75 | 76 | // RetryDelay implements the konfig.Loader interface, is the delay between each retry 77 | func (l *Loader) RetryDelay() time.Duration { 78 | return l.cfg.RetryDelay 79 | } 80 | 81 | // StopOnFailure returns whether a load failure should stop the config and the registered closers 82 | func (l *Loader) StopOnFailure() bool { 83 | return l.cfg.StopOnFailure 84 | } 85 | -------------------------------------------------------------------------------- /loader/klflag/flagloader_test.go: -------------------------------------------------------------------------------- 1 | package klflag 2 | 3 | import ( 4 | "flag" 5 | "strings" 6 | "testing" 7 | "time" 8 | 9 | "github.com/lalamove/konfig" 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | func TestFlagLoader(t *testing.T) { 14 | t.Run( 15 | "multiple flags", 16 | func(t *testing.T) { 17 | konfig.Init(konfig.DefaultConfig()) 18 | 19 | var f = flag.NewFlagSet("foo", flag.ContinueOnError) 20 | f.Bool("foo", true, "") 21 | 22 | var loader = New(&Config{ 23 | FlagSet: f, 24 | }) 25 | 26 | var v = konfig.Values{} 27 | 28 | loader.Load(v) 29 | require.Equal(t, "true", v["foo"]) 30 | require.Equal(t, defaultName, loader.Name()) 31 | }, 32 | ) 33 | 34 | t.Run( 35 | "with replacer and prefix", 36 | func(t *testing.T) { 37 | konfig.Init(konfig.DefaultConfig()) 38 | 39 | var fs = flag.NewFlagSet("foo", flag.ContinueOnError) 40 | fs.Bool("foo", true, "usage") 41 | 42 | var loader = New(&Config{ 43 | Prefix: "foo_", 44 | Replacer: strings.NewReplacer("foo", "bar"), 45 | FlagSet: fs, 46 | }) 47 | 48 | var v = konfig.Values{} 49 | 50 | loader.Load(v) 51 | require.Equal(t, "true", v["foo_bar"]) 52 | }, 53 | ) 54 | 55 | t.Run( 56 | "default flag set", 57 | func(t *testing.T) { 58 | var loader = New(&Config{}) 59 | require.True(t, loader.cfg.FlagSet == flag.CommandLine) 60 | }, 61 | ) 62 | 63 | t.Run( 64 | "max retry retry delay stop on failure", 65 | func(t *testing.T) { 66 | var loader = New(&Config{StopOnFailure: true, MaxRetry: 1, RetryDelay: 10 * time.Second}) 67 | 68 | require.True(t, loader.StopOnFailure()) 69 | require.Equal(t, 1, loader.MaxRetry()) 70 | require.Equal(t, 10*time.Second, loader.RetryDelay()) 71 | }, 72 | ) 73 | } 74 | -------------------------------------------------------------------------------- /loader/klhttp/README.md: -------------------------------------------------------------------------------- 1 | # HTTP Loader 2 | Loads config from a source over HTTP 3 | 4 | # Usage 5 | 6 | Basic usage with a json source and a poll watcher 7 | ```go 8 | httpLoader := klhttp.New(&klhttp.Config{ 9 | Sources: []Source{ 10 | { 11 | URL: "https://konfig.io/config.json", 12 | Method: "GET", 13 | Parser: kpjson.Parser, 14 | }, 15 | }, 16 | Watch: true, 17 | Rater: kwpoll.Time(10 * time.Second), // Rater is the rater for the poll watcher 18 | }) 19 | ``` 20 | -------------------------------------------------------------------------------- /loader/klhttp/httploader.go: -------------------------------------------------------------------------------- 1 | package klhttp 2 | 3 | import ( 4 | "errors" 5 | "io" 6 | "net/http" 7 | "time" 8 | 9 | "github.com/lalamove/konfig" 10 | "github.com/lalamove/konfig/parser" 11 | "github.com/lalamove/konfig/watcher/kwpoll" 12 | ) 13 | 14 | var ( 15 | defaultRate = 10 * time.Second 16 | // ErrNoSources is the error thrown when creating an Loader without sources 17 | ErrNoSources = errors.New("No sources provided") 18 | ) 19 | 20 | const defaultName = "http" 21 | 22 | // Client is the interface used to send the HTTP request. 23 | // It is implemented by http.Client. 24 | type Client interface { 25 | Do(*http.Request) (*http.Response, error) 26 | } 27 | 28 | // Source is an HTTP source and a Parser 29 | type Source struct { 30 | URL string 31 | Method string 32 | Body io.Reader 33 | Parser parser.Parser 34 | // Prepare is a function to modify request before sending it 35 | Prepare func(*http.Request) 36 | // StatusCode is the status code expected from this source 37 | // If the status code of the response is different, an error is returned. 38 | // Default is 200. 39 | StatusCode int 40 | } 41 | 42 | // Config is the configuration of the Loader 43 | type Config struct { 44 | // Name is the name of the loader 45 | Name string 46 | // StopOnFailure tells whether a failure to load configs should closed the config and all registered closers 47 | StopOnFailure bool 48 | // Sources is a list of remote sources 49 | Sources []Source 50 | // Client is the client used to fetch the file, default is http.DefaultClient 51 | Client Client 52 | // MaxRetry is the maximum number of retries when an error occurs 53 | MaxRetry int 54 | // RetryDelay is the delay between each retry 55 | RetryDelay time.Duration 56 | // Watch sets the whether changes should be watched 57 | Watch bool 58 | // Rater is the rater to pass to the poll write 59 | Rater kwpoll.Rater 60 | // Debug sets the debug mode 61 | Debug bool 62 | } 63 | 64 | // Loader loads a configuration remotely 65 | type Loader struct { 66 | *kwpoll.PollWatcher 67 | cfg *Config 68 | } 69 | 70 | // New returns a new Loader with the given Config. 71 | func New(cfg *Config) *Loader { 72 | if cfg.Client == nil { 73 | cfg.Client = http.DefaultClient 74 | } 75 | 76 | if cfg.Sources == nil || len(cfg.Sources) == 0 { 77 | panic(ErrNoSources) 78 | } 79 | 80 | if cfg.Name == "" { 81 | cfg.Name = defaultName 82 | } 83 | 84 | var l = &Loader{ 85 | cfg: cfg, 86 | } 87 | 88 | for i, source := range cfg.Sources { 89 | if source.Method == "" { 90 | source.Method = http.MethodGet 91 | } 92 | cfg.Sources[i] = source 93 | } 94 | 95 | if cfg.Watch { 96 | var v = konfig.Values{} 97 | var err = l.Load(v) 98 | if err != nil { 99 | panic(err) 100 | } 101 | l.PollWatcher = kwpoll.New(&kwpoll.Config{ 102 | Loader: l, 103 | Rater: cfg.Rater, 104 | InitValue: v, 105 | Diff: true, 106 | Debug: cfg.Debug, 107 | }) 108 | } 109 | 110 | return l 111 | } 112 | 113 | // Name returns the name of the loader 114 | func (r *Loader) Name() string { return r.cfg.Name } 115 | 116 | // Load loads the config from sources and parses the response 117 | func (r *Loader) Load(s konfig.Values) error { 118 | for _, source := range r.cfg.Sources { 119 | if b, err := source.Do(r.cfg.Client); err == nil { 120 | if err := source.Parser.Parse(b, s); err != nil { 121 | return err 122 | } 123 | } else { 124 | return err 125 | } 126 | } 127 | return nil 128 | } 129 | 130 | // MaxRetry returns the MaxRetry config property, it implements the konfig.Loader interface 131 | func (r *Loader) MaxRetry() int { 132 | return r.cfg.MaxRetry 133 | } 134 | 135 | // RetryDelay returns the RetryDelay config property, it implements the konfig.Loader interface 136 | func (r *Loader) RetryDelay() time.Duration { 137 | return r.cfg.RetryDelay 138 | } 139 | 140 | // StopOnFailure returns whether a load failure should stop the config and the registered closers 141 | func (r *Loader) StopOnFailure() bool { 142 | return r.cfg.StopOnFailure 143 | } 144 | -------------------------------------------------------------------------------- /loader/klhttp/httploader_test.go: -------------------------------------------------------------------------------- 1 | package klhttp 2 | 3 | import ( 4 | "errors" 5 | "io/ioutil" 6 | "net/http" 7 | "strings" 8 | "testing" 9 | "time" 10 | 11 | "github.com/golang/mock/gomock" 12 | "github.com/lalamove/konfig" 13 | "github.com/lalamove/konfig/mocks" 14 | "github.com/lalamove/konfig/watcher/kwpoll" 15 | "github.com/stretchr/testify/require" 16 | ) 17 | 18 | type RequestMatcher struct { 19 | req *http.Request 20 | msg string 21 | } 22 | 23 | func (r *RequestMatcher) Matches(x interface{}) bool { 24 | if v, ok := x.(*http.Request); ok { 25 | if v.Method != r.req.Method || v.URL.String() != r.req.URL.String() { 26 | r.msg = "method or url are different" 27 | return false 28 | } 29 | 30 | if r.req.Body != nil && v.Body == nil { 31 | r.msg = "body are different" 32 | return false 33 | } 34 | 35 | if r.req.Body != nil { 36 | var b, _ = ioutil.ReadAll(r.req.Body) 37 | var b2, _ = ioutil.ReadAll(v.Body) 38 | 39 | if string(b) != string(b2) { 40 | r.msg = "body are different" 41 | return false 42 | } 43 | } 44 | 45 | return true 46 | } 47 | return false 48 | } 49 | 50 | func (r *RequestMatcher) String() string { 51 | return r.msg 52 | } 53 | 54 | func TestLoad(t *testing.T) { 55 | var testCases = []struct { 56 | name string 57 | setUp func(ctrl *gomock.Controller) *Loader 58 | err bool 59 | }{ 60 | { 61 | name: "single source no error get request", 62 | setUp: func(ctrl *gomock.Controller) *Loader { 63 | var c = mocks.NewMockClient(ctrl) 64 | var p1 = mocks.NewMockParser(ctrl) 65 | 66 | var hl = New(&Config{ 67 | Client: c, 68 | Sources: []Source{ 69 | { 70 | URL: "http://source.com", 71 | Parser: p1, 72 | }, 73 | }, 74 | }) 75 | 76 | var r = ioutil.NopCloser(strings.NewReader(``)) 77 | var req, _ = http.NewRequest("GET", "http://source.com", nil) 78 | c.EXPECT().Do(&RequestMatcher{ 79 | req: req, 80 | }).Times(1).Return( 81 | &http.Response{ 82 | StatusCode: 200, 83 | Body: r, 84 | }, 85 | nil, 86 | ) 87 | 88 | p1.EXPECT().Parse(r, konfig.Values{}).Times(1).Return(nil) 89 | 90 | return hl 91 | }, 92 | err: false, 93 | }, 94 | { 95 | name: "multiple sources no error get request", 96 | setUp: func(ctrl *gomock.Controller) *Loader { 97 | var c = mocks.NewMockClient(ctrl) 98 | var p1 = mocks.NewMockParser(ctrl) 99 | var p2 = mocks.NewMockParser(ctrl) 100 | 101 | var hl = New(&Config{ 102 | Client: c, 103 | Sources: []Source{ 104 | { 105 | URL: "http://source.com", 106 | Parser: p1, 107 | }, 108 | { 109 | Method: http.MethodPost, 110 | URL: "http://source.com", 111 | Parser: p2, 112 | }, 113 | }, 114 | }) 115 | 116 | var r = ioutil.NopCloser(strings.NewReader(``)) 117 | var req1, _ = http.NewRequest("GET", "http://source.com", nil) 118 | var req2, _ = http.NewRequest("POST", "http://source.com", nil) 119 | 120 | gomock.InOrder( 121 | c.EXPECT().Do(&RequestMatcher{ 122 | req: req1, 123 | }).Times(1).Return( 124 | &http.Response{ 125 | StatusCode: 200, 126 | Body: r, 127 | }, 128 | nil, 129 | ), 130 | c.EXPECT().Do(&RequestMatcher{ 131 | req: req2, 132 | }).Times(1).Return( 133 | &http.Response{ 134 | StatusCode: 200, 135 | Body: r, 136 | }, 137 | nil, 138 | ), 139 | ) 140 | 141 | p1.EXPECT().Parse(r, konfig.Values{}).Times(1).Return(nil) 142 | p2.EXPECT().Parse(r, konfig.Values{}).Times(1).Return(nil) 143 | 144 | return hl 145 | }, 146 | err: false, 147 | }, 148 | { 149 | name: "multiple sources watch no error get request", 150 | setUp: func(ctrl *gomock.Controller) *Loader { 151 | var c = mocks.NewMockClient(ctrl) 152 | var p1 = mocks.NewMockParser(ctrl) 153 | var p2 = mocks.NewMockParser(ctrl) 154 | 155 | var r = ioutil.NopCloser(strings.NewReader(``)) 156 | var req1, _ = http.NewRequest("GET", "http://source.com", nil) 157 | var req2, _ = http.NewRequest("POST", "http://source.com", nil) 158 | 159 | gomock.InOrder( 160 | c.EXPECT().Do(&RequestMatcher{ 161 | req: req1, 162 | }).Times(1).Return( 163 | &http.Response{ 164 | StatusCode: 200, 165 | Body: r, 166 | }, 167 | nil, 168 | ), 169 | c.EXPECT().Do(&RequestMatcher{ 170 | req: req2, 171 | }).Times(1).Return( 172 | &http.Response{ 173 | StatusCode: 200, 174 | Body: r, 175 | }, 176 | nil, 177 | ), 178 | ) 179 | 180 | p1.EXPECT().Parse(r, konfig.Values{}).Times(1).Return(nil) 181 | p2.EXPECT().Parse(r, konfig.Values{}).Times(1).Return(nil) 182 | 183 | var hl = New(&Config{ 184 | Client: c, 185 | Watch: true, 186 | Rater: kwpoll.Time(100 * time.Millisecond), 187 | Sources: []Source{ 188 | { 189 | URL: "http://source.com", 190 | Parser: p1, 191 | }, 192 | { 193 | Method: http.MethodPost, 194 | URL: "http://source.com", 195 | Parser: p2, 196 | }, 197 | }, 198 | }) 199 | 200 | gomock.InOrder( 201 | c.EXPECT().Do(&RequestMatcher{ 202 | req: req1, 203 | }).Times(1).Return( 204 | &http.Response{ 205 | StatusCode: 200, 206 | Body: r, 207 | }, 208 | nil, 209 | ), 210 | c.EXPECT().Do(&RequestMatcher{ 211 | req: req2, 212 | }).Times(1).Return( 213 | &http.Response{ 214 | StatusCode: 200, 215 | Body: r, 216 | }, 217 | nil, 218 | ), 219 | ) 220 | 221 | p1.EXPECT().Parse(r, konfig.Values{}).Times(1).Return(nil) 222 | p2.EXPECT().Parse(r, konfig.Values{}).Times(1).Return(nil) 223 | 224 | return hl 225 | }, 226 | err: false, 227 | }, 228 | { 229 | name: "multiple sources no error get request", 230 | setUp: func(ctrl *gomock.Controller) *Loader { 231 | var c = mocks.NewMockClient(ctrl) 232 | var p1 = mocks.NewMockParser(ctrl) 233 | var p2 = mocks.NewMockParser(ctrl) 234 | 235 | var hl = New(&Config{ 236 | Client: c, 237 | Sources: []Source{ 238 | { 239 | URL: "http://source.com", 240 | Parser: p1, 241 | }, 242 | { 243 | Method: http.MethodPost, 244 | URL: "http://source.com", 245 | Parser: p2, 246 | }, 247 | }, 248 | }) 249 | 250 | var r = ioutil.NopCloser(strings.NewReader(``)) 251 | var req1, _ = http.NewRequest("GET", "http://source.com", nil) 252 | var req2, _ = http.NewRequest("POST", "http://source.com", nil) 253 | 254 | gomock.InOrder( 255 | c.EXPECT().Do(&RequestMatcher{ 256 | req: req1, 257 | }).Times(1).Return( 258 | &http.Response{ 259 | StatusCode: 200, 260 | Body: r, 261 | }, 262 | nil, 263 | ), 264 | c.EXPECT().Do(&RequestMatcher{ 265 | req: req2, 266 | }).Times(1).Return( 267 | nil, 268 | errors.New(""), 269 | ), 270 | ) 271 | 272 | p1.EXPECT().Parse(r, konfig.Values{}).Times(1).Return(nil) 273 | 274 | return hl 275 | }, 276 | err: true, 277 | }, 278 | { 279 | name: "single source error wrong status code", 280 | setUp: func(ctrl *gomock.Controller) *Loader { 281 | var c = mocks.NewMockClient(ctrl) 282 | var p1 = mocks.NewMockParser(ctrl) 283 | 284 | var hl = New(&Config{ 285 | Client: c, 286 | Sources: []Source{ 287 | { 288 | URL: "http://source.com", 289 | Parser: p1, 290 | StatusCode: 201, 291 | }, 292 | }, 293 | }) 294 | 295 | var r = ioutil.NopCloser(strings.NewReader(``)) 296 | var req, _ = http.NewRequest("GET", "http://source.com", nil) 297 | c.EXPECT().Do(&RequestMatcher{ 298 | req: req, 299 | }).Times(1).Return( 300 | &http.Response{ 301 | StatusCode: 400, 302 | Body: r, 303 | }, 304 | nil, 305 | ) 306 | return hl 307 | }, 308 | err: true, 309 | }, 310 | } 311 | 312 | for _, testCase := range testCases { 313 | t.Run( 314 | testCase.name, 315 | func(t *testing.T) { 316 | var ctrl = gomock.NewController(t) 317 | defer ctrl.Finish() 318 | 319 | konfig.Init(konfig.DefaultConfig()) 320 | var hl = testCase.setUp(ctrl) 321 | 322 | var err = hl.Load(konfig.Values{}) 323 | if testCase.err { 324 | require.NotNil(t, err, "err should not be nil") 325 | return 326 | } 327 | require.Nil(t, err, "err should be nil") 328 | }, 329 | ) 330 | } 331 | } 332 | 333 | func TestNew(t *testing.T) { 334 | t.Run( 335 | "default http client", 336 | func(t *testing.T) { 337 | var ctrl = gomock.NewController(t) 338 | defer ctrl.Finish() 339 | 340 | var p = mocks.NewMockParser(ctrl) 341 | 342 | var hl = New(&Config{ 343 | Sources: []Source{ 344 | { 345 | URL: "http://url.com", 346 | Parser: p, 347 | }, 348 | }, 349 | }) 350 | 351 | require.Equal(t, http.DefaultClient, hl.cfg.Client) 352 | }, 353 | ) 354 | t.Run( 355 | "panic no sources", 356 | func(t *testing.T) { 357 | var ctrl = gomock.NewController(t) 358 | defer ctrl.Finish() 359 | 360 | require.Panics(t, func() { 361 | New(&Config{ 362 | Sources: []Source{}, 363 | }) 364 | }) 365 | }, 366 | ) 367 | } 368 | 369 | func TestLoaderMethods(t *testing.T) { 370 | 371 | var ctrl = gomock.NewController(t) 372 | defer ctrl.Finish() 373 | 374 | var p = mocks.NewMockParser(ctrl) 375 | 376 | var hl = New(&Config{ 377 | Name: "httploader", 378 | MaxRetry: 1, 379 | RetryDelay: 1 * time.Second, 380 | StopOnFailure: true, 381 | Sources: []Source{ 382 | { 383 | URL: "http://url.com", 384 | Parser: p, 385 | }, 386 | }, 387 | }) 388 | 389 | require.True(t, hl.StopOnFailure()) 390 | require.Equal(t, "httploader", hl.Name()) 391 | require.Equal(t, 1*time.Second, hl.RetryDelay()) 392 | require.Equal(t, 1, hl.MaxRetry()) 393 | } 394 | -------------------------------------------------------------------------------- /loader/klhttp/source.go: -------------------------------------------------------------------------------- 1 | package klhttp 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | ) 8 | 9 | // Do makes an http request and sends the body to the parser 10 | func (s Source) Do(c Client) (io.Reader, error) { 11 | var req, err = http.NewRequest( 12 | s.Method, 13 | s.URL, 14 | s.Body, 15 | ) 16 | if err != nil { 17 | return nil, err 18 | } 19 | 20 | // call the prepare method if there is one 21 | if s.Prepare != nil { 22 | s.Prepare(req) 23 | } 24 | 25 | // make the request 26 | var res *http.Response 27 | res, err = c.Do(req) 28 | if err != nil { 29 | return nil, err 30 | } 31 | 32 | // check status code 33 | if (s.StatusCode != 0 && res.StatusCode != s.StatusCode) || 34 | (res.StatusCode != http.StatusOK) { 35 | 36 | return nil, fmt.Errorf( 37 | "Error while fetching config at %s, status code: %d", 38 | s.URL, 39 | res.StatusCode, 40 | ) 41 | } 42 | 43 | return res.Body, nil 44 | } 45 | -------------------------------------------------------------------------------- /loader/klreader/README.md: -------------------------------------------------------------------------------- 1 | # Reader Loader 2 | Loads config from an io.Reader 3 | 4 | # Usage 5 | ```go 6 | readerLoader := klreader.New(&klreader.Config{ 7 | Parser: kpjson.Parser, 8 | Reader: strings.NewReader(`{"foo":"bar"}`), 9 | }) 10 | ``` 11 | -------------------------------------------------------------------------------- /loader/klreader/klreader.go: -------------------------------------------------------------------------------- 1 | package klreader 2 | 3 | import ( 4 | "io" 5 | "time" 6 | 7 | "github.com/lalamove/konfig" 8 | "github.com/lalamove/konfig/parser" 9 | ) 10 | 11 | const ( 12 | defaultName = "reader" 13 | ) 14 | 15 | // Config is the structure for the Loader's config 16 | type Config struct { 17 | Name string 18 | Reader io.Reader 19 | Parser parser.Parser 20 | MaxRetry int 21 | RetryDelay time.Duration 22 | StopOnFailure bool 23 | } 24 | 25 | // Loader is the Loader's structure 26 | type Loader struct { 27 | cfg *Config 28 | } 29 | 30 | // New returns a new loader with the given config 31 | func New(cfg *Config) *Loader { 32 | if cfg.Name == "" { 33 | cfg.Name = defaultName 34 | } 35 | return &Loader{ 36 | cfg: cfg, 37 | } 38 | } 39 | 40 | // Name implements konfig.Loader. It returns the loader's name for metrics purpose. 41 | func (l *Loader) Name() string { return l.cfg.Name } 42 | 43 | // MaxRetry implements konfig.Loader interface and returns the maximum number 44 | // of time Load method can be retried 45 | func (l *Loader) MaxRetry() int { 46 | return l.cfg.MaxRetry 47 | } 48 | 49 | // RetryDelay implements konfig.Loader interface and returns the delay between each retry 50 | func (l *Loader) RetryDelay() time.Duration { 51 | return l.cfg.RetryDelay 52 | } 53 | 54 | // Load implements the konfig.Loader interface. It reads from its io.Reader and adds the data to the konfig.Values 55 | func (l *Loader) Load(cfg konfig.Values) error { 56 | return l.cfg.Parser.Parse(l.cfg.Reader, cfg) 57 | } 58 | 59 | // StopOnFailure returns whether a load failure should stop the config and the registered closers 60 | func (l *Loader) StopOnFailure() bool { 61 | return l.cfg.StopOnFailure 62 | } 63 | -------------------------------------------------------------------------------- /loader/klreader/reader_test.go: -------------------------------------------------------------------------------- 1 | package klreader 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | "time" 7 | 8 | "github.com/golang/mock/gomock" 9 | "github.com/lalamove/konfig" 10 | "github.com/lalamove/konfig/mocks" 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | func TestLoader(t *testing.T) { 15 | var ctrl = gomock.NewController(t) 16 | defer ctrl.Finish() 17 | 18 | var mockParser = mocks.NewMockParser(ctrl) 19 | 20 | var r = strings.NewReader(`foo=bar`) 21 | 22 | var loader = New(&Config{ 23 | Reader: r, 24 | Parser: mockParser, 25 | RetryDelay: 5 * time.Minute, 26 | MaxRetry: 5, 27 | }) 28 | 29 | mockParser.EXPECT().Parse(r, konfig.Values{}).Return(nil) 30 | 31 | loader.Load(konfig.Values{}) 32 | 33 | require.Equal(t, defaultName, loader.Name()) 34 | require.Equal(t, 5*time.Minute, loader.RetryDelay()) 35 | require.Equal(t, 5, loader.MaxRetry()) 36 | require.False(t, loader.StopOnFailure()) 37 | } 38 | -------------------------------------------------------------------------------- /loader/klvault/README.md: -------------------------------------------------------------------------------- 1 | # Vault Loader 2 | Loads config values from a vault secrets engine 3 | 4 | # Usage 5 | 6 | Basic usage with Kubernetes auth provider and renewal 7 | ```go 8 | vaultLoader := klvault.New(&klvault.Config{ 9 | Secrets: []klvault.Secret{ 10 | { 11 | Key: "/database/creds/db" 12 | }, 13 | }, 14 | Client: vaultClient, // from github.com/hashicorp/vault/api 15 | AuthProvider: k8s.New(&k8s.Config{ 16 | Client: vaultClient, 17 | K8sTokenPath: "/var/run/secrets/kubernetes.io/serviceaccount/token", 18 | }), 19 | Renew: true, 20 | }) 21 | ``` 22 | 23 | It is possible to pass additional params to the vault secrets engine in the following manner: 24 | 25 | `Key: "/aws/creds/example-role?ttl=20m"` 26 | 27 | KV Secrets Engine - Version 2 (Versioned KV Store) is also supported by the loader, key from the versioned KV store can be accessed as follows: 28 | 29 | `Key: "/secret/data/my-versioned-key"` 30 | 31 | This will return the latest version of the key, a particular version of the secret can be accessed as follows: 32 | 33 | `Key: "/secret/data/my-versioned-key?version=1"` 34 | -------------------------------------------------------------------------------- /loader/klvault/auth/k8s/k8s.go: -------------------------------------------------------------------------------- 1 | package k8s 2 | 3 | import ( 4 | "bytes" 5 | "encoding/base64" 6 | "errors" 7 | "io" 8 | "io/ioutil" 9 | "strings" 10 | "time" 11 | 12 | "github.com/francoispqt/gojay" 13 | vault "github.com/hashicorp/vault/api" 14 | "github.com/lalamove/konfig/loader/klvault" 15 | "github.com/lalamove/nui/nfs" 16 | ) 17 | 18 | var _ klvault.AuthProvider = (*VaultAuth)(nil) 19 | 20 | const ( 21 | loginPath = "/auth/kubernetes/login" 22 | k8sTokenKeyNamespace = "kubernetes.io/serviceaccount/namespace" 23 | k8sTokenKeyServiceAccount = "kubernetes.io/serviceaccount/service-account.name" 24 | ) 25 | 26 | var ( 27 | errNoAuth = errors.New("No authentication in login response") 28 | errNoClient = errors.New("No client provided") 29 | errMalformedToken = errors.New("K8s token is malformed") 30 | fileSystem = nfs.OSFileSystem{} 31 | ) 32 | 33 | // VaultAuth is the structure representing a vault authentication provider 34 | type VaultAuth struct { 35 | cfg *Config 36 | k8sToken string 37 | role string 38 | logicalClient klvault.LogicalClient 39 | } 40 | 41 | // Config is the config of a VaultAuth provider 42 | type Config struct { 43 | // Client is the vault client 44 | Client *vault.Client 45 | // K8sTokenPath is the path to the kubernetes service account jwt 46 | K8sTokenPath string 47 | // Role is the role string 48 | Role string 49 | // RoleFunc is a function to build the role 50 | RoleFunc func(string) (string, error) 51 | // FileSystem is the file system to use 52 | // If no value provided it uses the os file system 53 | FileSystem nfs.FileSystem 54 | } 55 | 56 | // New creates a new K8sVaultauth with the given config cfg. 57 | func New(cfg *Config) *VaultAuth { 58 | // if no vault client 59 | if cfg.Client == nil { 60 | panic(errNoClient) 61 | } 62 | // if no file system use the default file system, 63 | if cfg.FileSystem == nil { 64 | cfg.FileSystem = fileSystem 65 | } 66 | 67 | var k8sVault = &VaultAuth{ 68 | cfg: cfg, 69 | logicalClient: cfg.Client.Logical(), 70 | } 71 | 72 | // load the k8s token 73 | var token string 74 | var err error 75 | if token, err = k8sVault.readK8sToken(); err != nil { 76 | panic(err) 77 | } 78 | k8sVault.k8sToken = token 79 | 80 | var role string 81 | // if role is in config, use it 82 | if k8sVault.cfg.Role != "" { 83 | role = k8sVault.cfg.Role 84 | } else if k8sVault.cfg.RoleFunc != nil { 85 | // if we have a role func run it 86 | if role, err = k8sVault.cfg.RoleFunc(token); err != nil { 87 | panic(err) 88 | } 89 | } else { 90 | // use the default role func 91 | if role, err = k8sVault.buildRole(token); err != nil { 92 | panic(err) 93 | } 94 | } 95 | k8sVault.role = role 96 | 97 | return k8sVault 98 | } 99 | 100 | // Token returns a vault token or an error if it encountered one. 101 | // {"jwt": "'"$KUBE_TOKEN"'", "role": "{{ SERVICE_ACCOUNT_NAME }}"} 102 | func (k *VaultAuth) Token() (string, time.Duration, error) { 103 | var s, err = k.logicalClient.Write( 104 | loginPath, 105 | map[string]interface{}{ 106 | "jwt": k.k8sToken, 107 | "role": k.role, 108 | }, 109 | ) 110 | if err != nil { 111 | return "", 0, err 112 | } 113 | // if we don't have auth return an error 114 | if s.Auth == nil { 115 | return "", 0, errNoAuth 116 | } 117 | // return the client token 118 | return s.Auth.ClientToken, time.Duration(s.Auth.LeaseDuration) * time.Second, nil 119 | } 120 | 121 | func (k *VaultAuth) readK8sToken() (string, error) { 122 | var f io.ReadCloser 123 | var err error 124 | if f, err = k.cfg.FileSystem.Open(k.cfg.K8sTokenPath); err != nil { 125 | return "", err 126 | } 127 | 128 | var b []byte 129 | b, err = ioutil.ReadAll(f) 130 | if err != nil { 131 | return "", err 132 | } 133 | return string(b), nil 134 | 135 | } 136 | 137 | func (k *VaultAuth) buildRole(k8sToken string) (string, error) { 138 | // the token is a JWT, we split it by dots and take what's at index 1 139 | var tokenSpl = strings.Split(k8sToken, ".") 140 | if len(tokenSpl) != 3 { 141 | return "", errMalformedToken 142 | } 143 | 144 | var b64TokenData = tokenSpl[1] 145 | 146 | var tokenData, err = base64.RawStdEncoding.DecodeString(b64TokenData) 147 | if err != nil { 148 | return "", err 149 | } 150 | 151 | var dec = gojay.BorrowDecoder(bytes.NewReader(tokenData)) 152 | defer dec.Release() 153 | 154 | var namespace string 155 | var role string 156 | 157 | err = dec.Decode(gojay.DecodeObjectFunc(func(dec *gojay.Decoder, k string) error { 158 | switch k { 159 | case k8sTokenKeyNamespace: 160 | return dec.String(&namespace) 161 | case k8sTokenKeyServiceAccount: 162 | return dec.String(&role) 163 | } 164 | return nil 165 | })) 166 | 167 | if err != nil { 168 | return "", err 169 | } 170 | return namespace + "-" + role, nil 171 | } 172 | -------------------------------------------------------------------------------- /loader/klvault/auth/k8s/k8s_test.go: -------------------------------------------------------------------------------- 1 | package k8s 2 | 3 | import ( 4 | "encoding/base64" 5 | "errors" 6 | "io/ioutil" 7 | "strings" 8 | "testing" 9 | "time" 10 | 11 | "github.com/golang/mock/gomock" 12 | vault "github.com/hashicorp/vault/api" 13 | "github.com/lalamove/konfig/mocks" 14 | "github.com/lalamove/nui/nfs" 15 | "github.com/stretchr/testify/require" 16 | ) 17 | 18 | func TestNewK8sAuth(t *testing.T) { 19 | t.Run( 20 | "new no error, uses default role builder", 21 | func(t *testing.T) { 22 | var ctrl = gomock.NewController(t) 23 | defer ctrl.Finish() 24 | 25 | var fs = nfs.NewMockFileSystem(ctrl) 26 | 27 | fs.EXPECT(). 28 | Open("test"). 29 | Return( 30 | ioutil.NopCloser(strings.NewReader( 31 | "12345."+base64.RawStdEncoding.EncodeToString([]byte( 32 | `{"kubernetes.io/serviceaccount/namespace":"dev","kubernetes.io/serviceaccount/service-account.name":"vault-config-loader"}`, 33 | ))+".12345")), 34 | nil, 35 | ) 36 | 37 | var c, _ = vault.NewClient(vault.DefaultConfig()) 38 | 39 | var k8sAuth = New(&Config{ 40 | K8sTokenPath: "test", 41 | Client: c, 42 | FileSystem: fs, 43 | }) 44 | 45 | require.Equal(t, "dev-vault-config-loader", k8sAuth.role) 46 | }, 47 | ) 48 | 49 | t.Run( 50 | "new no error, uses config role", 51 | func(t *testing.T) { 52 | var ctrl = gomock.NewController(t) 53 | defer ctrl.Finish() 54 | 55 | var fs = nfs.NewMockFileSystem(ctrl) 56 | 57 | fs.EXPECT(). 58 | Open("test"). 59 | Return( 60 | ioutil.NopCloser(strings.NewReader( 61 | "12345."+base64.RawStdEncoding.EncodeToString([]byte( 62 | `{"kubernetes.io/serviceaccount/namespace":"dev","kubernetes.io/serviceaccount/service-account.name":"vault-config-loader"}`, 63 | ))+".12345")), 64 | nil, 65 | ) 66 | 67 | var c, _ = vault.NewClient(vault.DefaultConfig()) 68 | 69 | var k8sAuth = New(&Config{ 70 | K8sTokenPath: "test", 71 | Client: c, 72 | FileSystem: fs, 73 | Role: "foobar", 74 | }) 75 | 76 | require.Equal(t, "foobar", k8sAuth.role) 77 | }, 78 | ) 79 | 80 | t.Run( 81 | "new no error, uses config role error invalid base64", 82 | func(t *testing.T) { 83 | var ctrl = gomock.NewController(t) 84 | defer ctrl.Finish() 85 | 86 | var fs = nfs.NewMockFileSystem(ctrl) 87 | 88 | fs.EXPECT(). 89 | Open("test"). 90 | Return( 91 | ioutil.NopCloser(strings.NewReader( 92 | "12345.!##%$.12345")), 93 | nil, 94 | ) 95 | 96 | var c, _ = vault.NewClient(vault.DefaultConfig()) 97 | 98 | require.Panics( 99 | t, 100 | func() { 101 | New(&Config{ 102 | K8sTokenPath: "test", 103 | Client: c, 104 | FileSystem: fs, 105 | }) 106 | }, 107 | ) 108 | }, 109 | ) 110 | 111 | t.Run( 112 | "new no error, uses config role func", 113 | func(t *testing.T) { 114 | var ctrl = gomock.NewController(t) 115 | defer ctrl.Finish() 116 | 117 | var fs = nfs.NewMockFileSystem(ctrl) 118 | 119 | fs.EXPECT(). 120 | Open("test"). 121 | Return( 122 | ioutil.NopCloser(strings.NewReader( 123 | "12345."+base64.RawStdEncoding.EncodeToString([]byte( 124 | `{"kubernetes.io/serviceaccount/namespace":"dev","kubernetes.io/serviceaccount/service-account.name":"vault-config-loader"}`, 125 | ))+".12345")), 126 | nil, 127 | ) 128 | 129 | var c, _ = vault.NewClient(vault.DefaultConfig()) 130 | 131 | var k8sAuth = New(&Config{ 132 | K8sTokenPath: "test", 133 | Client: c, 134 | FileSystem: fs, 135 | RoleFunc: func(string) (string, error) { 136 | return "foobar", nil 137 | }, 138 | }) 139 | 140 | require.Equal(t, "foobar", k8sAuth.role) 141 | }, 142 | ) 143 | 144 | t.Run( 145 | "new no error, uses config role func with error", 146 | func(t *testing.T) { 147 | var ctrl = gomock.NewController(t) 148 | defer ctrl.Finish() 149 | 150 | var fs = nfs.NewMockFileSystem(ctrl) 151 | 152 | fs.EXPECT(). 153 | Open("test"). 154 | Return( 155 | ioutil.NopCloser(strings.NewReader( 156 | "12345."+base64.RawStdEncoding.EncodeToString([]byte( 157 | `{"kubernetes.io/serviceaccount/namespace":"dev","kubernetes.io/serviceaccount/service-account.name":"vault-config-loader"}`, 158 | ))+".12345")), 159 | nil, 160 | ) 161 | 162 | var c, _ = vault.NewClient(vault.DefaultConfig()) 163 | 164 | require.Panics(t, func() { 165 | New(&Config{ 166 | K8sTokenPath: "test", 167 | Client: c, 168 | FileSystem: fs, 169 | RoleFunc: func(string) (string, error) { 170 | return "", errors.New("err") 171 | }, 172 | }) 173 | }) 174 | }, 175 | ) 176 | 177 | t.Run( 178 | "new panics no client", 179 | func(t *testing.T) { 180 | var ctrl = gomock.NewController(t) 181 | defer ctrl.Finish() 182 | 183 | require.Panics(t, func() { 184 | New(&Config{ 185 | K8sTokenPath: "test", 186 | RoleFunc: func(string) (string, error) { 187 | return "foobar", nil 188 | }, 189 | }) 190 | }) 191 | }, 192 | ) 193 | } 194 | 195 | func TestBuildRole(t *testing.T) { 196 | var testCases = []struct { 197 | name string 198 | token string 199 | expectedRole string 200 | err bool 201 | }{ 202 | { 203 | token: "12345." + base64.RawStdEncoding.EncodeToString([]byte( 204 | `{"kubernetes.io/serviceaccount/namespace":"dev","kubernetes.io/serviceaccount/service-account.name":"vault-config-loader"}`, 205 | )) + ".ABCD", 206 | expectedRole: "dev-vault-config-loader", 207 | }, 208 | { 209 | token: "ABCDE", 210 | err: true, 211 | }, 212 | { 213 | token: "12345." + base64.RawStdEncoding.EncodeToString([]byte( 214 | `{"kubernetes.io/serviceaccount/namespace":"dev","kubernetes.io/serviceaccount/service-account.name":"vault-config-loader"}`, 215 | )) + ".ABCD", 216 | expectedRole: "dev-vault-config-loader", 217 | }, 218 | } 219 | 220 | for _, testCase := range testCases { 221 | t.Run( 222 | testCase.name, 223 | func(t *testing.T) { 224 | var k8sAuth = &VaultAuth{} 225 | var s, err = k8sAuth.buildRole(testCase.token) 226 | if testCase.err { 227 | require.NotNil(t, err, "err should not be nil") 228 | return 229 | } 230 | require.Nil(t, err, "err should be nil") 231 | require.Equal(t, testCase.expectedRole, s) 232 | }, 233 | ) 234 | } 235 | } 236 | 237 | func TestToken(t *testing.T) { 238 | t.Run( 239 | "no error", 240 | func(t *testing.T) { 241 | var ctrl = gomock.NewController(t) 242 | defer ctrl.Finish() 243 | 244 | var logicalClient = mocks.NewMockLogicalClient(ctrl) 245 | logicalClient.EXPECT().Write( 246 | loginPath, 247 | map[string]interface{}{ 248 | "jwt": "123", 249 | "role": "role", 250 | }, 251 | ).Times(1).Return(&vault.Secret{ 252 | Auth: &vault.SecretAuth{ 253 | ClientToken: "123", 254 | LeaseDuration: 3600, 255 | }, 256 | }, nil) 257 | 258 | var k = &VaultAuth{ 259 | k8sToken: "123", 260 | role: "role", 261 | logicalClient: logicalClient, 262 | } 263 | 264 | var token, d, err = k.Token() 265 | require.Equal(t, "123", token) 266 | require.Equal(t, 3600*time.Second, d) 267 | require.Nil(t, err) 268 | }, 269 | ) 270 | 271 | t.Run( 272 | "error when calling vault", 273 | func(t *testing.T) { 274 | var ctrl = gomock.NewController(t) 275 | defer ctrl.Finish() 276 | 277 | var logicalClient = mocks.NewMockLogicalClient(ctrl) 278 | logicalClient.EXPECT().Write( 279 | loginPath, 280 | map[string]interface{}{ 281 | "jwt": "123", 282 | "role": "role", 283 | }, 284 | ).Times(1).Return( 285 | nil, 286 | errors.New("err"), 287 | ) 288 | 289 | var k = &VaultAuth{ 290 | k8sToken: "123", 291 | role: "role", 292 | logicalClient: logicalClient, 293 | } 294 | 295 | var _, _, err = k.Token() 296 | require.NotNil(t, err) 297 | }, 298 | ) 299 | } 300 | -------------------------------------------------------------------------------- /loader/klvault/auth/token/token.go: -------------------------------------------------------------------------------- 1 | package token 2 | 3 | import "time" 4 | 5 | // Token auth provider 6 | type Token struct { 7 | T string 8 | } 9 | 10 | // Token returns a vault token or an error if it encountered one. 11 | // {"jwt": "'"$KUBE_TOKEN"'", "role": "{{ SERVICE_ACCOUNT_NAME }}"} 12 | func (k *Token) Token() (string, time.Duration, error) { 13 | return k.T, 10 * time.Second, nil 14 | } 15 | -------------------------------------------------------------------------------- /loader/klvault/authprovider.go: -------------------------------------------------------------------------------- 1 | package klvault 2 | 3 | import "time" 4 | 5 | // AuthProvider is the interface for a Vault authentication provider 6 | type AuthProvider interface { 7 | Token() (string, time.Duration, error) 8 | } 9 | -------------------------------------------------------------------------------- /loader/klvault/vaultloader.go: -------------------------------------------------------------------------------- 1 | package klvault 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "net/url" 8 | "os" 9 | "strings" 10 | "sync" 11 | "time" 12 | 13 | vault "github.com/hashicorp/vault/api" 14 | "github.com/lalamove/konfig" 15 | "github.com/lalamove/konfig/watcher/kwpoll" 16 | "github.com/lalamove/nui/nlogger" 17 | "github.com/lalamove/nui/nstrings" 18 | ) 19 | 20 | var _ konfig.Loader = (*Loader)(nil) 21 | 22 | var ( 23 | defaultTTL = 45 * time.Minute 24 | defaultTTLRatio = 75 25 | // ErrNoClient is the error thrown when trying to create a Loader without vault.Client 26 | ErrNoClient = errors.New("No vault client provided") 27 | // ErrNoAuthProvider is the error thrown when trying to create a Loader without an AuthProvider 28 | ErrNoAuthProvider = errors.New("No auth provider given") 29 | // ErrNoSecretKey is the error thrown when trying to create a Loader without a SecretKey 30 | ErrNoSecretKey = errors.New("No secret key given") 31 | ) 32 | 33 | const defaultName = "vault" 34 | 35 | // LogicalClient is a interface for the vault logical client 36 | type LogicalClient interface { 37 | Read(key string) (*vault.Secret, error) 38 | Write(key string, data map[string]interface{}) (*vault.Secret, error) 39 | ReadWithData(key string, data map[string][]string) (*vault.Secret, error) 40 | } 41 | 42 | // Secret is a secret to load 43 | type Secret struct { 44 | // Key is the URL to fetch the secret from (e.g. /v1/database/creds/mydb) 45 | Key string 46 | // KeysPrefix sets a prefix to be prepended to all keys in the config store 47 | KeysPrefix string 48 | // Replacer transforms vault secret's keys 49 | Replacer nstrings.Replacer 50 | } 51 | 52 | // Config is the config for the Loader 53 | type Config struct { 54 | // Name is the name of the loader 55 | Name string 56 | // StopOnFailure tells whether a failure to load configs should closed the config and all registered closers 57 | StopOnFailure bool 58 | // Secrets is the list of secrets to load 59 | Secrets []Secret 60 | // AuthProvider is the vault auth provider 61 | AuthProvider AuthProvider 62 | // Client is the vault client for the vault loader 63 | Client *vault.Client 64 | // MaxRetry is the maximum number of times the load method can be retried 65 | MaxRetry int 66 | // RetryDelay is the time between each retry 67 | RetryDelay time.Duration 68 | // Debug enables debug mode 69 | Debug bool 70 | // Logger is the logger used for debug logs 71 | Logger nlogger.Provider 72 | // TTLRatio is the factor to multiply the key's TTL by to deduce the moment 73 | // the Loader should ask vault for new credentials. Default value is 75. 74 | // Example: ttl = 1h, ttl * 75 / 100 = 45m, the loader will refresh key after 45m 75 | TTLRatio int 76 | // Renew sets whether the vault loader should renew it self 77 | Renew bool 78 | } 79 | 80 | // Loader is the structure representing a Loader 81 | type Loader struct { 82 | *kwpoll.PollWatcher 83 | cfg *Config 84 | logicalClient LogicalClient 85 | mut *sync.Mutex 86 | ttl time.Duration 87 | } 88 | 89 | // New creates a new Loader with the given config 90 | func New(cfg *Config) *Loader { 91 | if cfg.Secrets == nil || len(cfg.Secrets) == 0 { 92 | panic(ErrNoSecretKey) 93 | } 94 | if cfg.AuthProvider == nil { 95 | panic(ErrNoAuthProvider) 96 | } 97 | if cfg.Client == nil { 98 | panic(ErrNoClient) 99 | } 100 | if cfg.Logger == nil { 101 | cfg.Logger = defaultLogger() 102 | } 103 | if cfg.Name == "" { 104 | cfg.Name = defaultName 105 | } 106 | if cfg.TTLRatio == 0 { 107 | cfg.TTLRatio = defaultTTLRatio 108 | } 109 | var vl = &Loader{ 110 | cfg: cfg, 111 | logicalClient: cfg.Client.Logical(), 112 | mut: &sync.Mutex{}, 113 | ttl: defaultTTL, 114 | } 115 | 116 | var pw *kwpoll.PollWatcher 117 | if cfg.Renew { 118 | pw = kwpoll.New( 119 | &kwpoll.Config{ 120 | Debug: cfg.Debug, 121 | Logger: cfg.Logger, 122 | Rater: vl, 123 | }, 124 | ) 125 | } 126 | vl.PollWatcher = pw 127 | 128 | return vl 129 | } 130 | 131 | // Name returns the name of the loader 132 | func (vl *Loader) Name() string { return vl.cfg.Name } 133 | 134 | // MaxRetry is the maximum number of times the load method can be retried 135 | func (vl *Loader) MaxRetry() int { 136 | return vl.cfg.MaxRetry 137 | } 138 | 139 | // RetryDelay is the delay between each retry 140 | func (vl *Loader) RetryDelay() time.Duration { 141 | return vl.cfg.RetryDelay 142 | } 143 | 144 | // Load implements konfig.Loader interface. 145 | // It fetches a token from the auth provider and sets the token in the vault client. 146 | // Then it loads the secret and assigns it values to the konfig.Store. 147 | func (vl *Loader) Load(cs konfig.Values) error { 148 | if vl.cfg.Debug { 149 | vl.cfg.Logger.Get().Debug( 150 | "Loading vault config", 151 | ) 152 | } 153 | // everytime we load we get a new token 154 | // maybe we could improve implementation to use a shorter ticker and check if config if different, if yes, reload it 155 | var token, ttl, err = vl.cfg.AuthProvider.Token() 156 | if err != nil { 157 | vl.cfg.Logger.Get().Error(err.Error()) 158 | 159 | return err 160 | } 161 | // we set the token in the client 162 | vl.cfg.Client.SetToken(token) 163 | 164 | var leaseDuration = int(ttl / time.Second) 165 | for _, secret := range vl.cfg.Secrets { 166 | // we fetch our secret 167 | var s *vault.Secret 168 | var sData map[string]interface{} 169 | 170 | k := strings.TrimSpace(secret.Key) 171 | k = strings.Trim(k, "/") 172 | if k == "" { 173 | return err 174 | } 175 | 176 | p, err := url.Parse(k) 177 | if err != nil { 178 | return err 179 | } 180 | s, err = vl.logicalClient.ReadWithData(p.Path, p.Query()) 181 | if err != nil { 182 | return err 183 | } 184 | // checking for KV V2 for vault secret store 185 | // confirming version exists on metadata and it is an int 186 | if m, ok := s.Data["metadata"].(map[string]interface{}); ok { 187 | kvData, dataOK := s.Data["data"].(map[string]interface{}) 188 | _, versionJSONNumberOK := m["version"].(json.Number) 189 | if versionJSONNumberOK && dataOK { 190 | sData = kvData 191 | } 192 | } else { 193 | sData = s.Data 194 | } 195 | if vl.cfg.Debug { 196 | vl.cfg.Logger.Get().Debug( 197 | fmt.Sprintf("Got secret, expiring in: %d", s.LeaseDuration), 198 | ) 199 | } 200 | 201 | // if the current secret lease is smaller than the previous smaller lease 202 | // or there is no previous lease 203 | if s.LeaseDuration != 0 && (leaseDuration == 0 || s.LeaseDuration < leaseDuration) { 204 | leaseDuration = s.LeaseDuration 205 | } 206 | // we set our data on the config store 207 | for k, v := range sData { 208 | var nK = secret.KeysPrefix + k 209 | if secret.Replacer != nil { 210 | nK = secret.Replacer.Replace(nK) 211 | } 212 | cs.Set(nK, v) 213 | } 214 | } 215 | 216 | // reset the ttl for renewal 217 | vl.resetTTL(vl.cfg.TTLRatio, ttl, time.Duration(leaseDuration)*time.Second) 218 | return nil 219 | } 220 | 221 | // Time returns the TTL of the vault loader 222 | // It is used in the ticker watcher a source. 223 | func (vl *Loader) Time() time.Duration { 224 | return vl.ttl 225 | } 226 | 227 | // StopOnFailure returns whether a load failure should stop the config and the registered closers 228 | func (vl *Loader) StopOnFailure() bool { 229 | return vl.cfg.StopOnFailure 230 | } 231 | 232 | func (vl *Loader) resetTTL(ttlFac int, tokenTTL, secretTTL time.Duration) { 233 | var ttl = tokenTTL 234 | if secretTTL < tokenTTL { 235 | ttl = secretTTL 236 | } 237 | ttl = ttl * time.Duration(ttlFac) / 100 238 | vl.mut.Lock() 239 | if ttl != vl.ttl { 240 | vl.ttl = ttl 241 | } 242 | vl.mut.Unlock() 243 | } 244 | 245 | func defaultLogger() nlogger.Provider { 246 | return nlogger.NewProvider(nlogger.New(os.Stdout, "VAULT CONFIG | ")) 247 | } 248 | -------------------------------------------------------------------------------- /loader/klvault/vaultloader_test.go: -------------------------------------------------------------------------------- 1 | package klvault 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "sync" 7 | "testing" 8 | "time" 9 | 10 | gomock "github.com/golang/mock/gomock" 11 | vault "github.com/hashicorp/vault/api" 12 | "github.com/lalamove/konfig" 13 | "github.com/lalamove/konfig/mocks" 14 | "github.com/stretchr/testify/require" 15 | ) 16 | 17 | func TestVaultLoader(t *testing.T) { 18 | var testCases = []struct { 19 | name string 20 | setUp func(ctrl *gomock.Controller) *Loader 21 | asserts func(t *testing.T, vl *Loader, cfg konfig.Values) 22 | err bool 23 | }{ 24 | { 25 | name: "BasicNoError", 26 | setUp: func(ctrl *gomock.Controller) *Loader { 27 | var aP = mocks.NewMockAuthProvider(ctrl) 28 | aP.EXPECT().Token().Return( 29 | "DUMMYTOKEN", 30 | 1*time.Hour, 31 | nil, 32 | ) 33 | 34 | var c, _ = vault.NewClient(vault.DefaultConfig()) 35 | 36 | var vl = New(&Config{ 37 | Client: c, 38 | Secrets: []Secret{ 39 | {Key: "/dummy/secret/path"}, 40 | {Key: "/dummy/secret/path2"}, 41 | {Key: "/dummy/secret/data/path3"}, 42 | {Key: "/dummy/secret/data/path3?version=1"}, 43 | }, 44 | AuthProvider: aP, 45 | Debug: true, 46 | }) 47 | 48 | var lC = mocks.NewMockLogicalClient(ctrl) 49 | vl.logicalClient = lC 50 | lC.EXPECT().ReadWithData("dummy/secret/path", map[string][]string{}).Return( 51 | &vault.Secret{ 52 | Data: map[string]interface{}{ 53 | "FOO": "BAR", 54 | }, 55 | LeaseDuration: int(2 * time.Hour / time.Second), 56 | }, 57 | nil, 58 | ) 59 | 60 | lC.EXPECT().ReadWithData("dummy/secret/path2", map[string][]string{}).Return( 61 | &vault.Secret{ 62 | Data: map[string]interface{}{ 63 | "BAR": "FOO", 64 | }, 65 | LeaseDuration: int(1 * time.Hour / time.Second), 66 | }, 67 | nil, 68 | ) 69 | lC.EXPECT().ReadWithData("dummy/secret/data/path3", map[string][]string{}).Return( 70 | &vault.Secret{ 71 | Data: map[string]interface{}{ 72 | "data": map[string]interface{}{ 73 | "VERSIONEDFOO": "FOO2", 74 | }, 75 | "metadata": map[string]interface{}{ 76 | "created_time": "2018-03-22T02:24:06.945319214Z", 77 | "deletion_time": "", 78 | "destroyed": false, 79 | "version": json.Number("1"), 80 | }, 81 | }, 82 | LeaseDuration: int(1 * time.Hour / time.Second), 83 | }, 84 | nil, 85 | ) 86 | lC.EXPECT().ReadWithData("dummy/secret/data/path3", map[string][]string{"version": {"1"}}).Return( 87 | &vault.Secret{ 88 | Data: map[string]interface{}{ 89 | "data": map[string]interface{}{ 90 | "OLDFOO": "FOO1", 91 | }, 92 | "metadata": map[string]interface{}{ 93 | "created_time": "2018-03-22T02:24:06.945319214Z", 94 | "deletion_time": "", 95 | "destroyed": false, 96 | "version": json.Number("1"), 97 | }, 98 | }, 99 | LeaseDuration: int(1 * time.Hour / time.Second), 100 | }, 101 | nil, 102 | ) 103 | 104 | return vl 105 | }, 106 | asserts: func(t *testing.T, vl *Loader, cfg konfig.Values) { 107 | require.Equal( 108 | t, 109 | 45*time.Minute, 110 | vl.Time(), 111 | ) 112 | require.Equal( 113 | t, 114 | "BAR", 115 | cfg["FOO"], 116 | ) 117 | require.Equal( 118 | t, 119 | "FOO", 120 | cfg["BAR"], 121 | ) 122 | require.Equal( 123 | t, 124 | "FOO2", 125 | cfg["VERSIONEDFOO"], 126 | ) 127 | require.Equal( 128 | t, 129 | "FOO1", 130 | cfg["OLDFOO"], 131 | ) 132 | require.Equal( 133 | t, 134 | defaultName, 135 | vl.Name(), 136 | ) 137 | }, 138 | }, 139 | { 140 | name: "ErrorOnAuthProvider", 141 | err: true, 142 | setUp: func(ctrl *gomock.Controller) *Loader { 143 | var aP = mocks.NewMockAuthProvider(ctrl) 144 | aP.EXPECT().Token().Return( 145 | "", 146 | time.Duration(0), 147 | errors.New(""), 148 | ) 149 | 150 | var c, _ = vault.NewClient(vault.DefaultConfig()) 151 | 152 | var vl = New(&Config{ 153 | Client: c, 154 | Secrets: []Secret{{Key: "/dummy/secret/path"}}, 155 | AuthProvider: aP, 156 | }) 157 | return vl 158 | }, 159 | asserts: func(t *testing.T, vl *Loader, cfg konfig.Values) {}, 160 | }, 161 | { 162 | name: "ErrorFetchingSecret", 163 | err: true, 164 | setUp: func(ctrl *gomock.Controller) *Loader { 165 | var aP = mocks.NewMockAuthProvider(ctrl) 166 | aP.EXPECT().Token().Return( 167 | "DUMMYTOKEN", 168 | 1*time.Hour, 169 | nil, 170 | ) 171 | 172 | var c, _ = vault.NewClient(vault.DefaultConfig()) 173 | 174 | var vl = New(&Config{ 175 | Client: c, 176 | Secrets: []Secret{{Key: "/dummy/secret/path"}}, 177 | AuthProvider: aP, 178 | }) 179 | 180 | var lC = mocks.NewMockLogicalClient(ctrl) 181 | vl.logicalClient = lC 182 | lC.EXPECT().ReadWithData("dummy/secret/path", map[string][]string{}).Return( 183 | nil, 184 | errors.New(""), 185 | ) 186 | 187 | return vl 188 | }, 189 | asserts: func(t *testing.T, vl *Loader, cfg konfig.Values) {}, 190 | }, 191 | } 192 | 193 | for _, testCase := range testCases { 194 | t.Run(testCase.name, func(t *testing.T) { 195 | var ctrl = gomock.NewController(t) 196 | defer ctrl.Finish() 197 | var vl = testCase.setUp(ctrl) 198 | konfig.Init(&konfig.Config{}) 199 | var c = konfig.Values{} 200 | 201 | var err = vl.Load(c) 202 | 203 | if testCase.err { 204 | require.NotNil(t, err, "err should not be nil") 205 | return 206 | } 207 | 208 | require.Nil(t, err, "err should be nil") 209 | testCase.asserts(t, vl, c) 210 | }) 211 | } 212 | } 213 | 214 | func TestResetTTL(t *testing.T) { 215 | var testCases = []struct { 216 | name string 217 | tokenTTL time.Duration 218 | secretTTL time.Duration 219 | expectedTTL time.Duration 220 | }{ 221 | { 222 | name: "token TTL is smaller than secret TTL", 223 | tokenTTL: 1 * time.Hour, 224 | secretTTL: 2 * time.Hour, 225 | expectedTTL: 45 * time.Minute, 226 | }, 227 | { 228 | name: "token TTL is smaller than secret TTL", 229 | tokenTTL: 1 * time.Hour, 230 | secretTTL: 30 * time.Minute, 231 | expectedTTL: 1350 * time.Second, 232 | }, 233 | } 234 | 235 | for _, testCase := range testCases { 236 | t.Run( 237 | testCase.name, 238 | func(t *testing.T) { 239 | var vl = &Loader{ 240 | mut: &sync.Mutex{}, 241 | } 242 | vl.resetTTL(75, testCase.tokenTTL, testCase.secretTTL) 243 | require.Equal(t, testCase.expectedTTL, vl.Time()) 244 | }, 245 | ) 246 | } 247 | } 248 | 249 | func TestNew(t *testing.T) { 250 | t.Run( 251 | "no secret key panics", 252 | func(t *testing.T) { 253 | require.Panics( 254 | t, 255 | func() { 256 | New(&Config{}) 257 | }, 258 | ) 259 | }, 260 | ) 261 | 262 | t.Run( 263 | "no auth provider panics", 264 | func(t *testing.T) { 265 | require.Panics( 266 | t, 267 | func() { 268 | New(&Config{ 269 | Secrets: []Secret{{Key: "/dummy/secret/path"}}, 270 | }) 271 | }, 272 | ) 273 | }, 274 | ) 275 | 276 | t.Run( 277 | "no vault client panics", 278 | func(t *testing.T) { 279 | var ctrl = gomock.NewController(t) 280 | defer ctrl.Finish() 281 | var aP = mocks.NewMockAuthProvider(ctrl) 282 | require.Panics( 283 | t, 284 | func() { 285 | New(&Config{ 286 | Secrets: []Secret{{Key: "/dummy/secret/path"}}, 287 | AuthProvider: aP, 288 | }) 289 | }, 290 | ) 291 | }, 292 | ) 293 | 294 | t.Run( 295 | "no panic, no renewal", 296 | func(t *testing.T) { 297 | var ctrl = gomock.NewController(t) 298 | defer ctrl.Finish() 299 | var aP = mocks.NewMockAuthProvider(ctrl) 300 | var c, _ = vault.NewClient( 301 | vault.DefaultConfig(), 302 | ) 303 | var vl = New(&Config{ 304 | 305 | Secrets: []Secret{{Key: "/dummy/secret/path"}}, 306 | AuthProvider: aP, 307 | Client: c, 308 | }) 309 | 310 | require.Nil(t, vl.PollWatcher) 311 | }, 312 | ) 313 | 314 | t.Run( 315 | "no panic, with renewal", 316 | func(t *testing.T) { 317 | var ctrl = gomock.NewController(t) 318 | defer ctrl.Finish() 319 | var aP = mocks.NewMockAuthProvider(ctrl) 320 | var c, _ = vault.NewClient( 321 | vault.DefaultConfig(), 322 | ) 323 | var vl = New(&Config{ 324 | Secrets: []Secret{{Key: "/dummy/secretr/path"}}, 325 | AuthProvider: aP, 326 | Client: c, 327 | Renew: true, 328 | }) 329 | 330 | require.NotNil(t, vl.PollWatcher) 331 | }, 332 | ) 333 | } 334 | 335 | func TestMaxRetryRetryDelay(t *testing.T) { 336 | var ctrl = gomock.NewController(t) 337 | defer ctrl.Finish() 338 | var aP = mocks.NewMockAuthProvider(ctrl) 339 | var c, _ = vault.NewClient( 340 | vault.DefaultConfig(), 341 | ) 342 | var vl = New(&Config{ 343 | Secrets: []Secret{{Key: "/dummy/secretr/path"}}, 344 | AuthProvider: aP, 345 | Client: c, 346 | Renew: true, 347 | StopOnFailure: true, 348 | MaxRetry: 1, 349 | RetryDelay: 1 * time.Second, 350 | }) 351 | 352 | require.True(t, vl.StopOnFailure()) 353 | require.Equal(t, 1, vl.MaxRetry()) 354 | require.Equal(t, 1*time.Second, vl.RetryDelay()) 355 | } 356 | -------------------------------------------------------------------------------- /loader_mock_test.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: ./loader.go 3 | 4 | // Package konfig is a generated GoMock package. 5 | package konfig 6 | 7 | import ( 8 | gomock "github.com/golang/mock/gomock" 9 | reflect "reflect" 10 | time "time" 11 | ) 12 | 13 | // MockLoader is a mock of Loader interface 14 | type MockLoader struct { 15 | ctrl *gomock.Controller 16 | recorder *MockLoaderMockRecorder 17 | } 18 | 19 | // MockLoaderMockRecorder is the mock recorder for MockLoader 20 | type MockLoaderMockRecorder struct { 21 | mock *MockLoader 22 | } 23 | 24 | // NewMockLoader creates a new mock instance 25 | func NewMockLoader(ctrl *gomock.Controller) *MockLoader { 26 | mock := &MockLoader{ctrl: ctrl} 27 | mock.recorder = &MockLoaderMockRecorder{mock} 28 | return mock 29 | } 30 | 31 | // EXPECT returns an object that allows the caller to indicate expected use 32 | func (m *MockLoader) EXPECT() *MockLoaderMockRecorder { 33 | return m.recorder 34 | } 35 | 36 | // StopOnFailure mocks base method 37 | func (m *MockLoader) StopOnFailure() bool { 38 | m.ctrl.T.Helper() 39 | ret := m.ctrl.Call(m, "StopOnFailure") 40 | ret0, _ := ret[0].(bool) 41 | return ret0 42 | } 43 | 44 | // StopOnFailure indicates an expected call of StopOnFailure 45 | func (mr *MockLoaderMockRecorder) StopOnFailure() *gomock.Call { 46 | mr.mock.ctrl.T.Helper() 47 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StopOnFailure", reflect.TypeOf((*MockLoader)(nil).StopOnFailure)) 48 | } 49 | 50 | // Name mocks base method 51 | func (m *MockLoader) Name() string { 52 | m.ctrl.T.Helper() 53 | ret := m.ctrl.Call(m, "Name") 54 | ret0, _ := ret[0].(string) 55 | return ret0 56 | } 57 | 58 | // Name indicates an expected call of Name 59 | func (mr *MockLoaderMockRecorder) Name() *gomock.Call { 60 | mr.mock.ctrl.T.Helper() 61 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockLoader)(nil).Name)) 62 | } 63 | 64 | // Load mocks base method 65 | func (m *MockLoader) Load(arg0 Values) error { 66 | m.ctrl.T.Helper() 67 | ret := m.ctrl.Call(m, "Load", arg0) 68 | ret0, _ := ret[0].(error) 69 | return ret0 70 | } 71 | 72 | // Load indicates an expected call of Load 73 | func (mr *MockLoaderMockRecorder) Load(arg0 interface{}) *gomock.Call { 74 | mr.mock.ctrl.T.Helper() 75 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Load", reflect.TypeOf((*MockLoader)(nil).Load), arg0) 76 | } 77 | 78 | // MaxRetry mocks base method 79 | func (m *MockLoader) MaxRetry() int { 80 | m.ctrl.T.Helper() 81 | ret := m.ctrl.Call(m, "MaxRetry") 82 | ret0, _ := ret[0].(int) 83 | return ret0 84 | } 85 | 86 | // MaxRetry indicates an expected call of MaxRetry 87 | func (mr *MockLoaderMockRecorder) MaxRetry() *gomock.Call { 88 | mr.mock.ctrl.T.Helper() 89 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MaxRetry", reflect.TypeOf((*MockLoader)(nil).MaxRetry)) 90 | } 91 | 92 | // RetryDelay mocks base method 93 | func (m *MockLoader) RetryDelay() time.Duration { 94 | m.ctrl.T.Helper() 95 | ret := m.ctrl.Call(m, "RetryDelay") 96 | ret0, _ := ret[0].(time.Duration) 97 | return ret0 98 | } 99 | 100 | // RetryDelay indicates an expected call of RetryDelay 101 | func (mr *MockLoaderMockRecorder) RetryDelay() *gomock.Call { 102 | mr.mock.ctrl.T.Helper() 103 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RetryDelay", reflect.TypeOf((*MockLoader)(nil).RetryDelay)) 104 | } 105 | -------------------------------------------------------------------------------- /loaderwatcher.go: -------------------------------------------------------------------------------- 1 | package konfig 2 | 3 | // LoaderWatcher is an interface that implements both loader and watcher 4 | type LoaderWatcher interface { 5 | Loader 6 | Watcher 7 | } 8 | 9 | type loaderWatcher struct { 10 | Loader 11 | Watcher 12 | values Values 13 | name string 14 | s *S 15 | metrics *loaderMetrics 16 | loaderHooks LoaderHooks 17 | } 18 | 19 | // NewLoaderWatcher creates a new LoaderWatcher from a Loader and a Watcher 20 | func NewLoaderWatcher(l Loader, w Watcher) LoaderWatcher { 21 | return &loaderWatcher{ 22 | Loader: l, 23 | Watcher: w, 24 | } 25 | } 26 | 27 | func (c *S) newLoaderWatcher(l Loader, w Watcher, loaderHooks LoaderHooks) *loaderWatcher { 28 | var lw = &loaderWatcher{ 29 | Loader: l, 30 | Watcher: w, 31 | s: c, 32 | loaderHooks: loaderHooks, 33 | } 34 | 35 | if c.cfg.Metrics { 36 | lw.setMetrics() 37 | } 38 | 39 | return lw 40 | } 41 | -------------------------------------------------------------------------------- /metrics.go: -------------------------------------------------------------------------------- 1 | package konfig 2 | 3 | import "github.com/prometheus/client_golang/prometheus" 4 | 5 | var ( 6 | // MetricsConfigReload is the label for the prometheus counter for loader reload 7 | MetricsConfigReload = "konfig_loader_reload" 8 | // MetricsConfigReloadDuration is the label for the prometheus summary vector for loader reload duration 9 | MetricsConfigReloadDuration = "konfig_loader_reload_duration" 10 | ) 11 | 12 | const ( 13 | metricsSuccessLabel = "success" 14 | metricsFailureLabel = "failure" 15 | ) 16 | 17 | // LoaderMetrics is the structure holding the promtheus metrics objects 18 | type loaderMetrics struct { 19 | configReloadSuccess prometheus.Counter 20 | configReloadFailure prometheus.Counter 21 | configReloadDuration prometheus.Observer 22 | } 23 | 24 | func (lw *loaderWatcher) setMetrics() { 25 | var ( 26 | configReloadCounterVec = lw.s.metrics[MetricsConfigReload].(*prometheus.CounterVec) 27 | configReloadDurationSummaryVec = lw.s.metrics[MetricsConfigReloadDuration].(*prometheus.SummaryVec) 28 | ) 29 | 30 | lw.metrics = &loaderMetrics{ 31 | configReloadSuccess: configReloadCounterVec. 32 | WithLabelValues( 33 | metricsSuccessLabel, 34 | lw.s.name, 35 | lw.Name(), 36 | ), 37 | configReloadFailure: configReloadCounterVec. 38 | WithLabelValues( 39 | metricsFailureLabel, 40 | lw.s.name, 41 | lw.Name(), 42 | ), 43 | configReloadDuration: configReloadDurationSummaryVec. 44 | WithLabelValues( 45 | lw.s.name, 46 | lw.Name(), 47 | ), 48 | } 49 | } 50 | 51 | func (c *S) initMetrics() { 52 | c.metrics = map[string]prometheus.Collector{ 53 | MetricsConfigReload: prometheus.NewCounterVec( 54 | prometheus.CounterOpts{ 55 | Name: MetricsConfigReload, 56 | Help: "Number of config loader reload", 57 | }, 58 | []string{"result", "store", "loader"}, 59 | ), 60 | MetricsConfigReloadDuration: prometheus.NewSummaryVec( 61 | prometheus.SummaryOpts{ 62 | Name: MetricsConfigReloadDuration, 63 | Help: "Histogram for the config reload duration", 64 | Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, 65 | }, 66 | []string{"store", "loader"}, 67 | ), 68 | } 69 | } 70 | 71 | func (c *S) registerMetrics() error { 72 | for _, metric := range c.metrics { 73 | var err = prometheus.Register(metric) 74 | if err != nil && err != err.(prometheus.AlreadyRegisteredError) { 75 | return err 76 | } 77 | } 78 | return nil 79 | } 80 | -------------------------------------------------------------------------------- /metrics_test.go: -------------------------------------------------------------------------------- 1 | package konfig 2 | -------------------------------------------------------------------------------- /mocks/authprovider_mock.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: ./loader/klvault/authprovider.go 3 | 4 | // Package mocks is a generated GoMock package. 5 | package mocks 6 | 7 | import ( 8 | gomock "github.com/golang/mock/gomock" 9 | reflect "reflect" 10 | time "time" 11 | ) 12 | 13 | // MockAuthProvider is a mock of AuthProvider interface 14 | type MockAuthProvider struct { 15 | ctrl *gomock.Controller 16 | recorder *MockAuthProviderMockRecorder 17 | } 18 | 19 | // MockAuthProviderMockRecorder is the mock recorder for MockAuthProvider 20 | type MockAuthProviderMockRecorder struct { 21 | mock *MockAuthProvider 22 | } 23 | 24 | // NewMockAuthProvider creates a new mock instance 25 | func NewMockAuthProvider(ctrl *gomock.Controller) *MockAuthProvider { 26 | mock := &MockAuthProvider{ctrl: ctrl} 27 | mock.recorder = &MockAuthProviderMockRecorder{mock} 28 | return mock 29 | } 30 | 31 | // EXPECT returns an object that allows the caller to indicate expected use 32 | func (m *MockAuthProvider) EXPECT() *MockAuthProviderMockRecorder { 33 | return m.recorder 34 | } 35 | 36 | // Token mocks base method 37 | func (m *MockAuthProvider) Token() (string, time.Duration, error) { 38 | m.ctrl.T.Helper() 39 | ret := m.ctrl.Call(m, "Token") 40 | ret0, _ := ret[0].(string) 41 | ret1, _ := ret[1].(time.Duration) 42 | ret2, _ := ret[2].(error) 43 | return ret0, ret1, ret2 44 | } 45 | 46 | // Token indicates an expected call of Token 47 | func (mr *MockAuthProviderMockRecorder) Token() *gomock.Call { 48 | mr.mock.ctrl.T.Helper() 49 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Token", reflect.TypeOf((*MockAuthProvider)(nil).Token)) 50 | } 51 | -------------------------------------------------------------------------------- /mocks/client_mock.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: ./loader/klhttp/httploader.go 3 | 4 | // Package mocks is a generated GoMock package. 5 | package mocks 6 | 7 | import ( 8 | gomock "github.com/golang/mock/gomock" 9 | http "net/http" 10 | reflect "reflect" 11 | ) 12 | 13 | // MockClient is a mock of Client interface 14 | type MockClient struct { 15 | ctrl *gomock.Controller 16 | recorder *MockClientMockRecorder 17 | } 18 | 19 | // MockClientMockRecorder is the mock recorder for MockClient 20 | type MockClientMockRecorder struct { 21 | mock *MockClient 22 | } 23 | 24 | // NewMockClient creates a new mock instance 25 | func NewMockClient(ctrl *gomock.Controller) *MockClient { 26 | mock := &MockClient{ctrl: ctrl} 27 | mock.recorder = &MockClientMockRecorder{mock} 28 | return mock 29 | } 30 | 31 | // EXPECT returns an object that allows the caller to indicate expected use 32 | func (m *MockClient) EXPECT() *MockClientMockRecorder { 33 | return m.recorder 34 | } 35 | 36 | // Do mocks base method 37 | func (m *MockClient) Do(arg0 *http.Request) (*http.Response, error) { 38 | m.ctrl.T.Helper() 39 | ret := m.ctrl.Call(m, "Do", arg0) 40 | ret0, _ := ret[0].(*http.Response) 41 | ret1, _ := ret[1].(error) 42 | return ret0, ret1 43 | } 44 | 45 | // Do indicates an expected call of Do 46 | func (mr *MockClientMockRecorder) Do(arg0 interface{}) *gomock.Call { 47 | mr.mock.ctrl.T.Helper() 48 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Do", reflect.TypeOf((*MockClient)(nil).Do), arg0) 49 | } 50 | -------------------------------------------------------------------------------- /mocks/consulkv_mock.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: ./loader/klconsul/consulloader.go 3 | 4 | // Package mocks is a generated GoMock package. 5 | package mocks 6 | 7 | import ( 8 | reflect "reflect" 9 | 10 | gomock "github.com/golang/mock/gomock" 11 | api "github.com/hashicorp/consul/api" 12 | ) 13 | 14 | // MockConsulKV is a mock of ConsulKV interface 15 | type MockConsulKV struct { 16 | ctrl *gomock.Controller 17 | recorder *MockConsulKVMockRecorder 18 | } 19 | 20 | // MockConsulKVMockRecorder is the mock recorder for MockConsulKV 21 | type MockConsulKVMockRecorder struct { 22 | mock *MockConsulKV 23 | } 24 | 25 | // NewMockConsulKV creates a new mock instance 26 | func NewMockConsulKV(ctrl *gomock.Controller) *MockConsulKV { 27 | mock := &MockConsulKV{ctrl: ctrl} 28 | mock.recorder = &MockConsulKVMockRecorder{mock} 29 | return mock 30 | } 31 | 32 | // EXPECT returns an object that allows the caller to indicate expected use 33 | func (m *MockConsulKV) EXPECT() *MockConsulKVMockRecorder { 34 | return m.recorder 35 | } 36 | 37 | // Get mocks base method 38 | func (m *MockConsulKV) Get(key string, q *api.QueryOptions) (*api.KVPair, *api.QueryMeta, error) { 39 | m.ctrl.T.Helper() 40 | ret := m.ctrl.Call(m, "Get", key, q) 41 | ret0, _ := ret[0].(*api.KVPair) 42 | ret1, _ := ret[1].(*api.QueryMeta) 43 | ret2, _ := ret[2].(error) 44 | return ret0, ret1, ret2 45 | } 46 | 47 | // Get indicates an expected call of Get 48 | func (mr *MockConsulKVMockRecorder) Get(key, q interface{}) *gomock.Call { 49 | mr.mock.ctrl.T.Helper() 50 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockConsulKV)(nil).Get), key, q) 51 | } 52 | -------------------------------------------------------------------------------- /mocks/contexter_mock.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/lalamove/nui/ncontext (interfaces: Contexter) 3 | 4 | // Package mocks is a generated GoMock package. 5 | package mocks 6 | 7 | import ( 8 | context "context" 9 | gomock "github.com/golang/mock/gomock" 10 | reflect "reflect" 11 | time "time" 12 | ) 13 | 14 | // MockContexter is a mock of Contexter interface 15 | type MockContexter struct { 16 | ctrl *gomock.Controller 17 | recorder *MockContexterMockRecorder 18 | } 19 | 20 | // MockContexterMockRecorder is the mock recorder for MockContexter 21 | type MockContexterMockRecorder struct { 22 | mock *MockContexter 23 | } 24 | 25 | // NewMockContexter creates a new mock instance 26 | func NewMockContexter(ctrl *gomock.Controller) *MockContexter { 27 | mock := &MockContexter{ctrl: ctrl} 28 | mock.recorder = &MockContexterMockRecorder{mock} 29 | return mock 30 | } 31 | 32 | // EXPECT returns an object that allows the caller to indicate expected use 33 | func (m *MockContexter) EXPECT() *MockContexterMockRecorder { 34 | return m.recorder 35 | } 36 | 37 | // WithCancel mocks base method 38 | func (m *MockContexter) WithCancel(arg0 context.Context) (context.Context, context.CancelFunc) { 39 | m.ctrl.T.Helper() 40 | ret := m.ctrl.Call(m, "WithCancel", arg0) 41 | ret0, _ := ret[0].(context.Context) 42 | ret1, _ := ret[1].(context.CancelFunc) 43 | return ret0, ret1 44 | } 45 | 46 | // WithCancel indicates an expected call of WithCancel 47 | func (mr *MockContexterMockRecorder) WithCancel(arg0 interface{}) *gomock.Call { 48 | mr.mock.ctrl.T.Helper() 49 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WithCancel", reflect.TypeOf((*MockContexter)(nil).WithCancel), arg0) 50 | } 51 | 52 | // WithDeadline mocks base method 53 | func (m *MockContexter) WithDeadline(arg0 context.Context, arg1 time.Time) (context.Context, context.CancelFunc) { 54 | m.ctrl.T.Helper() 55 | ret := m.ctrl.Call(m, "WithDeadline", arg0, arg1) 56 | ret0, _ := ret[0].(context.Context) 57 | ret1, _ := ret[1].(context.CancelFunc) 58 | return ret0, ret1 59 | } 60 | 61 | // WithDeadline indicates an expected call of WithDeadline 62 | func (mr *MockContexterMockRecorder) WithDeadline(arg0, arg1 interface{}) *gomock.Call { 63 | mr.mock.ctrl.T.Helper() 64 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WithDeadline", reflect.TypeOf((*MockContexter)(nil).WithDeadline), arg0, arg1) 65 | } 66 | 67 | // WithTimeout mocks base method 68 | func (m *MockContexter) WithTimeout(arg0 context.Context, arg1 time.Duration) (context.Context, context.CancelFunc) { 69 | m.ctrl.T.Helper() 70 | ret := m.ctrl.Call(m, "WithTimeout", arg0, arg1) 71 | ret0, _ := ret[0].(context.Context) 72 | ret1, _ := ret[1].(context.CancelFunc) 73 | return ret0, ret1 74 | } 75 | 76 | // WithTimeout indicates an expected call of WithTimeout 77 | func (mr *MockContexterMockRecorder) WithTimeout(arg0, arg1 interface{}) *gomock.Call { 78 | mr.mock.ctrl.T.Helper() 79 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WithTimeout", reflect.TypeOf((*MockContexter)(nil).WithTimeout), arg0, arg1) 80 | } 81 | -------------------------------------------------------------------------------- /mocks/kv_mock.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: go.etcd.io/etcd/clientv3 (interfaces: KV) 3 | 4 | // Package mocks is a generated GoMock package. 5 | package mocks 6 | 7 | import ( 8 | context "context" 9 | gomock "github.com/golang/mock/gomock" 10 | clientv3 "go.etcd.io/etcd/clientv3" 11 | reflect "reflect" 12 | ) 13 | 14 | // MockKV is a mock of KV interface 15 | type MockKV struct { 16 | ctrl *gomock.Controller 17 | recorder *MockKVMockRecorder 18 | } 19 | 20 | // MockKVMockRecorder is the mock recorder for MockKV 21 | type MockKVMockRecorder struct { 22 | mock *MockKV 23 | } 24 | 25 | // NewMockKV creates a new mock instance 26 | func NewMockKV(ctrl *gomock.Controller) *MockKV { 27 | mock := &MockKV{ctrl: ctrl} 28 | mock.recorder = &MockKVMockRecorder{mock} 29 | return mock 30 | } 31 | 32 | // EXPECT returns an object that allows the caller to indicate expected use 33 | func (m *MockKV) EXPECT() *MockKVMockRecorder { 34 | return m.recorder 35 | } 36 | 37 | // Compact mocks base method 38 | func (m *MockKV) Compact(arg0 context.Context, arg1 int64, arg2 ...clientv3.CompactOption) (*clientv3.CompactResponse, error) { 39 | m.ctrl.T.Helper() 40 | varargs := []interface{}{arg0, arg1} 41 | for _, a := range arg2 { 42 | varargs = append(varargs, a) 43 | } 44 | ret := m.ctrl.Call(m, "Compact", varargs...) 45 | ret0, _ := ret[0].(*clientv3.CompactResponse) 46 | ret1, _ := ret[1].(error) 47 | return ret0, ret1 48 | } 49 | 50 | // Compact indicates an expected call of Compact 51 | func (mr *MockKVMockRecorder) Compact(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { 52 | mr.mock.ctrl.T.Helper() 53 | varargs := append([]interface{}{arg0, arg1}, arg2...) 54 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Compact", reflect.TypeOf((*MockKV)(nil).Compact), varargs...) 55 | } 56 | 57 | // Delete mocks base method 58 | func (m *MockKV) Delete(arg0 context.Context, arg1 string, arg2 ...clientv3.OpOption) (*clientv3.DeleteResponse, error) { 59 | m.ctrl.T.Helper() 60 | varargs := []interface{}{arg0, arg1} 61 | for _, a := range arg2 { 62 | varargs = append(varargs, a) 63 | } 64 | ret := m.ctrl.Call(m, "Delete", varargs...) 65 | ret0, _ := ret[0].(*clientv3.DeleteResponse) 66 | ret1, _ := ret[1].(error) 67 | return ret0, ret1 68 | } 69 | 70 | // Delete indicates an expected call of Delete 71 | func (mr *MockKVMockRecorder) Delete(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { 72 | mr.mock.ctrl.T.Helper() 73 | varargs := append([]interface{}{arg0, arg1}, arg2...) 74 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockKV)(nil).Delete), varargs...) 75 | } 76 | 77 | // Do mocks base method 78 | func (m *MockKV) Do(arg0 context.Context, arg1 clientv3.Op) (clientv3.OpResponse, error) { 79 | m.ctrl.T.Helper() 80 | ret := m.ctrl.Call(m, "Do", arg0, arg1) 81 | ret0, _ := ret[0].(clientv3.OpResponse) 82 | ret1, _ := ret[1].(error) 83 | return ret0, ret1 84 | } 85 | 86 | // Do indicates an expected call of Do 87 | func (mr *MockKVMockRecorder) Do(arg0, arg1 interface{}) *gomock.Call { 88 | mr.mock.ctrl.T.Helper() 89 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Do", reflect.TypeOf((*MockKV)(nil).Do), arg0, arg1) 90 | } 91 | 92 | // Get mocks base method 93 | func (m *MockKV) Get(arg0 context.Context, arg1 string, arg2 ...clientv3.OpOption) (*clientv3.GetResponse, error) { 94 | m.ctrl.T.Helper() 95 | varargs := []interface{}{arg0, arg1} 96 | for _, a := range arg2 { 97 | varargs = append(varargs, a) 98 | } 99 | ret := m.ctrl.Call(m, "Get", varargs...) 100 | ret0, _ := ret[0].(*clientv3.GetResponse) 101 | ret1, _ := ret[1].(error) 102 | return ret0, ret1 103 | } 104 | 105 | // Get indicates an expected call of Get 106 | func (mr *MockKVMockRecorder) Get(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { 107 | mr.mock.ctrl.T.Helper() 108 | varargs := append([]interface{}{arg0, arg1}, arg2...) 109 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockKV)(nil).Get), varargs...) 110 | } 111 | 112 | // Put mocks base method 113 | func (m *MockKV) Put(arg0 context.Context, arg1, arg2 string, arg3 ...clientv3.OpOption) (*clientv3.PutResponse, error) { 114 | m.ctrl.T.Helper() 115 | varargs := []interface{}{arg0, arg1, arg2} 116 | for _, a := range arg3 { 117 | varargs = append(varargs, a) 118 | } 119 | ret := m.ctrl.Call(m, "Put", varargs...) 120 | ret0, _ := ret[0].(*clientv3.PutResponse) 121 | ret1, _ := ret[1].(error) 122 | return ret0, ret1 123 | } 124 | 125 | // Put indicates an expected call of Put 126 | func (mr *MockKVMockRecorder) Put(arg0, arg1, arg2 interface{}, arg3 ...interface{}) *gomock.Call { 127 | mr.mock.ctrl.T.Helper() 128 | varargs := append([]interface{}{arg0, arg1, arg2}, arg3...) 129 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Put", reflect.TypeOf((*MockKV)(nil).Put), varargs...) 130 | } 131 | 132 | // Txn mocks base method 133 | func (m *MockKV) Txn(arg0 context.Context) clientv3.Txn { 134 | m.ctrl.T.Helper() 135 | ret := m.ctrl.Call(m, "Txn", arg0) 136 | ret0, _ := ret[0].(clientv3.Txn) 137 | return ret0 138 | } 139 | 140 | // Txn indicates an expected call of Txn 141 | func (mr *MockKVMockRecorder) Txn(arg0 interface{}) *gomock.Call { 142 | mr.mock.ctrl.T.Helper() 143 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Txn", reflect.TypeOf((*MockKV)(nil).Txn), arg0) 144 | } 145 | -------------------------------------------------------------------------------- /mocks/loader_mock.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: ./loader.go 3 | 4 | // Package mocks is a generated GoMock package. 5 | package mocks 6 | 7 | import ( 8 | gomock "github.com/golang/mock/gomock" 9 | konfig "github.com/lalamove/konfig" 10 | reflect "reflect" 11 | time "time" 12 | ) 13 | 14 | // MockLoader is a mock of Loader interface 15 | type MockLoader struct { 16 | ctrl *gomock.Controller 17 | recorder *MockLoaderMockRecorder 18 | } 19 | 20 | // MockLoaderMockRecorder is the mock recorder for MockLoader 21 | type MockLoaderMockRecorder struct { 22 | mock *MockLoader 23 | } 24 | 25 | // NewMockLoader creates a new mock instance 26 | func NewMockLoader(ctrl *gomock.Controller) *MockLoader { 27 | mock := &MockLoader{ctrl: ctrl} 28 | mock.recorder = &MockLoaderMockRecorder{mock} 29 | return mock 30 | } 31 | 32 | // EXPECT returns an object that allows the caller to indicate expected use 33 | func (m *MockLoader) EXPECT() *MockLoaderMockRecorder { 34 | return m.recorder 35 | } 36 | 37 | // StopOnFailure mocks base method 38 | func (m *MockLoader) StopOnFailure() bool { 39 | m.ctrl.T.Helper() 40 | ret := m.ctrl.Call(m, "StopOnFailure") 41 | ret0, _ := ret[0].(bool) 42 | return ret0 43 | } 44 | 45 | // StopOnFailure indicates an expected call of StopOnFailure 46 | func (mr *MockLoaderMockRecorder) StopOnFailure() *gomock.Call { 47 | mr.mock.ctrl.T.Helper() 48 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StopOnFailure", reflect.TypeOf((*MockLoader)(nil).StopOnFailure)) 49 | } 50 | 51 | // Name mocks base method 52 | func (m *MockLoader) Name() string { 53 | m.ctrl.T.Helper() 54 | ret := m.ctrl.Call(m, "Name") 55 | ret0, _ := ret[0].(string) 56 | return ret0 57 | } 58 | 59 | // Name indicates an expected call of Name 60 | func (mr *MockLoaderMockRecorder) Name() *gomock.Call { 61 | mr.mock.ctrl.T.Helper() 62 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockLoader)(nil).Name)) 63 | } 64 | 65 | // Load mocks base method 66 | func (m *MockLoader) Load(arg0 konfig.Values) error { 67 | m.ctrl.T.Helper() 68 | ret := m.ctrl.Call(m, "Load", arg0) 69 | ret0, _ := ret[0].(error) 70 | return ret0 71 | } 72 | 73 | // Load indicates an expected call of Load 74 | func (mr *MockLoaderMockRecorder) Load(arg0 interface{}) *gomock.Call { 75 | mr.mock.ctrl.T.Helper() 76 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Load", reflect.TypeOf((*MockLoader)(nil).Load), arg0) 77 | } 78 | 79 | // MaxRetry mocks base method 80 | func (m *MockLoader) MaxRetry() int { 81 | m.ctrl.T.Helper() 82 | ret := m.ctrl.Call(m, "MaxRetry") 83 | ret0, _ := ret[0].(int) 84 | return ret0 85 | } 86 | 87 | // MaxRetry indicates an expected call of MaxRetry 88 | func (mr *MockLoaderMockRecorder) MaxRetry() *gomock.Call { 89 | mr.mock.ctrl.T.Helper() 90 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MaxRetry", reflect.TypeOf((*MockLoader)(nil).MaxRetry)) 91 | } 92 | 93 | // RetryDelay mocks base method 94 | func (m *MockLoader) RetryDelay() time.Duration { 95 | m.ctrl.T.Helper() 96 | ret := m.ctrl.Call(m, "RetryDelay") 97 | ret0, _ := ret[0].(time.Duration) 98 | return ret0 99 | } 100 | 101 | // RetryDelay indicates an expected call of RetryDelay 102 | func (mr *MockLoaderMockRecorder) RetryDelay() *gomock.Call { 103 | mr.mock.ctrl.T.Helper() 104 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RetryDelay", reflect.TypeOf((*MockLoader)(nil).RetryDelay)) 105 | } 106 | -------------------------------------------------------------------------------- /mocks/logicalclient_mock.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: ./loader/klvault/vaultloader.go 3 | 4 | // Package mocks is a generated GoMock package. 5 | package mocks 6 | 7 | import ( 8 | gomock "github.com/golang/mock/gomock" 9 | api "github.com/hashicorp/vault/api" 10 | reflect "reflect" 11 | ) 12 | 13 | // MockLogicalClient is a mock of LogicalClient interface 14 | type MockLogicalClient struct { 15 | ctrl *gomock.Controller 16 | recorder *MockLogicalClientMockRecorder 17 | } 18 | 19 | // MockLogicalClientMockRecorder is the mock recorder for MockLogicalClient 20 | type MockLogicalClientMockRecorder struct { 21 | mock *MockLogicalClient 22 | } 23 | 24 | // NewMockLogicalClient creates a new mock instance 25 | func NewMockLogicalClient(ctrl *gomock.Controller) *MockLogicalClient { 26 | mock := &MockLogicalClient{ctrl: ctrl} 27 | mock.recorder = &MockLogicalClientMockRecorder{mock} 28 | return mock 29 | } 30 | 31 | // EXPECT returns an object that allows the caller to indicate expected use 32 | func (m *MockLogicalClient) EXPECT() *MockLogicalClientMockRecorder { 33 | return m.recorder 34 | } 35 | 36 | // Read mocks base method 37 | func (m *MockLogicalClient) Read(key string) (*api.Secret, error) { 38 | m.ctrl.T.Helper() 39 | ret := m.ctrl.Call(m, "Read", key) 40 | ret0, _ := ret[0].(*api.Secret) 41 | ret1, _ := ret[1].(error) 42 | return ret0, ret1 43 | } 44 | 45 | // Read indicates an expected call of Read 46 | func (mr *MockLogicalClientMockRecorder) Read(key interface{}) *gomock.Call { 47 | mr.mock.ctrl.T.Helper() 48 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Read", reflect.TypeOf((*MockLogicalClient)(nil).Read), key) 49 | } 50 | 51 | // Write mocks base method 52 | func (m *MockLogicalClient) Write(key string, data map[string]interface{}) (*api.Secret, error) { 53 | m.ctrl.T.Helper() 54 | ret := m.ctrl.Call(m, "Write", key, data) 55 | ret0, _ := ret[0].(*api.Secret) 56 | ret1, _ := ret[1].(error) 57 | return ret0, ret1 58 | } 59 | 60 | // Write indicates an expected call of Write 61 | func (mr *MockLogicalClientMockRecorder) Write(key, data interface{}) *gomock.Call { 62 | mr.mock.ctrl.T.Helper() 63 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Write", reflect.TypeOf((*MockLogicalClient)(nil).Write), key, data) 64 | } 65 | 66 | // ReadWithData mocks base method 67 | func (m *MockLogicalClient) ReadWithData(key string, data map[string][]string) (*api.Secret, error) { 68 | m.ctrl.T.Helper() 69 | ret := m.ctrl.Call(m, "ReadWithData", key, data) 70 | ret0, _ := ret[0].(*api.Secret) 71 | ret1, _ := ret[1].(error) 72 | return ret0, ret1 73 | } 74 | 75 | // ReadWithData indicates an expected call of ReadWithData 76 | func (mr *MockLogicalClientMockRecorder) ReadWithData(key, data interface{}) *gomock.Call { 77 | mr.mock.ctrl.T.Helper() 78 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadWithData", reflect.TypeOf((*MockLogicalClient)(nil).ReadWithData), key, data) 79 | } 80 | -------------------------------------------------------------------------------- /mocks/parser_mock.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: ./parser/parser.go 3 | 4 | // Package mocks is a generated GoMock package. 5 | package mocks 6 | 7 | import ( 8 | gomock "github.com/golang/mock/gomock" 9 | konfig "github.com/lalamove/konfig" 10 | io "io" 11 | reflect "reflect" 12 | ) 13 | 14 | // MockParser is a mock of Parser interface 15 | type MockParser struct { 16 | ctrl *gomock.Controller 17 | recorder *MockParserMockRecorder 18 | } 19 | 20 | // MockParserMockRecorder is the mock recorder for MockParser 21 | type MockParserMockRecorder struct { 22 | mock *MockParser 23 | } 24 | 25 | // NewMockParser creates a new mock instance 26 | func NewMockParser(ctrl *gomock.Controller) *MockParser { 27 | mock := &MockParser{ctrl: ctrl} 28 | mock.recorder = &MockParserMockRecorder{mock} 29 | return mock 30 | } 31 | 32 | // EXPECT returns an object that allows the caller to indicate expected use 33 | func (m *MockParser) EXPECT() *MockParserMockRecorder { 34 | return m.recorder 35 | } 36 | 37 | // Parse mocks base method 38 | func (m *MockParser) Parse(arg0 io.Reader, arg1 konfig.Values) error { 39 | m.ctrl.T.Helper() 40 | ret := m.ctrl.Call(m, "Parse", arg0, arg1) 41 | ret0, _ := ret[0].(error) 42 | return ret0 43 | } 44 | 45 | // Parse indicates an expected call of Parse 46 | func (mr *MockParserMockRecorder) Parse(arg0, arg1 interface{}) *gomock.Call { 47 | mr.mock.ctrl.T.Helper() 48 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Parse", reflect.TypeOf((*MockParser)(nil).Parse), arg0, arg1) 49 | } 50 | -------------------------------------------------------------------------------- /mocks/watcher_mock.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: ./watcher.go 3 | 4 | // Package mocks is a generated GoMock package. 5 | package mocks 6 | 7 | import ( 8 | gomock "github.com/golang/mock/gomock" 9 | reflect "reflect" 10 | ) 11 | 12 | // MockWatcher is a mock of Watcher interface 13 | type MockWatcher struct { 14 | ctrl *gomock.Controller 15 | recorder *MockWatcherMockRecorder 16 | } 17 | 18 | // MockWatcherMockRecorder is the mock recorder for MockWatcher 19 | type MockWatcherMockRecorder struct { 20 | mock *MockWatcher 21 | } 22 | 23 | // NewMockWatcher creates a new mock instance 24 | func NewMockWatcher(ctrl *gomock.Controller) *MockWatcher { 25 | mock := &MockWatcher{ctrl: ctrl} 26 | mock.recorder = &MockWatcherMockRecorder{mock} 27 | return mock 28 | } 29 | 30 | // EXPECT returns an object that allows the caller to indicate expected use 31 | func (m *MockWatcher) EXPECT() *MockWatcherMockRecorder { 32 | return m.recorder 33 | } 34 | 35 | // Start mocks base method 36 | func (m *MockWatcher) Start() error { 37 | m.ctrl.T.Helper() 38 | ret := m.ctrl.Call(m, "Start") 39 | ret0, _ := ret[0].(error) 40 | return ret0 41 | } 42 | 43 | // Start indicates an expected call of Start 44 | func (mr *MockWatcherMockRecorder) Start() *gomock.Call { 45 | mr.mock.ctrl.T.Helper() 46 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockWatcher)(nil).Start)) 47 | } 48 | 49 | // Done mocks base method 50 | func (m *MockWatcher) Done() <-chan struct{} { 51 | m.ctrl.T.Helper() 52 | ret := m.ctrl.Call(m, "Done") 53 | ret0, _ := ret[0].(<-chan struct{}) 54 | return ret0 55 | } 56 | 57 | // Done indicates an expected call of Done 58 | func (mr *MockWatcherMockRecorder) Done() *gomock.Call { 59 | mr.mock.ctrl.T.Helper() 60 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Done", reflect.TypeOf((*MockWatcher)(nil).Done)) 61 | } 62 | 63 | // Watch mocks base method 64 | func (m *MockWatcher) Watch() <-chan struct{} { 65 | m.ctrl.T.Helper() 66 | ret := m.ctrl.Call(m, "Watch") 67 | ret0, _ := ret[0].(<-chan struct{}) 68 | return ret0 69 | } 70 | 71 | // Watch indicates an expected call of Watch 72 | func (mr *MockWatcherMockRecorder) Watch() *gomock.Call { 73 | mr.mock.ctrl.T.Helper() 74 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Watch", reflect.TypeOf((*MockWatcher)(nil).Watch)) 75 | } 76 | 77 | // Close mocks base method 78 | func (m *MockWatcher) Close() error { 79 | m.ctrl.T.Helper() 80 | ret := m.ctrl.Call(m, "Close") 81 | ret0, _ := ret[0].(error) 82 | return ret0 83 | } 84 | 85 | // Close indicates an expected call of Close 86 | func (mr *MockWatcherMockRecorder) Close() *gomock.Call { 87 | mr.mock.ctrl.T.Helper() 88 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockWatcher)(nil).Close)) 89 | } 90 | 91 | // Err mocks base method 92 | func (m *MockWatcher) Err() error { 93 | m.ctrl.T.Helper() 94 | ret := m.ctrl.Call(m, "Err") 95 | ret0, _ := ret[0].(error) 96 | return ret0 97 | } 98 | 99 | // Err indicates an expected call of Err 100 | func (mr *MockWatcherMockRecorder) Err() *gomock.Call { 101 | mr.mock.ctrl.T.Helper() 102 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Err", reflect.TypeOf((*MockWatcher)(nil).Err)) 103 | } 104 | -------------------------------------------------------------------------------- /parser/kpjson/README.md: -------------------------------------------------------------------------------- 1 | # JSON Parser 2 | JSON parser parses a JSON file to a map[string]interface{} and then traverses the map and adds values into the config store flattening the JSON using dot notation for keys. 3 | 4 | Ex: 5 | ``` 6 | { 7 | "foo": "bar", 8 | "nested": { 9 | "firstName": "john", 10 | "lastName": "doe", 11 | "list": [ 12 | 1, 13 | 2, 14 | ] 15 | } 16 | } 17 | ``` 18 | Will add the following key/value to the config 19 | ``` 20 | "foo" => "bar" 21 | "nested.firstName" => "john" 22 | "nested.lastName" => "doe" 23 | "nested.list" => []int{1,2} 24 | ``` 25 | 26 | # Usage 27 | ``` 28 | err := kpjson.Parser.Parse(strings.NewReader(`{"foo":"bar"}`), konfig.Values{}) 29 | ``` 30 | -------------------------------------------------------------------------------- /parser/kpjson/jsonparser.go: -------------------------------------------------------------------------------- 1 | package kpjson 2 | 3 | import ( 4 | "encoding/json" 5 | "io" 6 | 7 | "github.com/lalamove/konfig" 8 | "github.com/lalamove/konfig/parser" 9 | "github.com/lalamove/konfig/parser/kpmap" 10 | ) 11 | 12 | // Parser parses the given json io.Reader and adds values in dot.path notation into the konfig.Store 13 | var Parser = parser.Func(func(r io.Reader, s konfig.Values) error { 14 | // unmarshal the JSON into map[string]interface{} 15 | var dec = json.NewDecoder(r) 16 | 17 | var d = make(map[string]interface{}) 18 | var err = dec.Decode(&d) 19 | if err != nil { 20 | return err 21 | } 22 | 23 | kpmap.PopFlatten(d, s) 24 | 25 | return nil 26 | }) 27 | -------------------------------------------------------------------------------- /parser/kpjson/jsonparser_test.go: -------------------------------------------------------------------------------- 1 | package kpjson 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | "github.com/lalamove/konfig" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestJSONParser(t *testing.T) { 12 | var testCases = []struct { 13 | name string 14 | json string 15 | asserts func(t *testing.T, s konfig.Values) 16 | }{ 17 | { 18 | name: "simple 1 level json object", 19 | json: `{"foo":"bar","bar":"foo","int":1}`, 20 | asserts: func(t *testing.T, v konfig.Values) { 21 | require.Equal( 22 | t, 23 | "bar", 24 | v["foo"], 25 | ) 26 | 27 | require.Equal( 28 | t, 29 | "foo", 30 | v["bar"], 31 | ) 32 | 33 | require.Equal( 34 | t, 35 | float64(1), 36 | v["int"], 37 | ) 38 | }, 39 | }, 40 | { 41 | name: "nested objects", 42 | json: `{ 43 | "foo": "bar", 44 | "bar": { 45 | "foo": "hello world!", 46 | "bool": true, 47 | "nested": { 48 | "john": "doe" 49 | } 50 | } 51 | }`, 52 | asserts: func(t *testing.T, v konfig.Values) { 53 | require.Equal( 54 | t, 55 | "bar", 56 | v["foo"], 57 | ) 58 | 59 | require.Equal( 60 | t, 61 | "hello world!", 62 | v["bar.foo"], 63 | ) 64 | 65 | require.Equal( 66 | t, 67 | true, 68 | v["bar.bool"], 69 | ) 70 | 71 | require.Equal( 72 | t, 73 | "doe", 74 | v["bar.nested.john"], 75 | ) 76 | }, 77 | }, 78 | } 79 | 80 | for _, testCase := range testCases { 81 | t.Run( 82 | testCase.name, 83 | func(t *testing.T) { 84 | konfig.Init(konfig.DefaultConfig()) 85 | 86 | var v = konfig.Values{} 87 | var err = Parser.Parse( 88 | strings.NewReader( 89 | testCase.json, 90 | ), 91 | v, 92 | ) 93 | 94 | require.Nil(t, err) 95 | testCase.asserts(t, v) 96 | }, 97 | ) 98 | } 99 | } 100 | 101 | func TestParserErr(t *testing.T) { 102 | var err = Parser.Parse( 103 | strings.NewReader( 104 | `invalid`, 105 | ), 106 | konfig.Values{}, 107 | ) 108 | require.NotNil(t, err) 109 | } 110 | -------------------------------------------------------------------------------- /parser/kpkeyval/README.md: -------------------------------------------------------------------------------- 1 | # KV Parser 2 | KV parser is a key value parser to parse an io.Reader's content of key/values with a configurable separator and add it into a konfig.Store. 3 | 4 | Ex: 5 | ``` 6 | bar=foo 7 | foo=bar 8 | ``` 9 | Will add the following key/value to the config 10 | ``` 11 | "foo" => "bar" 12 | "bar" => "foo" 13 | ``` 14 | 15 | # Usage 16 | ``` 17 | func main() { 18 | var v = konfig.Values{} 19 | var p = kpkeyval.New(&kpkeyval.Config{}) 20 | 21 | p.Parse(strings.NewReader( 22 | "bar=foo\nfoo=bar", 23 | ), v) 24 | 25 | fmt.Println(v) // map[bar:foo foo:bar] 26 | } 27 | ``` 28 | -------------------------------------------------------------------------------- /parser/kpkeyval/kvparser.go: -------------------------------------------------------------------------------- 1 | // Package kpkeyval provides a key value parser to parse an io.Reader's content 2 | // of key/values with a configurable separator and add it into a konfig.Store. 3 | package kpkeyval 4 | 5 | import ( 6 | "bufio" 7 | "errors" 8 | "io" 9 | "strings" 10 | 11 | "github.com/lalamove/konfig" 12 | "github.com/lalamove/konfig/parser" 13 | ) 14 | 15 | // DefaultSep is the default key value separator 16 | const DefaultSep = "=" 17 | 18 | // ErrInvalidConfigFileFormat is the error returned when a problem is encountered when parsing the 19 | // config file 20 | var ( 21 | ErrInvalidConfigFileFormat = errors.New("Err invalid file format") 22 | // make sure Parser implements fileloader.Parser 23 | _ parser.Parser = (*Parser)(nil) 24 | ) 25 | 26 | // Config is the configuration of the key value parser 27 | type Config struct { 28 | // Sep is the separator between keys and values 29 | Sep string 30 | } 31 | 32 | // Parser implements fileloader.Parser 33 | // It parses a file of key/values with a specific separator 34 | // and stores in the konfig.Store 35 | type Parser struct { 36 | cfg *Config 37 | } 38 | 39 | // New creates a new parser with the given config 40 | func New(cfg *Config) *Parser { 41 | if cfg.Sep == "" { 42 | cfg.Sep = DefaultSep 43 | } 44 | return &Parser{ 45 | cfg: cfg, 46 | } 47 | } 48 | 49 | // Parse implement the fileloader.Parser interface 50 | func (k *Parser) Parse(r io.Reader, cfg konfig.Values) error { 51 | var scanner = bufio.NewScanner(r) 52 | for scanner.Scan() { 53 | var cfgKey = strings.Split(scanner.Text(), k.cfg.Sep) 54 | if len(cfgKey) < 2 { 55 | return ErrInvalidConfigFileFormat 56 | } 57 | cfg.Set(cfgKey[0], strings.Join(cfgKey[1:], k.cfg.Sep)) 58 | } 59 | return scanner.Err() 60 | } 61 | -------------------------------------------------------------------------------- /parser/kpkeyval/kvparser_test.go: -------------------------------------------------------------------------------- 1 | package kpkeyval 2 | 3 | import ( 4 | "errors" 5 | "io" 6 | "strings" 7 | "testing" 8 | 9 | "github.com/lalamove/konfig" 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | type ErrReader struct{} 14 | 15 | func (e ErrReader) Read([]byte) (int, error) { 16 | return 0, errors.New("") 17 | } 18 | 19 | func TestKVParser(t *testing.T) { 20 | var testCases = []struct { 21 | name string 22 | err bool 23 | reader io.Reader 24 | sep string 25 | asserts func(t *testing.T, cfg konfig.Values) 26 | }{ 27 | { 28 | name: `no error, default separator`, 29 | err: false, 30 | reader: strings.NewReader("BAR=FOO\nFOO=BAR"), 31 | asserts: func(t *testing.T, cfg konfig.Values) { 32 | require.Equal(t, "BAR", cfg["FOO"]) 33 | require.Equal(t, "FOO", cfg["BAR"]) 34 | }, 35 | }, 36 | { 37 | name: `no error, custom separator`, 38 | err: false, 39 | sep: ":", 40 | reader: strings.NewReader("BAR:FOO\nFOO:BAR"), 41 | asserts: func(t *testing.T, cfg konfig.Values) { 42 | require.Equal(t, "BAR", cfg["FOO"]) 43 | require.Equal(t, "FOO", cfg["BAR"]) 44 | }, 45 | }, 46 | { 47 | name: `err invalid format`, 48 | err: true, 49 | sep: ":", 50 | reader: strings.NewReader("BAR\nFOO"), 51 | }, 52 | { 53 | name: `err scanner`, 54 | err: true, 55 | sep: ":", 56 | reader: ErrReader{}, 57 | }, 58 | } 59 | 60 | for _, testCase := range testCases { 61 | t.Run( 62 | testCase.name, 63 | func(t *testing.T) { 64 | var v = konfig.Values{} 65 | var p = New(&Config{ 66 | Sep: testCase.sep, 67 | }) 68 | 69 | var err = p.Parse(testCase.reader, v) 70 | if testCase.err { 71 | require.NotNil(t, err, "err should not be nil") 72 | return 73 | } 74 | require.Nil(t, err, "err should be nil") 75 | 76 | testCase.asserts(t, v) 77 | }, 78 | ) 79 | } 80 | } 81 | 82 | func TestParserErr(t *testing.T) { 83 | var err = New(&Config{}).Parse( 84 | strings.NewReader( 85 | `invalid`, 86 | ), 87 | konfig.Values{}, 88 | ) 89 | require.NotNil(t, err) 90 | } 91 | -------------------------------------------------------------------------------- /parser/kpmap/README.md: -------------------------------------------------------------------------------- 1 | # Map Parser 2 | kpmap package provides a function `PopFlatten` for populating a konfig.Store by flatteing a map[string]interface{}, which is used by json/toml/yaml parser. 3 | 4 | # Usage 5 | ``` 6 | func main() { 7 | var m = map[string]interface{}{ 8 | "test": map[string]interface{}{ 9 | "foo": "bar", 10 | }, 11 | "testIface": map[interface{}]interface{}{ 12 | 1: "bar", 13 | "testIface": map[interface{}]interface{}{ 14 | "foo": "bar", 15 | }, 16 | "test": map[string]interface{}{ 17 | "foo": "bar", 18 | }, 19 | }, 20 | } 21 | var v = konfig.Values{} 22 | kpmap.PopFlatten(m, v) 23 | 24 | fmt.Println(v) // map[test.foo:bar testIface.1:bar testIface.test.foo:bar testIface.testIface.foo:bar] 25 | } 26 | ``` 27 | -------------------------------------------------------------------------------- /parser/kpmap/mapparser.go: -------------------------------------------------------------------------------- 1 | package kpmap 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/lalamove/konfig" 7 | ) 8 | 9 | func traverseMapIface(m map[interface{}]interface{}, s konfig.Values, p string) { 10 | for k, v := range m { 11 | var ks = fmt.Sprintf("%v", k) 12 | switch vt := v.(type) { 13 | case map[string]interface{}: 14 | traverseMap(vt, s, p+ks+konfig.KeySep) 15 | case map[interface{}]interface{}: 16 | traverseMapIface(vt, s, p+ks+konfig.KeySep) 17 | default: 18 | s.Set(p+ks, v) 19 | } 20 | } 21 | } 22 | 23 | func traverseMap(m map[string]interface{}, s konfig.Values, p string) { 24 | for k, v := range m { 25 | switch vt := v.(type) { 26 | case map[string]interface{}: 27 | traverseMap(vt, s, p+k+konfig.KeySep) 28 | case map[interface{}]interface{}: 29 | traverseMapIface(vt, s, p+k+konfig.KeySep) 30 | default: 31 | s.Set(p+k, v) 32 | } 33 | } 34 | } 35 | 36 | // PopFlatten populates a konfig.Store by flatteing a map[string]interface{} 37 | func PopFlatten(m map[string]interface{}, s konfig.Values) { 38 | traverseMap(m, s, "") 39 | } 40 | -------------------------------------------------------------------------------- /parser/kpmap/mapparser_test.go: -------------------------------------------------------------------------------- 1 | package kpmap 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/lalamove/konfig" 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestMapPopFlatten(t *testing.T) { 11 | var m = map[string]interface{}{ 12 | "test": map[string]interface{}{ 13 | "foo": "bar", 14 | }, 15 | "testIface": map[interface{}]interface{}{ 16 | 1: "bar", 17 | "testIface": map[interface{}]interface{}{ 18 | "foo": "bar", 19 | }, 20 | "test": map[string]interface{}{ 21 | "foo": "bar", 22 | }, 23 | }, 24 | } 25 | 26 | var v = konfig.Values{} 27 | PopFlatten(m, v) 28 | 29 | require.Equal( 30 | t, 31 | konfig.Values{ 32 | "test.foo": "bar", 33 | "testIface.1": "bar", 34 | "testIface.testIface.foo": "bar", 35 | "testIface.test.foo": "bar", 36 | }, 37 | v, 38 | ) 39 | } 40 | -------------------------------------------------------------------------------- /parser/kptoml/README.md: -------------------------------------------------------------------------------- 1 | # TOML Parser 2 | TOML parser parses a TOML file to a map[string]interface{} and then traverses the map and adds values into the config store flattening the TOML using dot notation for keys. 3 | 4 | Ex: 5 | ``` 6 | foo = bar 7 | 8 | [nested] 9 | firstName = "john" 10 | lastName = "doe" 11 | list = [1, 2] 12 | ``` 13 | Will add the following key/value to the config 14 | ``` 15 | "foo" => "bar" 16 | "nested.firstName" => "john" 17 | "nested.lastName" => "doe" 18 | "nested.list" => []int{1,2} 19 | ``` 20 | 21 | # Usage 22 | ``` 23 | err := kptoml.Parser.Parse(strings.NewReader(`foo: "bar"`), konfig.Values{}) 24 | ``` 25 | -------------------------------------------------------------------------------- /parser/kptoml/tomlparser.go: -------------------------------------------------------------------------------- 1 | package kptoml 2 | 3 | import ( 4 | "io" 5 | 6 | "github.com/BurntSushi/toml" 7 | "github.com/lalamove/konfig" 8 | "github.com/lalamove/konfig/parser" 9 | "github.com/lalamove/konfig/parser/kpmap" 10 | ) 11 | 12 | // Parser parses the given json io.Reader and adds values in dot.path notation into the konfig.Store 13 | var Parser = parser.Func(func(r io.Reader, s konfig.Values) error { 14 | // unmarshal the JSON into map[string]interface{} 15 | var d = make(map[string]interface{}) 16 | var _, err = toml.DecodeReader(r, &d) 17 | if err != nil { 18 | return err 19 | } 20 | 21 | kpmap.PopFlatten(d, s) 22 | 23 | return nil 24 | }) 25 | -------------------------------------------------------------------------------- /parser/kptoml/tomlparser_test.go: -------------------------------------------------------------------------------- 1 | package kptoml 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | "github.com/lalamove/konfig" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestJSONParser(t *testing.T) { 12 | var testCases = []struct { 13 | name string 14 | toml string 15 | asserts func(t *testing.T, v konfig.Values) 16 | }{ 17 | { 18 | name: "simple 1 level toml object", 19 | toml: ` 20 | foo = "bar" 21 | bar = "foo" 22 | int = 1 23 | `, 24 | asserts: func(t *testing.T, v konfig.Values) { 25 | require.Equal( 26 | t, 27 | "bar", 28 | v["foo"], 29 | ) 30 | 31 | require.Equal( 32 | t, 33 | "foo", 34 | v["bar"], 35 | ) 36 | 37 | require.Equal( 38 | t, 39 | int64(1), 40 | v["int"], 41 | ) 42 | }, 43 | }, 44 | { 45 | name: "nested objects", 46 | toml: ` 47 | foo = "bar" 48 | 49 | [bar] 50 | foo = "hello world!" 51 | bool = true 52 | 53 | [bar.nested] 54 | john = "doe" 55 | `, 56 | asserts: func(t *testing.T, v konfig.Values) { 57 | require.Equal( 58 | t, 59 | "bar", 60 | v["foo"], 61 | ) 62 | 63 | require.Equal( 64 | t, 65 | "hello world!", 66 | v["bar.foo"], 67 | ) 68 | 69 | require.Equal( 70 | t, 71 | true, 72 | v["bar.bool"], 73 | ) 74 | 75 | require.Equal( 76 | t, 77 | "doe", 78 | v["bar.nested.john"], 79 | ) 80 | }, 81 | }, 82 | } 83 | 84 | for _, testCase := range testCases { 85 | t.Run( 86 | testCase.name, 87 | func(t *testing.T) { 88 | konfig.Init(konfig.DefaultConfig()) 89 | 90 | var v = konfig.Values{} 91 | var err = Parser.Parse( 92 | strings.NewReader( 93 | testCase.toml, 94 | ), 95 | v, 96 | ) 97 | 98 | require.Nil(t, err) 99 | testCase.asserts(t, v) 100 | }, 101 | ) 102 | } 103 | } 104 | 105 | func TestParserErr(t *testing.T) { 106 | var err = Parser.Parse( 107 | strings.NewReader( 108 | `invalid`, 109 | ), 110 | konfig.Values{}, 111 | ) 112 | require.NotNil(t, err) 113 | } 114 | -------------------------------------------------------------------------------- /parser/kpyaml/README.md: -------------------------------------------------------------------------------- 1 | # YAML Parser 2 | YAML parser parses a YAML file to a map[string]interface{} and then traverses the map and adds values into the config store flattening the YAML using dot notation for keys. 3 | 4 | Ex: 5 | ``` 6 | foo: "bar" 7 | nested: 8 | firstName: "john" 9 | lastName: "doe" 10 | list: 11 | - 1 12 | - 2 13 | ``` 14 | Will add the following key/value to the config 15 | ``` 16 | "foo" => "bar" 17 | "nested.firstName" => "john" 18 | "nested.lastName" => "doe" 19 | "nested.list" => []int{1,2} 20 | ``` 21 | 22 | # Usage 23 | ``` 24 | err := kpyaml.Parser.Parse(strings.NewReader(`foo: "bar"`), konfig.Values{}) 25 | ``` 26 | -------------------------------------------------------------------------------- /parser/kpyaml/yamlparser.go: -------------------------------------------------------------------------------- 1 | package kpyaml 2 | 3 | import ( 4 | "io" 5 | 6 | "github.com/lalamove/konfig" 7 | "github.com/lalamove/konfig/parser" 8 | "github.com/lalamove/konfig/parser/kpmap" 9 | yaml "gopkg.in/yaml.v2" 10 | ) 11 | 12 | // Parser is the YAML Parser it implements parser.Parser 13 | var Parser = parser.Func(func(r io.Reader, s konfig.Values) error { 14 | var dec = yaml.NewDecoder(r) 15 | 16 | var d = make(map[string]interface{}) 17 | var err = dec.Decode(&d) 18 | if err != nil { 19 | return err 20 | } 21 | 22 | kpmap.PopFlatten(d, s) 23 | 24 | return nil 25 | }) 26 | -------------------------------------------------------------------------------- /parser/kpyaml/yamlparser_test.go: -------------------------------------------------------------------------------- 1 | package kpyaml 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | "github.com/lalamove/konfig" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestYAMLParser(t *testing.T) { 12 | var testCases = []struct { 13 | name string 14 | yaml string 15 | asserts func(t *testing.T, v konfig.Values) 16 | }{ 17 | { 18 | name: "simple 1 level yaml object", 19 | yaml: `foo: bar 20 | bar: foo 21 | int: 1`, 22 | asserts: func(t *testing.T, v konfig.Values) { 23 | require.Equal( 24 | t, 25 | "bar", 26 | v["foo"], 27 | ) 28 | 29 | require.Equal( 30 | t, 31 | "foo", 32 | v["bar"], 33 | ) 34 | 35 | require.Equal( 36 | t, 37 | 1, 38 | v["int"], 39 | ) 40 | }, 41 | }, 42 | { 43 | name: "nested objects", 44 | yaml: `foo: bar 45 | bar: 46 | foo: "hello world!" 47 | bool: true 48 | nested: 49 | john: "doe"`, 50 | asserts: func(t *testing.T, v konfig.Values) { 51 | require.Equal( 52 | t, 53 | "bar", 54 | v["foo"], 55 | ) 56 | 57 | require.Equal( 58 | t, 59 | "hello world!", 60 | v["bar.foo"], 61 | ) 62 | 63 | require.Equal( 64 | t, 65 | true, 66 | v["bar.bool"], 67 | ) 68 | 69 | require.Equal( 70 | t, 71 | "doe", 72 | v["bar.nested.john"], 73 | ) 74 | }, 75 | }, 76 | { 77 | name: "nested objects", 78 | yaml: `foo: bar 79 | bar: 80 | - "hello world!" 81 | - "yaml"`, 82 | asserts: func(t *testing.T, v konfig.Values) { 83 | require.Equal( 84 | t, 85 | []interface{}{"hello world!", "yaml"}, 86 | v["bar"], 87 | ) 88 | }, 89 | }, 90 | } 91 | 92 | for _, testCase := range testCases { 93 | t.Run( 94 | testCase.name, 95 | func(t *testing.T) { 96 | konfig.Init(konfig.DefaultConfig()) 97 | var v = konfig.Values{} 98 | var err = Parser.Parse( 99 | strings.NewReader( 100 | testCase.yaml, 101 | ), 102 | v, 103 | ) 104 | require.Nil(t, err) 105 | testCase.asserts(t, v) 106 | }, 107 | ) 108 | } 109 | } 110 | 111 | func TestParserErr(t *testing.T) { 112 | var err = Parser.Parse( 113 | strings.NewReader( 114 | `invalid`, 115 | ), 116 | konfig.Values{}, 117 | ) 118 | require.NotNil(t, err) 119 | } 120 | -------------------------------------------------------------------------------- /parser/parser.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "io" 5 | 6 | "github.com/lalamove/konfig" 7 | ) 8 | 9 | var _ Parser = (Func)(nil) 10 | 11 | var _ Parser = (*NopParser)(nil) 12 | 13 | // Parser is the interface to implement to parse a config file 14 | type Parser interface { 15 | Parse(io.Reader, konfig.Values) error 16 | } 17 | 18 | // Func is a function implementing the Parser interface 19 | type Func func(io.Reader, konfig.Values) error 20 | 21 | // Parse implements Parser interface 22 | func (f Func) Parse(r io.Reader, s konfig.Values) error { 23 | return f(r, s) 24 | } 25 | 26 | // NopParser is a nil parser, useful for unit test 27 | type NopParser struct { 28 | Err error 29 | } 30 | 31 | // Parse implements Parser interface 32 | func (p NopParser) Parse(r io.Reader, s konfig.Values) error { 33 | return p.Err 34 | } 35 | -------------------------------------------------------------------------------- /parser/parser_test.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "io" 5 | "testing" 6 | 7 | "github.com/lalamove/konfig" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestParserFunc(t *testing.T) { 12 | var ran bool 13 | var f = Func(func(r io.Reader, s konfig.Values) error { 14 | ran = true 15 | return nil 16 | }) 17 | f.Parse(nil, nil) 18 | require.True(t, ran) 19 | } 20 | 21 | func TestNopParser(t *testing.T) { 22 | var p = NopParser{} 23 | require.Nil(t, p.Parse(nil, nil)) 24 | } 25 | -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | echo "" > coverage.txt 5 | 6 | for d in $(go list ./... | grep -v vendor); do 7 | go test -race -v -coverprofile=profile.out -covermode=atomic $d 8 | if [ -f profile.out ]; then 9 | cat profile.out >> coverage.txt 10 | rm profile.out 11 | fi 12 | done 13 | -------------------------------------------------------------------------------- /test/configfile_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/lalamove/konfig" 8 | "github.com/lalamove/konfig/loader/klfile" 9 | "github.com/lalamove/konfig/parser/kpyaml" 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | type DBConfig struct { 14 | MySQL MySQLConfig `konfig:"mysql"` 15 | Redis RedisConfig `konfig:"redis"` 16 | } 17 | 18 | type RedisConfig struct { 19 | H string `konfig:"host"` 20 | } 21 | 22 | type MySQLConfig struct { 23 | U string `konfig:"username"` 24 | PW string `konfig:"password"` 25 | MaxOpenConn int 26 | } 27 | 28 | type VaultConfig struct { 29 | Enable bool 30 | Server string 31 | Secret string `konfig:"dbSecret"` 32 | } 33 | 34 | type YAMLConfig struct { 35 | Debug bool `konfig:"debug"` 36 | SQLDebug bool `konfig:"sqlDebug"` 37 | DB DBConfig 38 | Port string `konfig:"http.port"` 39 | Vault VaultConfig 40 | } 41 | 42 | func TestYAMLFile(t *testing.T) { 43 | var expectedConfig = YAMLConfig{ 44 | Debug: true, 45 | SQLDebug: true, 46 | Port: "8081", 47 | DB: DBConfig{ 48 | MySQL: MySQLConfig{ 49 | U: "username", 50 | PW: "password", 51 | MaxOpenConn: 10, 52 | }, 53 | Redis: RedisConfig{ 54 | H: "127.0.0.1", 55 | }, 56 | }, 57 | Vault: VaultConfig{ 58 | Enable: true, 59 | Server: "http://127.0.0.1:8200", 60 | Secret: "/secret/db1", 61 | }, 62 | } 63 | 64 | konfig.Init(&konfig.Config{ 65 | NoExitOnError: true, 66 | }) 67 | 68 | konfig.Bind(YAMLConfig{}) 69 | 70 | konfig.RegisterLoader( 71 | klfile.New(&klfile.Config{ 72 | Files: []klfile.File{ 73 | { 74 | Path: "./data/cfg.yml", 75 | Parser: kpyaml.Parser, 76 | }, 77 | }, 78 | MaxRetry: 2, 79 | RetryDelay: 1 * time.Second, 80 | Debug: true, 81 | }), 82 | ) 83 | 84 | if err := konfig.Load(); err != nil { 85 | t.Error(err) 86 | } 87 | 88 | require.Equal(t, expectedConfig, konfig.Value()) 89 | } 90 | -------------------------------------------------------------------------------- /test/data/cfg: -------------------------------------------------------------------------------- 1 | KEY=VALUE 2 | KEY1=VALUE1 3 | KEY2=VALUE2=efwefwef 4 | -------------------------------------------------------------------------------- /test/data/cfg.yml: -------------------------------------------------------------------------------- 1 | debug: true 2 | sqlDebug: true 3 | vault: 4 | enable: true 5 | server: "http://127.0.0.1:8200" 6 | dbSecret: "/secret/db1" 7 | db: 8 | mysql: 9 | username: "username" 10 | password: "password" 11 | maxOpenConn: 10 12 | redis: 13 | host: "127.0.0.1" 14 | http: 15 | port: "8081" 16 | -------------------------------------------------------------------------------- /util_test.go: -------------------------------------------------------------------------------- 1 | package konfig 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestUtils(t *testing.T) { 11 | var testCases = []struct { 12 | name string 13 | test func(t *testing.T) 14 | }{ 15 | { 16 | name: "MustGet", 17 | test: func(t *testing.T) { 18 | Set("test", true) 19 | require.Equal(t, true, MustGet("test")) 20 | require.Panics(t, func() { 21 | MustGet("foo") 22 | }) 23 | }, 24 | }, 25 | { 26 | name: "IntSuccess", 27 | test: func(t *testing.T) { 28 | Set("foo", 1) 29 | var i = Int("foo") 30 | require.Equal(t, 1, i) 31 | }, 32 | }, 33 | { 34 | name: "MustIntSuccess", 35 | test: func(t *testing.T) { 36 | Set("foo", 1) 37 | var i int 38 | require.NotPanics(t, func() { 39 | i = MustInt("foo") 40 | }) 41 | require.Equal(t, 1, i) 42 | }, 43 | }, 44 | { 45 | name: "MustIntPanic", 46 | test: func(t *testing.T) { 47 | require.Panics(t, func() { 48 | MustInt("foo") 49 | }) 50 | }, 51 | }, 52 | { 53 | name: "StringSuccess", 54 | test: func(t *testing.T) { 55 | Set("foo", "bar") 56 | var s = String("foo") 57 | require.Equal(t, "bar", s) 58 | }, 59 | }, 60 | { 61 | name: "MustStringSuccess", 62 | test: func(t *testing.T) { 63 | Set("foo", "bar") 64 | var s string 65 | require.NotPanics(t, func() { s = MustString("foo") }) 66 | require.Equal(t, "bar", s) 67 | }, 68 | }, 69 | { 70 | 71 | name: "MustStringPanics", 72 | test: func(t *testing.T) { 73 | require.Panics(t, func() { MustString("foo") }) 74 | }, 75 | }, 76 | { 77 | 78 | name: "FloatSuccess", 79 | test: func(t *testing.T) { 80 | Set("foo", 2.1) 81 | var f = Float("foo") 82 | require.Equal(t, 2.1, f) 83 | }, 84 | }, 85 | { 86 | name: "MustFloatSuccess", 87 | test: func(t *testing.T) { 88 | Set("foo", 1.1) 89 | var f float64 90 | require.NotPanics(t, func() { f = MustFloat("foo") }) 91 | require.Equal(t, 1.1, f) 92 | }, 93 | }, 94 | { 95 | name: "MustFloatPanics", 96 | test: func(t *testing.T) { 97 | require.Panics(t, func() { MustFloat("foo") }) 98 | }, 99 | }, 100 | 101 | { 102 | name: "BoolSuccess", 103 | test: func(t *testing.T) { 104 | Set("foo", true) 105 | var b = Bool("foo") 106 | require.Equal(t, true, b) 107 | }, 108 | }, 109 | { 110 | name: "MustBoolSuccess", 111 | test: func(t *testing.T) { 112 | Set("foo", true) 113 | var b bool 114 | require.NotPanics(t, func() { b = MustBool("foo") }) 115 | require.Equal(t, true, b) 116 | }, 117 | }, 118 | { 119 | name: "MustBoolPanics", 120 | test: func(t *testing.T) { 121 | require.Panics(t, func() { MustBool("foo") }) 122 | }, 123 | }, 124 | 125 | { 126 | name: "DurationSuccess", 127 | test: func(t *testing.T) { 128 | Set("foo", "1m") 129 | var d = Duration("foo") 130 | require.Equal(t, 1*time.Minute, d) 131 | }, 132 | }, 133 | { 134 | name: "MustDurationSuccess", 135 | test: func(t *testing.T) { 136 | Set("foo", "1m") 137 | var d time.Duration 138 | require.NotPanics(t, func() { d = MustDuration("foo") }) 139 | require.Equal(t, 1*time.Minute, d) 140 | }, 141 | }, 142 | { 143 | name: "MustDurationPanics", 144 | test: func(t *testing.T) { 145 | require.Panics(t, func() { MustDuration("foo") }) 146 | }, 147 | }, 148 | { 149 | name: "TimeSuccess", 150 | test: func(t *testing.T) { 151 | Set("foo", "2019-01-02T15:04:05Z07:00") 152 | var d = Time("foo") 153 | 154 | var ti, _ = time.Parse(time.RFC3339, "2019-01-02T15:04:05Z07:00") 155 | 156 | require.Equal(t, ti, d) 157 | }, 158 | }, 159 | { 160 | name: "MustTimeSuccess", 161 | test: func(t *testing.T) { 162 | Set("foo", "2019-01-02T15:04:05Z07:00") 163 | var d time.Time 164 | 165 | var ti, _ = time.Parse(time.RFC3339, "2019-01-02T15:04:05Z07:00") 166 | 167 | require.NotPanics(t, func() { d = MustTime("foo") }) 168 | 169 | require.Equal(t, ti, d) 170 | }, 171 | }, 172 | { 173 | name: "MustTimePanics", 174 | test: func(t *testing.T) { 175 | require.Panics(t, func() { MustTime("foo") }) 176 | }, 177 | }, 178 | 179 | { 180 | name: "StringSliceSuccess", 181 | test: func(t *testing.T) { 182 | Set("foo", []string{"bar"}) 183 | var b = StringSlice("foo") 184 | require.Equal(t, []string{"bar"}, b) 185 | }, 186 | }, 187 | { 188 | name: "MustStringSliceSuccess", 189 | test: func(t *testing.T) { 190 | Set("foo", []string{"bar"}) 191 | var b []string 192 | require.NotPanics(t, func() { b = MustStringSlice("foo") }) 193 | require.Equal(t, []string{"bar"}, b) 194 | }, 195 | }, 196 | { 197 | name: "MustStringSlicePanics", 198 | test: func(t *testing.T) { 199 | require.Panics(t, func() { MustStringSlice("foo") }) 200 | }, 201 | }, 202 | { 203 | name: "IntSliceSuccess", 204 | test: func(t *testing.T) { 205 | Set("foo", []int{1}) 206 | var b = IntSlice("foo") 207 | require.Equal(t, []int{1}, b) 208 | }, 209 | }, 210 | { 211 | name: "MustIntSliceSuccess", 212 | test: func(t *testing.T) { 213 | Set("foo", []int{1}) 214 | var b []int 215 | require.NotPanics(t, func() { b = MustIntSlice("foo") }) 216 | require.Equal(t, []int{1}, b) 217 | }, 218 | }, 219 | { 220 | name: "MustIntSlicePanics", 221 | test: func(t *testing.T) { 222 | require.Panics(t, func() { MustIntSlice("foo") }) 223 | }, 224 | }, 225 | { 226 | name: "StringMapSuccess", 227 | test: func(t *testing.T) { 228 | Set("foo", map[string]interface{}{"foo": "bar"}) 229 | var b = StringMap("foo") 230 | require.Equal(t, map[string]interface{}{"foo": "bar"}, b) 231 | }, 232 | }, 233 | { 234 | name: "MustStringMapSuccess", 235 | test: func(t *testing.T) { 236 | Set("foo", map[string]interface{}{"foo": "bar"}) 237 | var b map[string]interface{} 238 | require.NotPanics(t, func() { b = MustStringMap("foo") }) 239 | require.Equal(t, map[string]interface{}{"foo": "bar"}, b) 240 | }, 241 | }, 242 | { 243 | name: "MustStringMapPanics", 244 | test: func(t *testing.T) { 245 | require.Panics(t, func() { MustStringMap("foo") }) 246 | }, 247 | }, 248 | { 249 | name: "StringMapStringSuccess", 250 | test: func(t *testing.T) { 251 | Set("foo", map[string]string{"foo": "bar"}) 252 | var b = StringMapString("foo") 253 | require.Equal(t, map[string]string{"foo": "bar"}, b) 254 | }, 255 | }, 256 | { 257 | name: "MustStringMapStringSuccess", 258 | test: func(t *testing.T) { 259 | Set("foo", map[string]string{"foo": "bar"}) 260 | var b map[string]string 261 | require.NotPanics(t, func() { b = MustStringMapString("foo") }) 262 | require.Equal(t, map[string]string{"foo": "bar"}, b) 263 | }, 264 | }, 265 | { 266 | name: "MustStringMapStringPanics", 267 | test: func(t *testing.T) { 268 | require.Panics(t, func() { MustStringMapString("foo") }) 269 | }, 270 | }, 271 | { 272 | name: "Exists", 273 | test: func(t *testing.T) { 274 | Set("test", true) 275 | require.True(t, Exists("test")) 276 | require.False(t, Exists("foo")) 277 | }, 278 | }, 279 | } 280 | for _, testCase := range testCases { 281 | 282 | t.Run(testCase.name, func(t *testing.T) { 283 | reset() 284 | testCase.test(t) 285 | }) 286 | } 287 | } 288 | -------------------------------------------------------------------------------- /values.go: -------------------------------------------------------------------------------- 1 | package konfig 2 | 3 | import ( 4 | "reflect" 5 | 6 | "github.com/pkg/errors" 7 | ) 8 | 9 | // Values is the values attached to a loader 10 | type Values map[string]interface{} 11 | 12 | // Set adds a key value to the Values 13 | func (x Values) Set(k string, v interface{}) { 14 | x[k] = v 15 | } 16 | 17 | func (x Values) load(ox Values, c *S) ([]string, error) { 18 | c.mut.Lock() 19 | defer c.mut.Unlock() 20 | 21 | // load the previous key store 22 | var m = c.m.Load().(s) 23 | 24 | // we copy the previous store 25 | // but we omit what was on the previous values 26 | var updatedKeys = make([]string, 0, len(x)) 27 | var nm = make(s) 28 | for kk, vv := range m { 29 | // if key is not in old loader values 30 | // add it to the new store 31 | // else if key is in new loader values but value is different or key is 32 | // in previous values but not in new ones 33 | // we add it to updatedKeys list 34 | if _, ok := ox[kk]; !ok { 35 | nm[kk] = vv 36 | } else if v, ok := x[kk]; (ok && !reflect.DeepEqual(v, vv)) || !ok { // value is different or not present anymore 37 | updatedKeys = append(updatedKeys, kk) 38 | } 39 | } 40 | // we add the new values 41 | for kk, vv := range x { 42 | nm[kk] = vv 43 | // if key is not present in old store, add it to updatedKeys as we are adding a new key in 44 | if _, ok := m[kk]; !ok { 45 | updatedKeys = append(updatedKeys, kk) 46 | } 47 | } 48 | 49 | // if we have strict keys setup on the store and we have already loaded configs 50 | // we check those keys now, if they are not present, we will return the error. 51 | if c.strictKeys != nil && c.loaded { 52 | if err := nm.checkStrictKeys(c.strictKeys); err != nil { 53 | err = errors.Wrap(err, "Error while checking strict keys") 54 | c.cfg.Logger.Get().Error(err.Error()) 55 | return nil, err 56 | } 57 | } 58 | 59 | // if there is a value bound we set it there also 60 | if c.v != nil { 61 | c.v.setValues(nm) 62 | } 63 | 64 | // we didn't get any error, store the new config state 65 | c.m.Store(nm) 66 | 67 | return updatedKeys, nil 68 | } 69 | -------------------------------------------------------------------------------- /watcher.go: -------------------------------------------------------------------------------- 1 | package konfig 2 | 3 | // Watcher is the interface implementing a config watcher. 4 | // Config watcher trigger loaders. A file watcher or a simple 5 | // Timer can be valid watchers. 6 | type Watcher interface { 7 | // Start starts the watcher, it must no be blocking. 8 | Start() error 9 | // Done indicate whether the watcher is done or not 10 | Done() <-chan struct{} 11 | // Watch should block until an event unlocks it 12 | Watch() <-chan struct{} 13 | // Close closes the watcher, it returns a non nil error if it is already closed 14 | // or something prevents it from closing properly. 15 | Close() error 16 | // Err returns the error attached to the watcher 17 | Err() error 18 | } 19 | 20 | // NopWatcher is a nil watcher 21 | type NopWatcher struct{} 22 | 23 | var _ Watcher = NopWatcher{} 24 | 25 | // Done returns an already closed channel 26 | func (NopWatcher) Done() <-chan struct{} { 27 | var c = make(chan struct{}) 28 | close(c) 29 | return c 30 | } 31 | 32 | // Watch implements a basic watch that waits forever 33 | func (NopWatcher) Watch() <-chan struct{} { 34 | var c = make(chan struct{}) 35 | return c 36 | } 37 | 38 | // Close is a noop that returns nil 39 | func (NopWatcher) Close() error { 40 | return nil 41 | } 42 | 43 | //Err returns nil, because nothing can go wrong when you do nothing 44 | func (NopWatcher) Err() error { 45 | return nil 46 | } 47 | 48 | // Start implements watcher interface and always returns a nil error 49 | func (NopWatcher) Start() error { 50 | return nil 51 | } 52 | 53 | // Watch starts the watchers on loaders 54 | func Watch() error { 55 | return instance().Watch() 56 | } 57 | 58 | // Watch starts the watchers on loaders 59 | func (c *S) Watch() error { 60 | 61 | // if metrics are enabled, we register them in prometheus 62 | if c.cfg.Metrics { 63 | if err := c.registerMetrics(); err != nil { 64 | return err 65 | } 66 | } 67 | 68 | for _, wl := range c.WatcherLoaders { 69 | if err := wl.Start(); err != nil { 70 | return err 71 | } 72 | go c.watchLoader(wl, 1) 73 | } 74 | return nil 75 | } 76 | -------------------------------------------------------------------------------- /watcher/kwfile/README.md: -------------------------------------------------------------------------------- 1 | # File Watcher 2 | File Watcher watches over a file given in the config. 3 | 4 | # Usage 5 | ``` 6 | import ( 7 | "fmt" 8 | "io/ioutil" 9 | "os" 10 | "time" 11 | 12 | "github.com/lalamove/konfig/watcher/kwfile" 13 | ) 14 | 15 | func main() { 16 | f, _ := ioutil.TempFile("", "konfig") 17 | f.Write([]byte(`ABC`)) 18 | 19 | defer os.Remove(f.Name()) 20 | 21 | var n = kwfile.New(&kwfile.Config{ 22 | Files: []string{f.Name()}, 23 | Rate: 100 * time.Millisecond, 24 | Debug: true, 25 | }) 26 | 27 | n.Start() 28 | 29 | time.Sleep(100 * time.Millisecond) 30 | 31 | f.Write([]byte(`12345`)) 32 | 33 | var timer = time.NewTimer(200 * time.Millisecond) 34 | select { 35 | case now := <-timer.C: 36 | fmt.Println(now) 37 | break 38 | case <-n.Done(): 39 | fmt.Println("done!") 40 | break 41 | case <-n.Watch(): 42 | fmt.Println("file changed!") // will see this log 43 | break 44 | } 45 | } 46 | ``` 47 | -------------------------------------------------------------------------------- /watcher/kwfile/filewatcher.go: -------------------------------------------------------------------------------- 1 | package kwfile 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "time" 7 | 8 | "github.com/lalamove/konfig" 9 | "github.com/lalamove/nui/nlogger" 10 | "github.com/radovskyb/watcher" 11 | ) 12 | 13 | var _ konfig.Watcher = (*FileWatcher)(nil) 14 | var defaultRate = 10 * time.Second 15 | 16 | // Config is the config of a FileWatcher 17 | type Config struct { 18 | // Files is the path to the files to watch 19 | Files []string 20 | // Rate is the rate at which the file is watched 21 | Rate time.Duration 22 | // Debug sets the debug mode on the filewatcher 23 | Debug bool 24 | // Logger is the logger used to print messages 25 | Logger nlogger.Provider 26 | } 27 | 28 | // FileWatcher watches over a file given in the config 29 | type FileWatcher struct { 30 | cfg *Config 31 | w *watcher.Watcher 32 | err error 33 | watchChan chan struct{} 34 | } 35 | 36 | // New creates a new FileWatcher from the given *Config cfg 37 | func New(cfg *Config) *FileWatcher { 38 | if cfg.Logger == nil { 39 | cfg.Logger = defaultLogger() 40 | } 41 | if cfg.Rate == 0 { 42 | cfg.Rate = defaultRate 43 | } 44 | 45 | var w = watcher.New() 46 | 47 | for _, file := range cfg.Files { 48 | cfg.Logger.Get().Info("adding file to watch: " + file) 49 | if err := w.Add(file); err != nil { 50 | panic(err) 51 | } 52 | } 53 | 54 | return &FileWatcher{ 55 | cfg: cfg, 56 | w: w, 57 | watchChan: make(chan struct{}), 58 | } 59 | } 60 | 61 | // Done indicates whether the filewatcher is done 62 | func (fw *FileWatcher) Done() <-chan struct{} { 63 | return fw.w.Closed 64 | } 65 | 66 | // Start starts the file watcher 67 | func (fw *FileWatcher) Start() error { 68 | go fw.watch() 69 | go func() error { 70 | if err := fw.w.Start(fw.cfg.Rate); err != nil { 71 | fw.cfg.Logger.Get().Error(err.Error()) 72 | return err 73 | } 74 | return nil 75 | }() 76 | return nil 77 | } 78 | 79 | // Watch return the channel to which events are written 80 | func (fw *FileWatcher) Watch() <-chan struct{} { 81 | return fw.watchChan 82 | } 83 | 84 | func (fw *FileWatcher) watch() { 85 | for { 86 | select { 87 | // we get an event, write to the struct chan 88 | // log if debug mode 89 | case e := <-fw.w.Event: 90 | if fw.cfg.Debug { 91 | fw.cfg.Logger.Get().Debug(fmt.Sprintf( 92 | "Event received %v", 93 | e, 94 | )) 95 | } 96 | fw.watchChan <- struct{}{} 97 | case err := <-fw.w.Error: 98 | // log error 99 | fw.cfg.Logger.Get().Error(err.Error()) 100 | fw.err = err 101 | fw.Close() 102 | return 103 | case <-fw.w.Closed: 104 | // watcher is closed, return 105 | return 106 | } 107 | } 108 | } 109 | 110 | // Close closes the FileWatcher 111 | func (fw *FileWatcher) Close() error { 112 | fw.w.Close() 113 | return nil 114 | } 115 | 116 | // Err returns the file watcher error 117 | func (fw *FileWatcher) Err() error { 118 | return fw.err 119 | } 120 | 121 | func defaultLogger() nlogger.Provider { 122 | return nlogger.NewProvider(nlogger.New(os.Stdout, "FILEWATCHER | ")) 123 | } 124 | -------------------------------------------------------------------------------- /watcher/kwfile/filewatcher_test.go: -------------------------------------------------------------------------------- 1 | package kwfile 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "testing" 7 | "time" 8 | 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestWatcher(t *testing.T) { 13 | t.Run( 14 | "new panics", 15 | func(t *testing.T) { 16 | require.Panics( 17 | t, 18 | func() { 19 | New(&Config{ 20 | Files: []string{"donotexist"}, 21 | Rate: 0, // using default rate 22 | }) 23 | }, 24 | ) 25 | }, 26 | ) 27 | 28 | t.Run( 29 | "new with watcher", 30 | func(t *testing.T) { 31 | f, err := ioutil.TempFile("", "konfig") 32 | f.Write([]byte(`ABC`)) 33 | require.Nil(t, err) 34 | 35 | defer os.Remove(f.Name()) 36 | 37 | var n = New(&Config{ 38 | Files: []string{f.Name()}, 39 | Rate: 100 * time.Millisecond, 40 | Debug: true, 41 | }) 42 | 43 | require.Nil(t, n.Start()) 44 | 45 | n.w.Wait() 46 | 47 | time.Sleep(100 * time.Millisecond) 48 | 49 | _, err = f.Write([]byte(`1233454`)) 50 | require.Nil(t, err) 51 | 52 | var timer = time.NewTimer(200 * time.Millisecond) 53 | var watched bool 54 | select { 55 | case <-timer.C: 56 | break 57 | case <-n.Done(): 58 | break 59 | case <-n.Watch(): 60 | watched = true 61 | break 62 | } 63 | 64 | require.True(t, watched) 65 | require.Nil(t, n.Err()) 66 | }, 67 | ) 68 | 69 | t.Run( 70 | "close", 71 | func(t *testing.T) { 72 | f, err := ioutil.TempFile("", "konfig") 73 | f.Write([]byte(`ABC`)) 74 | require.Nil(t, err) 75 | 76 | defer os.Remove(f.Name()) 77 | 78 | var n = New(&Config{ 79 | Files: []string{f.Name()}, 80 | Rate: 100 * time.Millisecond, 81 | Debug: true, 82 | }) 83 | 84 | require.Nil(t, n.Start()) 85 | n.w.Wait() 86 | time.Sleep(100 * time.Millisecond) 87 | 88 | n.Close() 89 | <-n.Done() 90 | }, 91 | ) 92 | 93 | t.Run( 94 | "start panics", 95 | func(t *testing.T) { 96 | f, err := ioutil.TempFile("", "konfig") 97 | f.Write([]byte(`ABC`)) 98 | require.Nil(t, err) 99 | 100 | defer os.Remove(f.Name()) 101 | 102 | var n = New(&Config{ 103 | Files: []string{f.Name()}, 104 | Rate: 100 * time.Millisecond, 105 | Debug: true, 106 | }) 107 | 108 | n.cfg.Rate = 0 // duration is less than 1ns 109 | 110 | n.Start() 111 | }, 112 | ) 113 | } 114 | -------------------------------------------------------------------------------- /watcher/kwpoll/README.md: -------------------------------------------------------------------------------- 1 | # Poll Watcher 2 | Poll Watcher is a konfig.Watcher that sends events every x time given in the konfig. 3 | 4 | # Usage 5 | ``` 6 | import ( 7 | "fmt" 8 | "os" 9 | "time" 10 | 11 | "github.com/lalamove/konfig" 12 | "github.com/lalamove/konfig/loader/klenv" 13 | "github.com/lalamove/konfig/watcher/kwpoll" 14 | ) 15 | 16 | func main() { 17 | os.Setenv("foo", "bar") 18 | 19 | var l = klenv.New(&klenv.Config{ 20 | Vars: []string{ 21 | "foo", 22 | }, 23 | }) 24 | 25 | var v = konfig.Values{} 26 | l.Load(v) 27 | 28 | var w = kwpoll.New(&kwpoll.Config{ 29 | Rater: kwpoll.Time(100 * time.Millisecond), 30 | Loader: l, 31 | Diff: true, 32 | Debug: true, 33 | InitValue: v, 34 | }) 35 | w.Start() 36 | 37 | var timer = time.NewTimer(200 * time.Millisecond) 38 | var watched int 39 | 40 | os.Setenv("foo", "baz") // change the value 41 | 42 | main: 43 | for { 44 | select { 45 | case <-timer.C: 46 | w.Close() 47 | break main 48 | case <-w.Watch(): 49 | watched++ 50 | } 51 | } 52 | 53 | fmt.Println(watched) // will output 1 54 | } 55 | ``` 56 | -------------------------------------------------------------------------------- /watcher/kwpoll/pollwatcher.go: -------------------------------------------------------------------------------- 1 | package kwpoll 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "time" 8 | 9 | "github.com/davecgh/go-spew/spew" 10 | "github.com/lalamove/konfig" 11 | "github.com/lalamove/nui/nlogger" 12 | ) 13 | 14 | var ( 15 | _ konfig.Watcher = (*PollWatcher)(nil) 16 | // ErrNoLoader is the error returned when no Loader is set and Diff is set to true 17 | ErrNoLoader = errors.New("You must give a non nil Loader to the poll diff watcher") 18 | // ErrAlreadyClosed is the error returned when trying to close an already closed PollDiffWatcher 19 | ErrAlreadyClosed = errors.New("PollDiffWatcher already closed") 20 | // ErrNoWatcherSupplied is the error returned when Watch in general config is false but a watcher is still being registered 21 | ErrNoWatcherSupplied = errors.New("watcher has to be supplied when registering a watcher") 22 | // defaultDuration is used in Rater if no Rater was supplied 23 | defaultDuration = time.Second * 5 24 | ) 25 | 26 | // Rater is an interface that exposes a single 27 | // Time method which returns the time until the next tick 28 | type Rater interface { 29 | Time() time.Duration 30 | } 31 | 32 | // Time is a time.Duration which implements the Rater interface 33 | type Time time.Duration 34 | 35 | // Time returns the time.Duration 36 | func (t Time) Time() time.Duration { 37 | return time.Duration(t) 38 | } 39 | 40 | // Config is the config of a PollWatcher 41 | type Config struct { 42 | // Rater is the rater the PollWatcher calls to get the duration until the next tick 43 | Rater Rater 44 | // Debug sets the debug mode 45 | Debug bool 46 | // Logger is the logger used to log debug messages 47 | Logger nlogger.Provider 48 | // Diff tells whether we should check for diffs 49 | // If diff is set, a Getter is required 50 | Diff bool 51 | // Loader is a loader to fetch data to check diff 52 | Loader konfig.Loader 53 | // InitValue is the initial value to compare with whe Diff is true 54 | InitValue konfig.Values 55 | } 56 | 57 | // PollWatcher is a konfig.Watcher that sends events every x time given in the konfig. 58 | type PollWatcher struct { 59 | cfg *Config 60 | err error 61 | pv konfig.Values 62 | watchChan chan struct{} 63 | done chan struct{} 64 | } 65 | 66 | // New creates a new PollWatcher fromt the given config 67 | func New(cfg *Config) *PollWatcher { 68 | if cfg.Diff && cfg.Loader == nil { 69 | panic(ErrNoLoader) 70 | } 71 | if cfg.Logger == nil { 72 | cfg.Logger = defaultLogger() 73 | } 74 | if cfg.Rater == nil { 75 | cfg.Rater = defaultRater() 76 | } 77 | 78 | return &PollWatcher{ 79 | cfg: cfg, 80 | pv: cfg.InitValue, 81 | done: make(chan struct{}), 82 | watchChan: make(chan struct{}), 83 | } 84 | } 85 | 86 | // Done indicates whether the watcher is done or not 87 | func (t *PollWatcher) Done() <-chan struct{} { 88 | return t.done 89 | } 90 | 91 | // Start starts the ticker watcher 92 | func (t *PollWatcher) Start() error { 93 | if t == nil { 94 | panic(ErrNoWatcherSupplied) 95 | } 96 | 97 | if t.cfg.Debug { 98 | t.cfg.Logger.Get().Debug( 99 | fmt.Sprintf( 100 | "Starting ticker watcher with rate: %dms", 101 | t.cfg.Rater.Time()/time.Millisecond, 102 | ), 103 | ) 104 | } 105 | go t.watch() 106 | return nil 107 | } 108 | 109 | // Watch returns the channel to which events are written 110 | func (t *PollWatcher) Watch() <-chan struct{} { 111 | return t.watchChan 112 | } 113 | 114 | // Err returns the poll watcher error 115 | func (t *PollWatcher) Err() error { 116 | return t.err 117 | } 118 | 119 | func (t *PollWatcher) watch() { 120 | var rate = t.cfg.Rater.Time() 121 | 122 | t.cfg.Logger.Get().Debug( 123 | fmt.Sprintf( 124 | "Waiting rater duration: %dms", 125 | rate/time.Millisecond, 126 | ), 127 | ) 128 | 129 | time.Sleep(rate) 130 | for { 131 | select { 132 | case <-t.done: 133 | default: 134 | if t.cfg.Debug { 135 | t.cfg.Logger.Get().Debug("Tick") 136 | } 137 | if t.cfg.Diff { 138 | t.cfg.Logger.Get().Debug( 139 | "Checking difference", 140 | ) 141 | 142 | var v = konfig.Values{} 143 | var err = t.cfg.Loader.Load(v) 144 | // We got error, we close 145 | if err != nil { 146 | t.cfg.Logger.Get().Error(err.Error()) 147 | t.err = err 148 | t.Close() 149 | return 150 | } 151 | if !t.valuesEqual(v) { 152 | if t.cfg.Debug { 153 | t.cfg.Logger.Get().Debug( 154 | "Value is different: " + spew.Sdump(t.pv, v) + "\n", 155 | ) 156 | } 157 | t.watchChan <- struct{}{} 158 | t.pv = v 159 | } else { 160 | t.cfg.Logger.Get().Debug( 161 | "Values are the same, not updating", 162 | ) 163 | } 164 | } else { 165 | t.cfg.Logger.Get().Debug( 166 | "Sending watch event", 167 | ) 168 | t.watchChan <- struct{}{} 169 | } 170 | time.Sleep(t.cfg.Rater.Time()) 171 | } 172 | } 173 | } 174 | 175 | // Close closes the PollWatcher 176 | func (t *PollWatcher) Close() error { 177 | select { 178 | case <-t.done: 179 | return ErrAlreadyClosed 180 | default: 181 | close(t.done) 182 | } 183 | return nil 184 | } 185 | 186 | func (t *PollWatcher) valuesEqual(v konfig.Values) bool { 187 | if len(v) != len(t.pv) { 188 | return false 189 | } 190 | 191 | for k, x := range v { 192 | if y, ok := t.pv[k]; ok { 193 | if y != x { 194 | return false 195 | } 196 | continue 197 | } 198 | return false 199 | } 200 | 201 | return true 202 | } 203 | 204 | func defaultLogger() nlogger.Provider { 205 | return nlogger.NewProvider(nlogger.New(os.Stdout, "POLLWATCHER | ")) 206 | } 207 | 208 | func defaultRater() Rater { 209 | return Time(defaultDuration) 210 | } 211 | -------------------------------------------------------------------------------- /watcher/kwpoll/pollwatcher_test.go: -------------------------------------------------------------------------------- 1 | package kwpoll 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | "time" 7 | 8 | gomock "github.com/golang/mock/gomock" 9 | "github.com/lalamove/konfig" 10 | "github.com/lalamove/konfig/mocks" 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | func TestWatcher(t *testing.T) { 15 | t.Run( 16 | "panics not loader", 17 | func(t *testing.T) { 18 | require.Panics(t, func() { 19 | New(&Config{ 20 | Diff: true, 21 | }) 22 | }) 23 | }, 24 | ) 25 | t.Run( 26 | "panics nil watcher", 27 | func(t *testing.T) { 28 | require.Panics(t, func() { 29 | var w *PollWatcher 30 | w.Start() 31 | }) 32 | }, 33 | ) 34 | t.Run( 35 | "close error", 36 | func(t *testing.T) { 37 | var w = New(&Config{}) 38 | require.Nil(t, w.Close()) 39 | require.Equal(t, ErrAlreadyClosed, w.Close()) 40 | require.NotNil(t, w.Close()) 41 | }, 42 | ) 43 | t.Run( 44 | "basic watcher, no diff", 45 | func(t *testing.T) { 46 | var w = New(&Config{ 47 | Rater: Time(100 * time.Millisecond), 48 | Debug: true, 49 | }) 50 | w.Start() 51 | time.Sleep(200 * time.Millisecond) 52 | var timer = time.NewTimer(100 * time.Millisecond) 53 | select { 54 | case <-timer.C: 55 | t.Error("watcher should have ticked") 56 | w.Close() 57 | case <-w.Watch(): 58 | w.Close() 59 | return 60 | } 61 | }, 62 | ) 63 | t.Run( 64 | "watcher diff", 65 | func(t *testing.T) { 66 | var ctrl = gomock.NewController(t) 67 | defer ctrl.Finish() 68 | 69 | var g = mocks.NewMockLoader(ctrl) 70 | 71 | var v = konfig.Values{"foo": "bar"} 72 | 73 | gomock.InOrder( 74 | g.EXPECT().Load(konfig.Values{}).Times(1).Do(func(v konfig.Values) { 75 | v.Set("foo", "bar") 76 | }).Return(nil), 77 | g.EXPECT().Load(konfig.Values{}).Do(func(v konfig.Values) { 78 | v.Set("foo", "bar") 79 | }).Return(nil), 80 | g.EXPECT().Load(konfig.Values{}).Do(func(v konfig.Values) { 81 | v.Set("foo", "bar") 82 | }).Return(nil), 83 | g.EXPECT().Load(konfig.Values{}).Do(func(v konfig.Values) { 84 | v.Set("foo", "barr") 85 | }).Return(nil), 86 | g.EXPECT().Load(konfig.Values{}).Do(func(v konfig.Values) { 87 | v.Set("test", "barr") 88 | }).Return(nil), 89 | g.EXPECT().Load(konfig.Values{}).Do(func(v konfig.Values) { 90 | }).Return(nil), 91 | ) 92 | 93 | var w = New(&Config{ 94 | Rater: Time(100 * time.Millisecond), 95 | Loader: g, 96 | Diff: true, 97 | Debug: true, 98 | InitValue: v, 99 | }) 100 | w.Start() 101 | 102 | var timer = time.NewTimer(620 * time.Millisecond) 103 | var watched int 104 | main: 105 | for { 106 | select { 107 | case <-timer.C: 108 | w.Close() 109 | break main 110 | case <-w.Watch(): 111 | watched++ 112 | } 113 | } 114 | 115 | require.Equal(t, 3, watched) 116 | }, 117 | ) 118 | 119 | t.Run( 120 | "watcher diff err", 121 | func(t *testing.T) { 122 | var ctrl = gomock.NewController(t) 123 | defer ctrl.Finish() 124 | 125 | var g = mocks.NewMockLoader(ctrl) 126 | 127 | var v = konfig.Values{"foo": "bar"} 128 | 129 | gomock.InOrder( 130 | g.EXPECT().Load(konfig.Values{}).Times(1).Do(func(v konfig.Values) { 131 | v.Set("foo", "bar") 132 | }).Return(nil), 133 | g.EXPECT().Load(konfig.Values{}).Do(func(v konfig.Values) { 134 | v.Set("foo", "bar") 135 | }).Return(errors.New("Err")), 136 | ) 137 | 138 | var w = New(&Config{ 139 | Rater: Time(100 * time.Millisecond), 140 | Loader: g, 141 | Diff: true, 142 | Debug: true, 143 | InitValue: v, 144 | }) 145 | w.Start() 146 | 147 | time.Sleep(400 * time.Millisecond) 148 | var timer = time.NewTimer(400 * time.Millisecond) 149 | var done bool 150 | var err error 151 | select { 152 | case <-timer.C: 153 | t.Error("watcher should have ticked") 154 | w.Close() 155 | case <-w.Watch(): 156 | w.Close() 157 | return 158 | case <-w.Done(): 159 | done = true 160 | err = w.Err() 161 | break 162 | } 163 | require.True(t, done) 164 | require.NotNil(t, err) 165 | }, 166 | ) 167 | } 168 | -------------------------------------------------------------------------------- /watcher_mock_test.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: ./watcher.go 3 | 4 | // Package konfig is a generated GoMock package. 5 | package konfig 6 | 7 | import ( 8 | gomock "github.com/golang/mock/gomock" 9 | reflect "reflect" 10 | ) 11 | 12 | // MockWatcher is a mock of Watcher interface 13 | type MockWatcher struct { 14 | ctrl *gomock.Controller 15 | recorder *MockWatcherMockRecorder 16 | } 17 | 18 | // MockWatcherMockRecorder is the mock recorder for MockWatcher 19 | type MockWatcherMockRecorder struct { 20 | mock *MockWatcher 21 | } 22 | 23 | // NewMockWatcher creates a new mock instance 24 | func NewMockWatcher(ctrl *gomock.Controller) *MockWatcher { 25 | mock := &MockWatcher{ctrl: ctrl} 26 | mock.recorder = &MockWatcherMockRecorder{mock} 27 | return mock 28 | } 29 | 30 | // EXPECT returns an object that allows the caller to indicate expected use 31 | func (m *MockWatcher) EXPECT() *MockWatcherMockRecorder { 32 | return m.recorder 33 | } 34 | 35 | // Start mocks base method 36 | func (m *MockWatcher) Start() error { 37 | m.ctrl.T.Helper() 38 | ret := m.ctrl.Call(m, "Start") 39 | ret0, _ := ret[0].(error) 40 | return ret0 41 | } 42 | 43 | // Start indicates an expected call of Start 44 | func (mr *MockWatcherMockRecorder) Start() *gomock.Call { 45 | mr.mock.ctrl.T.Helper() 46 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockWatcher)(nil).Start)) 47 | } 48 | 49 | // Done mocks base method 50 | func (m *MockWatcher) Done() <-chan struct{} { 51 | m.ctrl.T.Helper() 52 | ret := m.ctrl.Call(m, "Done") 53 | ret0, _ := ret[0].(<-chan struct{}) 54 | return ret0 55 | } 56 | 57 | // Done indicates an expected call of Done 58 | func (mr *MockWatcherMockRecorder) Done() *gomock.Call { 59 | mr.mock.ctrl.T.Helper() 60 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Done", reflect.TypeOf((*MockWatcher)(nil).Done)) 61 | } 62 | 63 | // Watch mocks base method 64 | func (m *MockWatcher) Watch() <-chan struct{} { 65 | m.ctrl.T.Helper() 66 | ret := m.ctrl.Call(m, "Watch") 67 | ret0, _ := ret[0].(<-chan struct{}) 68 | return ret0 69 | } 70 | 71 | // Watch indicates an expected call of Watch 72 | func (mr *MockWatcherMockRecorder) Watch() *gomock.Call { 73 | mr.mock.ctrl.T.Helper() 74 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Watch", reflect.TypeOf((*MockWatcher)(nil).Watch)) 75 | } 76 | 77 | // Close mocks base method 78 | func (m *MockWatcher) Close() error { 79 | m.ctrl.T.Helper() 80 | ret := m.ctrl.Call(m, "Close") 81 | ret0, _ := ret[0].(error) 82 | return ret0 83 | } 84 | 85 | // Close indicates an expected call of Close 86 | func (mr *MockWatcherMockRecorder) Close() *gomock.Call { 87 | mr.mock.ctrl.T.Helper() 88 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockWatcher)(nil).Close)) 89 | } 90 | 91 | // Err mocks base method 92 | func (m *MockWatcher) Err() error { 93 | m.ctrl.T.Helper() 94 | ret := m.ctrl.Call(m, "Err") 95 | ret0, _ := ret[0].(error) 96 | return ret0 97 | } 98 | 99 | // Err indicates an expected call of Err 100 | func (mr *MockWatcherMockRecorder) Err() *gomock.Call { 101 | mr.mock.ctrl.T.Helper() 102 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Err", reflect.TypeOf((*MockWatcher)(nil).Err)) 103 | } 104 | -------------------------------------------------------------------------------- /watcher_test.go: -------------------------------------------------------------------------------- 1 | package konfig 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | gomock "github.com/golang/mock/gomock" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestWatcher(t *testing.T) { 12 | t.Run("Start returns nil", 13 | func(t *testing.T) { 14 | var w = NopWatcher{} 15 | require.Nil(t, w.Start()) 16 | 17 | var ctrl = gomock.NewController(t) 18 | defer ctrl.Finish() 19 | 20 | var mockL = NewMockLoader(ctrl) 21 | 22 | var c = New(&Config{}) 23 | 24 | c.RegisterLoaderWatcher( 25 | NewLoaderWatcher(mockL, w), 26 | ) 27 | 28 | c.Watch() 29 | require.Nil(t, w.Err()) 30 | time.Sleep(100 * time.Millisecond) 31 | }, 32 | ) 33 | } 34 | --------------------------------------------------------------------------------