├── .gitignore ├── .travis.yml ├── .travis └── install_etcd.sh ├── LICENSE ├── Makefile ├── README.md ├── ciphers └── secconf │ ├── README.md │ ├── keyring_test.go │ ├── sec_interface.go │ ├── sec_interface_test.go │ ├── secconf.go │ └── secconf_test.go ├── cli └── onioncli │ ├── README.md │ └── cli.go ├── configwatch ├── types.go ├── watch_test.go └── wtach.go ├── doc.go ├── env_layer.go ├── env_layer_test.go ├── go.mod ├── go.sum ├── helper.go ├── layers ├── directorylayer │ ├── directory_layer.go │ └── directory_layer_test.go ├── directorywatchlayer │ ├── directory_watch_layer.go │ └── directory_watch_layer_test.go ├── etcdlayer │ ├── etcd_layer.go │ └── etcd_layer_test.go ├── filewatchlayer │ ├── file_layer.go │ └── file_layer_test.go └── structlayer │ ├── struct_layer.go │ └── struct_layer_test.go ├── loaders ├── properties │ ├── properties.go │ └── properties_test.go ├── toml │ ├── toml-0.5.0_test.go │ ├── toml.go │ └── toml_test.go └── yaml │ ├── yaml.go │ └── yaml_test.go ├── map_layer.go ├── onion.go ├── onion_test.go ├── onionwriter ├── onion_writer.go └── onion_writer_test.go ├── stream_layer.go ├── stream_layer_test.go ├── utils.go └── utils_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | *.out 26 | .idea 27 | *.cp -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - 1.15 4 | - 1.16 5 | - 1.17 6 | - tip 7 | before_install: 8 | - go get -v github.com/smartystreets/goconvey 9 | - go get -v github.com/axw/gocov/gocov 10 | - go get -v github.com/mattn/goveralls 11 | - if ! go get code.google.com/p/go.tools/cmd/cover; then go get golang.org/x/tools/cmd/cover; fi 12 | - bash .travis/install_etcd.sh & 13 | - sleep 3 14 | script: 15 | - make update 16 | - make lint 17 | - goveralls -v -service travis-ci -repotoken $COVERALLS_TOKEN || make test 18 | 19 | -------------------------------------------------------------------------------- /.travis/install_etcd.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o errexit 3 | set -o nounset 4 | set -o pipefail 5 | 6 | ROOT=$(dirname "${BASH_SOURCE}")/../.. 7 | 8 | ETCD_VERSION=${ETCD_VERSION:-v3.5.0} 9 | 10 | mkdir -p "${ROOT}/third_party" 11 | cd "${ROOT}/third_party" 12 | curl -sL https://github.com/etcd-io/etcd/releases/download/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz \ 13 | | tar xzf - 14 | 15 | exec etcd-${ETCD_VERSION}-linux-amd64/etcd 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015, 2016, 2017, 2018, 2019 Forud Ghafouri (goraz) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | export ROOT:=$(realpath $(dir $(firstword $(MAKEFILE_LIST)))) 2 | test: 3 | go test -v -race ./... 4 | 5 | update: 6 | go get -u -t ./... 7 | 8 | tidy: 9 | go mod tidy 10 | 11 | bin/golangci-lint: 12 | mkdir -p $(ROOT)/bin 13 | echo "*"> $(ROOT)/bin/.gitignore 14 | curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(ROOT)/bin v1.27.0 15 | 16 | lint: bin/golangci-lint 17 | bin/golangci-lint run 18 | 19 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # onion 2 | 3 | [![Build Status](https://travis-ci.org/goraz/onion.svg)](https://travis-ci.org/goraz/onion) 4 | [![Coverage Status](https://coveralls.io/repos/goraz/onion/badge.svg?branch=develop&service=github)](https://coveralls.io/github/goraz/onion?branch=master) 5 | [![GoDoc](https://godoc.org/github.com/goraz/onion?status.svg)](https://godoc.org/github.com/goraz/onion) 6 | [![Go Report Card](https://goreportcard.com/badge/github.com/goraz/onion)](https://goreportcard.com/report/github.com/goraz/onion) 7 | 8 | import "github.com/goraz/onion" 9 | 10 | Package onion is a layer based, pluggable config manager for golang. 11 | 12 | The current version in `develop` branch is work in progress (see the [milestone](https://github.com/goraz/onion/milestone/1)), for older versions check the `v2` and `v3` branches and use the `gopkg.in/goraz/onion.v1` and `gopkg.in/goraz/onion.v2` 13 | For the next release we use the go module and tagging using semantic version. 14 | 15 | ``` 16 | Shrek: For your information, there's a lot more to ogres than people think. 17 | Donkey: Example? 18 | Shrek: Example... uh... ogres are like onions! 19 | [holds up an onion, which Donkey sniffs] 20 | Donkey: They stink? 21 | Shrek: Yes... No! 22 | Donkey: Oh, they make you cry? 23 | Shrek: No! 24 | Donkey: Oh, you leave 'em out in the sun, they get all brown, start sproutin' little white hairs... 25 | Shrek: [peels an onion] NO! Layers. Onions have layers. Ogres have layers... You get it? We both have layers. 26 | [walks off] 27 | Donkey: Oh, you both have LAYERS. Oh. You know, not everybody like onions. CAKE! Everybody loves cake! Cakes have layers! 28 | Shrek: I don't care what everyone likes! Ogres are not like cakes. 29 | Donkey: You know what ELSE everybody likes? Parfaits! Have you ever met a person, you say, "Let's get some parfait," they say, "Hell no, I don't like no parfait."? Parfaits are delicious! 30 | Shrek: NO! You dense, irritating, miniature beast of burden! Ogres are like onions! End of story! Bye-bye! See ya later. 31 | Donkey: Parfait's gotta be the most delicious thing on the whole damn planet! 32 | ``` 33 | ## Goals 34 | 35 | The main goal is to have minimal dependency based on usage. if you need normal config files in the file system, 36 | there should be no dependency to `etcd` or `consul`, if you have only `yaml` files, including `toml` or any other format 37 | is just not right. 38 | 39 | ## Usage 40 | 41 | Choose the layer first. normal file layer and json are built-in but for any other type 42 | you need to import the package for that layer. 43 | 44 | ### Example json file layer 45 | 46 | ```go 47 | package main 48 | 49 | import ( 50 | "fmt" 51 | 52 | "github.com/goraz/onion" 53 | ) 54 | 55 | func main() { 56 | // Create a file layer to load data from json file. onion loads the file based on the extension. 57 | // so the json file should have `.json` ext. 58 | l1, err := onion.NewFileLayer("/etc/shared.json", nil) 59 | if err != nil { 60 | panic(err) 61 | } 62 | 63 | // Create a layer based on the environment. it loads every environment with APP_ prefix 64 | // for example APP_TEST_STRING is available as o.Get("test.string") 65 | l2 := onion.NewEnvLayerPrefix("_", "APP") 66 | 67 | // Create the onion, the final result is union of l1 and l2 but l2 overwrite l1. 68 | o := onion.New(l1, l2) 69 | str := o.GetStringDefault("test.string", "empty") 70 | fmt.Println(str) 71 | // Now str is the string in this order 72 | // 1- if the APP_TEST_STRING is available in the env 73 | // 2- if the shared.json had key like this { "test" : { "string" : "value" }} then the str is "value" 74 | // 3- the provided default, "empty" 75 | } 76 | ``` 77 | 78 | ### Loading other file format 79 | 80 | Currently `onion` support `json` format out-of-the-box, while you need to blank import the loader package of others formats to use them: 81 | * `toml` (for 0.4.0 version) 82 | * `toml-0.5.0` (for 0.5.0 version) 83 | * `yaml` 84 | * `properties` 85 | 86 | For example: 87 | ```go 88 | import ( 89 | _ "github.com/goraz/onion/loaders/toml" // Needed to load TOML format 90 | ) 91 | ``` 92 | 93 | ### Watch file and etcd 94 | 95 | Also there is other layers, (like `etcd` and `filewatchlayer`) that watches for change. 96 | 97 | ```go 98 | package main 99 | 100 | import ( 101 | "fmt" 102 | 103 | "github.com/goraz/onion" 104 | "github.com/goraz/onion/layers/etcdlayer" 105 | "github.com/goraz/onion/layers/filewatchlayer" 106 | ) 107 | 108 | func main() { 109 | // Create a file layer to load data from json file. also it watches for change in the file 110 | l1, err := filewatchlayer.NewFileWatchLayer("/etc/shared.json", nil) 111 | if err != nil { 112 | panic(err) 113 | } 114 | 115 | l2, err := etcdlayer.NewEtcdLayer("/app/config", "json", []string{"http://127.0.0.1:2379"}, nil) 116 | if err != nil { 117 | panic(err) 118 | } 119 | 120 | // Create the onion, the final result is union of l1 and l2 but l2 overwrite l1. 121 | o := onion.New(l1, l2) 122 | // Get the latest version of the key 123 | str := o.GetStringDefault("test.string", "empty") 124 | fmt.Println(str) 125 | } 126 | ``` 127 | 128 | ### Encrypted config 129 | 130 | Also if you want to store data in encrypted content. currently only `secconf` (based on the [crypt](https://github.com/xordataexchange/crypt) project) is supported. 131 | also the [onioncli](https://github.com/goraz/onion/tree/develop/cli/onioncli) helps you to manage this keys. 132 | 133 | ```go 134 | package main 135 | 136 | import ( 137 | "bytes" 138 | "fmt" 139 | 140 | "github.com/goraz/onion" 141 | "github.com/goraz/onion/ciphers/secconf" 142 | "github.com/goraz/onion/layers/etcdlayer" 143 | "github.com/goraz/onion/layers/filewatchlayer" 144 | ) 145 | 146 | // Normally this should be in a safe place, not here 147 | const privateKey = `PRIVATE KEY` 148 | 149 | func main() { 150 | // The private key should be in the safe place. this is just a demo, also there is a cli tool 151 | // to create this `go get -u github.com/goraz/onion/cli/onioncli` 152 | cipher, err := secconf.NewCipher(bytes.NewReader([]byte(privateKey))) 153 | if err != nil { 154 | panic(err) 155 | } 156 | 157 | // Create a file layer to load data from json file. also it watches for change in the file 158 | // passing the cipher to this make means the file in base64 and pgp encrypted 159 | l1, err := filewatchlayer.NewFileWatchLayer("/etc/shared.json", cipher) 160 | if err != nil { 161 | panic(err) 162 | } 163 | 164 | // Create a etcd layer. it watches the /app/config key and it should be json file encoded with 165 | // base64 and pgp 166 | l2, err := etcdlayer.NewEtcdLayer("/app/config", "json", []string{"http://127.0.0.1:2379"}, cipher) 167 | if err != nil { 168 | panic(err) 169 | } 170 | 171 | // Create the onion, the final result is union of l1 and l2 but l2 overwrites l1. 172 | o := onion.New(l1, l2) 173 | // Get the latest version of the key 174 | str := o.GetStringDefault("test.string", "empty") 175 | fmt.Println(str) 176 | } 177 | ``` 178 | -------------------------------------------------------------------------------- /ciphers/secconf/README.md: -------------------------------------------------------------------------------- 1 | # Acknowledgement 2 | 3 | This package is from [crypt](https://github.com/xordataexchange/crypt) 4 | 5 | [MIT license](https://github.com/xordataexchange/crypt/blob/master/LICENSE) 6 | -------------------------------------------------------------------------------- /ciphers/secconf/keyring_test.go: -------------------------------------------------------------------------------- 1 | package secconf 2 | 3 | var pubring = `-----BEGIN PGP PUBLIC KEY BLOCK----- 4 | Version: GnuPG/MacGPG2 v2.0.22 (Darwin) 5 | Comment: GPGTools - https://gpgtools.org 6 | 7 | mQENBFRClxIBCACjlm8e2mI5TmeigPuF4HJqNxc6AFLoCsE3MQ6VtdEVqidXZ06L 8 | m7uIXYc3IfvAlID+1KrUJnO2IgLEMmXKSDI5aOch7VaeoXLKMt7yAX+N6DHaZl4l 9 | eUGlfyIuBGD7FY2rv4hHo2wOmlq/chnNA4T7wb2XzeaAjvvoxcedMZ2npVimjwsl 10 | MNDxSxYPlHR6lJgfYJHAxcWn7ZQJW2Kllv9jMQwzGqW4fxuKRhe20KStE/4+K9gL 11 | GWv6OoE2gcGLoXliIMchHobY0GEvVx+YUv5jAItRSXq4ajYjFLtsWLz6FYtK9CoO 12 | Va6T5EGqozKST/olW/FMmKLOTzpAilyoKB/HABEBAAG0LWFwcCAoYXBwIGNvbmZp 13 | Z3VyYXRpb24ga2V5KSA8YXBwQGV4YW1wbGUuY29tPokBNwQTAQoAIQUCVEKXEgIb 14 | AwULCQgHAwUVCgkICwUWAgMBAAIeAQIXgAAKCRA8TymBhIANsjB1CACi4kqqWNSq 15 | AID7LmMswh5FQDEPkI/WA0h75xead11FVSdvtjWANY4Wob8RBjeZNT0TaCa0IAoo 16 | k+tLqA5xNbbvalOPV2zfr86BcGMhIs900++PuVjOb7XaJPsEt5JwtzuLM+eDLIVh 17 | vMI7hQtgB39O8/AsWEW/E/JlVtHcrsQ7LfcQYmNZVSnL71a8w4G+A6Sto89fvpjY 18 | h9/M4+aHqMhO/NLLp8Ylj5TlyiWKHZlx5ufl2ejWMUot3wFhYADHPkhydmQV9IY1 19 | zzIpmB/75kvZqC4p92k7l8Ra82o+T75/dNy0HcgvgrfZQttxIM0WPEyVF5NjicSo 20 | akoggAAslhCNuQENBFRClxIBCADJltx4EgkFScH/EAmO6+mZb6+pcpjY/H97bX4w 21 | KUrQSDZjDAhoxsInKgqHwAo3QY261eYrAyHvoTA2kRAaVrYWeGu3RxMmX5LTjFsX 22 | IW44ocTJK1XziUQympgIEayOUHt+XJaMGL8RKXvNgttGkr2VPD0IWJCOaBr8ZxUG 23 | Fm/pRFeBe6tX02RVKx4QFPqCnb76bkvR1cNeFsV5eEz0WNRYzena+lD6Oqh074tk 24 | oC9Uwl7D0l7xq17HNqAqHdMIO/T/TMPYyb7vskHPL9g8EJSgU55Z2Cjx3IlbJCpA 25 | 204cbbak4h99kgAqb4O5jT3gDe03NzWXCeQVKmacahusqNxzABEBAAGJAR8EGAEK 26 | AAkFAlRClxICGwwACgkQPE8pgYSADbJFTwf/d6XIv8/BxaP5TB12FxRXevOVenYe 27 | xR6O0rXDKFkP54LHSh2uaJbjLHevF0tuiNLFWq9RPhVjC012OLdo+2ygEFwNfikd 28 | 1SMbUIkuZ6Nu2MvCdrpAwbcvLgeoe8bqf1B6EIb31/OxCmtHujpVw6cSAnpAVyYo 29 | PjPtEpcNatIHbOna4KANxwv1Rmc6XDxWIB0RIlbdZDUhEdLovLLWGjm4J++Cnn2n 30 | OFdZyyUxwYTjDCMWwsYrG2oPZ0Yle6fKEXX30E+dN9NSV1i+dJAYQi0am6augpg+ 31 | LmFWxQ6JPmUJVDay9wo6g2D4KbJQybSh8lmqpenHnKD1m/gCGadPmMl6Rw== 32 | =FKbO 33 | -----END PGP PUBLIC KEY BLOCK-----` 34 | 35 | var secring = ` 36 | -----BEGIN PGP PRIVATE KEY BLOCK----- 37 | Version: GnuPG/MacGPG2 v2.0.22 (Darwin) 38 | Comment: GPGTools - https://gpgtools.org 39 | 40 | lQOYBFRClxIBCACjlm8e2mI5TmeigPuF4HJqNxc6AFLoCsE3MQ6VtdEVqidXZ06L 41 | m7uIXYc3IfvAlID+1KrUJnO2IgLEMmXKSDI5aOch7VaeoXLKMt7yAX+N6DHaZl4l 42 | eUGlfyIuBGD7FY2rv4hHo2wOmlq/chnNA4T7wb2XzeaAjvvoxcedMZ2npVimjwsl 43 | MNDxSxYPlHR6lJgfYJHAxcWn7ZQJW2Kllv9jMQwzGqW4fxuKRhe20KStE/4+K9gL 44 | GWv6OoE2gcGLoXliIMchHobY0GEvVx+YUv5jAItRSXq4ajYjFLtsWLz6FYtK9CoO 45 | Va6T5EGqozKST/olW/FMmKLOTzpAilyoKB/HABEBAAEAB/wKvEBo68JJaiH2nJ9P 46 | qas92YVZMD9Al2rBoU2zOR4nsqW9SybGQP89aOHgQNyBcV1xG79lh8Eii+MsQUsZ 47 | IMQcV2GKV5sjyDWScQ8yHjNi5SuBs85sMs5s5XB2nkvyU6JF9J5QETicprgw2x84 48 | AIn1buvvGTs4vD6/h7Hcri5fRimBvh+dcH/48nXPH56cZEPl/53tJt/lWwlfFBX1 49 | phZMIPoHT1kihEt//Hn4raw30R/bm0CJP/PtiyRkNeebzJsIJXtzG30B8YZb6c/h 50 | TtobA4F4ZWtEEwotPBFtx4clS/+2amc+PY+ZGTKXjzvQChaz50gvtSUp9ns9X/G+ 51 | T/vRBADC3dNGE2Ut8DRE2C/MQ7DdZdHdxaHJSMV+08xI/OSDOxp3ea1S2cbjniIG 52 | cnuQ8ZXD4hWDKSZTGs2L4awdsL5eIhqACnxT3LXm0TBwBWDzE3CQZUQGc+2pFgDb 53 | 1Xc/By+OZgFCDlJhHuhK4Lf9EsH3HbV/Cmn8sDD+dKazLxUF1wQA1uiH8X/8dgcQ 54 | uH/RSH2C7+Sr2B2Tpha9kngg4/cB31v3YaBV2t55zBvhSObxCM97gl6FadrEjJsw 55 | FvN04DMWhlt2xWbLnt1v4suVo8V1Are4vqP8G/mWhJou2Ps/65nsFqStNHMA+xjQ 56 | h8hAqY/9Mmu9Vm6WNRON0WCT3Snil5ED/0zUGI2qogw35Uzu448FrrYlh97kj3wu 57 | RzOZB/mty2pVj9eJO0z6E3C6sYLvbxrd8TyFzs4fTP7WlwG5FMJu/I4cEBqUJ/rr 58 | +ulSV/HH7zLpD6hWZbuRYhY8uskkVH50be4bb7MrXtoeDKrKfM4+BKf39QaBDNfI 59 | jD0Perf+Ll0aRBm0LWFwcCAoYXBwIGNvbmZpZ3VyYXRpb24ga2V5KSA8YXBwQGV4 60 | YW1wbGUuY29tPokBNwQTAQoAIQUCVEKXEgIbAwULCQgHAwUVCgkICwUWAgMBAAIe 61 | AQIXgAAKCRA8TymBhIANsjB1CACi4kqqWNSqAID7LmMswh5FQDEPkI/WA0h75xea 62 | d11FVSdvtjWANY4Wob8RBjeZNT0TaCa0IAook+tLqA5xNbbvalOPV2zfr86BcGMh 63 | Is900++PuVjOb7XaJPsEt5JwtzuLM+eDLIVhvMI7hQtgB39O8/AsWEW/E/JlVtHc 64 | rsQ7LfcQYmNZVSnL71a8w4G+A6Sto89fvpjYh9/M4+aHqMhO/NLLp8Ylj5TlyiWK 65 | HZlx5ufl2ejWMUot3wFhYADHPkhydmQV9IY1zzIpmB/75kvZqC4p92k7l8Ra82o+ 66 | T75/dNy0HcgvgrfZQttxIM0WPEyVF5NjicSoakoggAAslhCNnQOYBFRClxIBCADJ 67 | ltx4EgkFScH/EAmO6+mZb6+pcpjY/H97bX4wKUrQSDZjDAhoxsInKgqHwAo3QY26 68 | 1eYrAyHvoTA2kRAaVrYWeGu3RxMmX5LTjFsXIW44ocTJK1XziUQympgIEayOUHt+ 69 | XJaMGL8RKXvNgttGkr2VPD0IWJCOaBr8ZxUGFm/pRFeBe6tX02RVKx4QFPqCnb76 70 | bkvR1cNeFsV5eEz0WNRYzena+lD6Oqh074tkoC9Uwl7D0l7xq17HNqAqHdMIO/T/ 71 | TMPYyb7vskHPL9g8EJSgU55Z2Cjx3IlbJCpA204cbbak4h99kgAqb4O5jT3gDe03 72 | NzWXCeQVKmacahusqNxzABEBAAEAB/47pozhaLDLpEonz9aMOImckfxgPx00Y+7T 73 | FpC27pkJLb0OLPLWEi5ESX/pMG21cQvfw8iCZMBneIJcOyuRJ6Rk3Mg+6OSlP7Wi 74 | LI+NtiI31sJ0poKd+Dm6YZ1oEdbGG9GXEA2qMe5jxSsxoi2BYg2AOd1zeUV5JhwK 75 | IPSLIxuFYeDV/erv0n73Lob/Xj7SzhwRNQUJuG9Ak+maha1oqHwTuzPox9e+kSkK 76 | +VOhW+9oTukxsg8lCD351X/VvHeJgZkfTshLbQdAbMUlBQ00O7TyprFFLKcd0MNL 77 | gdVz5vHson5NyEzxsCbnV0Hty5Am00r1hm3Y89/k9HmBr3f+IH6JBADK0ZN9m4Br 78 | xpc2fou40/HBKBPk/5sJoOcHklBM7j4COYqloYaYliZRKmeWfH3gPhYW+EOqsZtv 79 | BPZaS7RL0IU8GoC1GfIrHJ+4GwiZQm6URDvEVSWsWiaUkI+cnK1HX8zsWHq48tqF 80 | yVSOZ05Lh3Id65s3mnXzF3/zzQLMmKm1OwQA/nLDZSMRdr/WWW2nFpf5QH0y9eI3 81 | VU/4/QSIBLFL5iAXOebHDseCr7/G/W6hn00VTQIUq3UKDi+gy9epm9aBrdNyF3Ey 82 | PvuACFLduF4ZnPOeZ1YrBxCRPHnGf+3So2Kcl9c1+RzMJ/qY+lZCU6pMCgCkeAZP 83 | iTGeuExKr9OrIikD/Au6yH+Oc2GEvorhoWcerEeXFvvx1S+9oJBKnJl9y6PRJacy 84 | wkZ354RyD9AojMJliibaHdAdpGSrOL8NEYQGy/3YzW1sMS2GBw6yZJ/GPCRDVEaE 85 | Nkbi/Aj3Shh2+w/jeYsUgrJkZY/UeoJt/mdUO1+loRoqTdlOOJLpPcyF6WzQQU+J 86 | AR8EGAEKAAkFAlRClxICGwwACgkQPE8pgYSADbJFTwf/d6XIv8/BxaP5TB12FxRX 87 | evOVenYexR6O0rXDKFkP54LHSh2uaJbjLHevF0tuiNLFWq9RPhVjC012OLdo+2yg 88 | EFwNfikd1SMbUIkuZ6Nu2MvCdrpAwbcvLgeoe8bqf1B6EIb31/OxCmtHujpVw6cS 89 | AnpAVyYoPjPtEpcNatIHbOna4KANxwv1Rmc6XDxWIB0RIlbdZDUhEdLovLLWGjm4 90 | J++Cnn2nOFdZyyUxwYTjDCMWwsYrG2oPZ0Yle6fKEXX30E+dN9NSV1i+dJAYQi0a 91 | m6augpg+LmFWxQ6JPmUJVDay9wo6g2D4KbJQybSh8lmqpenHnKD1m/gCGadPmMl6 92 | Rw== 93 | =RvPL 94 | -----END PGP PRIVATE KEY BLOCK----- 95 | ` 96 | -------------------------------------------------------------------------------- /ciphers/secconf/sec_interface.go: -------------------------------------------------------------------------------- 1 | package secconf 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "io/ioutil" 7 | 8 | "github.com/goraz/onion" 9 | ) 10 | 11 | type cipher struct { 12 | secretKeyring []byte 13 | } 14 | 15 | func (c *cipher) Decrypt(r io.Reader) ([]byte, error) { 16 | data, err := ioutil.ReadAll(r) 17 | if err != nil { 18 | return nil, err 19 | } 20 | return Decode(data, bytes.NewReader(c.secretKeyring)) 21 | } 22 | 23 | // NewCipher create a new cipher based on the secconf encoding as specified in the following 24 | // format: 25 | // base64(gpg(gzip(data))) 26 | func NewCipher(secRing io.Reader) (onion.Cipher, error) { 27 | b, err := ioutil.ReadAll(secRing) 28 | if err != nil { 29 | return nil, err 30 | } 31 | 32 | return &cipher{ 33 | secretKeyring: b, 34 | }, nil 35 | } 36 | -------------------------------------------------------------------------------- /ciphers/secconf/sec_interface_test.go: -------------------------------------------------------------------------------- 1 | package secconf 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | 7 | . "github.com/smartystreets/goconvey/convey" 8 | ) 9 | 10 | func TestNewCipher(t *testing.T) { 11 | Convey("Testing the newcipher", t, func() { 12 | data := "lorem ipsum" 13 | pubReader := bytes.NewReader([]byte(pubring)) 14 | b, err := Encode([]byte(data), pubReader) 15 | So(err, ShouldBeNil) 16 | 17 | secReader := bytes.NewReader([]byte(secring)) 18 | c, err := NewCipher(secReader) 19 | So(err, ShouldBeNil) 20 | br, err := c.Decrypt(bytes.NewBuffer(b)) 21 | So(err, ShouldBeNil) 22 | So(string(br), ShouldEqual, data) 23 | }) 24 | } 25 | -------------------------------------------------------------------------------- /ciphers/secconf/secconf.go: -------------------------------------------------------------------------------- 1 | // Package secconf implements secconf encoding as specified in the following 2 | // format: 3 | // 4 | // base64(gpg(gzip(data))) 5 | // 6 | package secconf 7 | 8 | import ( 9 | "bytes" 10 | "compress/gzip" 11 | "encoding/base64" 12 | "io" 13 | "io/ioutil" 14 | 15 | "golang.org/x/crypto/openpgp" 16 | ) 17 | 18 | // Decode decodes data using the secconf codec. 19 | func Decode(data []byte, secretKeyring io.Reader) ([]byte, error) { 20 | r := bytes.NewReader(data) 21 | decoder := base64.NewDecoder(base64.StdEncoding, r) 22 | entityList, err := openpgp.ReadArmoredKeyRing(secretKeyring) 23 | if err != nil { 24 | return nil, err 25 | } 26 | md, err := openpgp.ReadMessage(decoder, entityList, nil, nil) 27 | if err != nil { 28 | return nil, err 29 | } 30 | gzReader, err := gzip.NewReader(md.UnverifiedBody) 31 | if err != nil { 32 | return nil, err 33 | } 34 | defer func() { _ = gzReader.Close() }() 35 | b, err := ioutil.ReadAll(gzReader) 36 | if err != nil { 37 | return nil, err 38 | } 39 | return b, nil 40 | } 41 | 42 | // Encode encodes data to a base64 encoded using the secconf codec. 43 | // data is encrypted with all public keys found in the supplied keyring. 44 | func Encode(data []byte, keyring io.Reader) ([]byte, error) { 45 | entityList, err := openpgp.ReadArmoredKeyRing(keyring) 46 | if err != nil { 47 | return nil, err 48 | } 49 | 50 | buffer := new(bytes.Buffer) 51 | encoder := base64.NewEncoder(base64.StdEncoding, buffer) 52 | pgpWriter, err := openpgp.Encrypt(encoder, entityList, nil, nil, nil) 53 | if err != nil { 54 | return nil, err 55 | } 56 | 57 | gzWriter := gzip.NewWriter(pgpWriter) 58 | if _, err := gzWriter.Write(data); err != nil { 59 | return nil, err 60 | } 61 | 62 | if err := gzWriter.Close(); err != nil { 63 | return nil, err 64 | } 65 | 66 | if err := pgpWriter.Close(); err != nil { 67 | return nil, err 68 | } 69 | 70 | if err := encoder.Close(); err != nil { 71 | return nil, err 72 | } 73 | 74 | return buffer.Bytes(), nil 75 | } 76 | -------------------------------------------------------------------------------- /ciphers/secconf/secconf_test.go: -------------------------------------------------------------------------------- 1 | package secconf 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | ) 7 | 8 | var encodingTests = []struct { 9 | in, out string 10 | }{ 11 | {"secret", "secret"}, 12 | } 13 | 14 | func TestEncoding(t *testing.T) { 15 | for _, tt := range encodingTests { 16 | encoded, err := Encode([]byte(tt.in), bytes.NewBufferString(pubring)) 17 | if err != nil { 18 | t.Errorf(err.Error()) 19 | } 20 | decoded, err := Decode(encoded, bytes.NewBufferString(secring)) 21 | if err != nil { 22 | t.Errorf(err.Error()) 23 | } 24 | if tt.out != string(decoded) { 25 | t.Errorf("want %s, got %s", tt.out, decoded) 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /cli/onioncli/README.md: -------------------------------------------------------------------------------- 1 | # Onion CLI 2 | 3 | Onion cli is a simple tool to move configuration from a layer to another layer. 4 | 5 | Currently it can read/write data from file and etcd keys. also it support for encrypt/decrypt data using PGP. 6 | 7 | ## Installation 8 | 9 | ``` 10 | go get -u github.com/goraz/onion/cli/onioncli 11 | ``` 12 | 13 | if you want to encrypt/decrypt data using PGP, you need to create a private/public key pair using gpg (or any other tool) 14 | for example this is a fast way to create a `TEST` key pair (not protected with password) : 15 | 16 | ```bash 17 | export EMAIL="joe@foo.bar" 18 | export NAME="app" 19 | export GNUPGHOME="$(mktemp -d)" 20 | cat >foo < .pubring.gpg 35 | gpg --export-secret-keys --armor "${EMAIL}" > .secring.gpg 36 | 37 | ``` 38 | 39 | This should create two file, `.pubring.gpg` and `.secring.gpg` contains your testing (respectively) public and private keys. 40 | 41 | ## Usage 42 | 43 | for testing, create a plain `config.yaml` 44 | ```bash 45 | cat > config.yaml < 0; i-- { 161 | v, ok := searchStringMap(o.data[o.ll[i-1]], path...) 162 | if ok { 163 | return v, ok 164 | } 165 | } 166 | return nil, false 167 | } 168 | 169 | // GetIntDefault return an int value from Onion, if the value is not exists or its not an 170 | // integer , default is returned 171 | func GetIntDefault(key string, def int) int { 172 | return o.GetIntDefault(key, def) 173 | } 174 | 175 | // GetIntDefault return an int value from Onion, if the value is not exists or its not an 176 | // integer , default is returned 177 | func (o *Onion) GetIntDefault(key string, def int) int { 178 | return int(o.GetInt64Default(key, int64(def))) 179 | } 180 | 181 | // GetInt return an int value, if the value is not there, then it return zero value 182 | func GetInt(key string) int { 183 | return o.GetInt(key) 184 | } 185 | 186 | // GetInt return an int value, if the value is not there, then it return zero value 187 | func (o *Onion) GetInt(key string) int { 188 | return o.GetIntDefault(key, 0) 189 | } 190 | 191 | // GetInt64Default return an int64 value from Onion, if the value is not exists or if the value is not 192 | // int64 then return the default 193 | func GetInt64Default(key string, def int64) int64 { 194 | return o.GetInt64Default(key, def) 195 | } 196 | 197 | // GetInt64Default return an int64 value from Onion, if the value is not exists or if the value is not 198 | // int64 then return the default 199 | func (o *Onion) GetInt64Default(key string, def int64) int64 { 200 | v, ok := o.Get(key) 201 | if !ok { 202 | return def 203 | } 204 | 205 | switch nv := v.(type) { 206 | case string: 207 | // Env is not typed and always is String, so try to convert it to int 208 | // if possible 209 | i, err := strconv.ParseInt(nv, 10, 64) 210 | if err != nil { 211 | return def 212 | } 213 | return i 214 | case int: 215 | return int64(nv) 216 | case int64: 217 | return nv 218 | case float32: 219 | return int64(nv) 220 | case float64: 221 | return int64(nv) 222 | default: 223 | return def 224 | } 225 | 226 | } 227 | 228 | // GetInt64 return the int64 value from config, if its not there, return zero 229 | func GetInt64(key string) int64 { 230 | return o.GetInt64(key) 231 | } 232 | 233 | // GetInt64 return the int64 value from config, if its not there, return zero 234 | func (o *Onion) GetInt64(key string) int64 { 235 | return o.GetInt64Default(key, 0) 236 | } 237 | 238 | // GetFloat32Default return an float32 value from Onion, if the value is not exists or its not a 239 | // float32, default is returned 240 | func GetFloat32Default(key string, def float32) float32 { 241 | return o.GetFloat32Default(key, def) 242 | } 243 | 244 | // GetFloat32Default return an float32 value from Onion, if the value is not exists or its not a 245 | // float32, default is returned 246 | func (o *Onion) GetFloat32Default(key string, def float32) float32 { 247 | return float32(o.GetFloat64Default(key, float64(def))) 248 | } 249 | 250 | // GetFloat32 return an float32 value, if the value is not there, then it returns zero value 251 | func GetFloat32(key string) float32 { 252 | return o.GetFloat32(key) 253 | } 254 | 255 | // GetFloat32 return an float32 value, if the value is not there, then it returns zero value 256 | func (o *Onion) GetFloat32(key string) float32 { 257 | return o.GetFloat32Default(key, 0) 258 | } 259 | 260 | // GetFloat64Default return an float64 value from Onion, if the value is not exists or if the value is not 261 | // float64 then return the default 262 | func GetFloat64Default(key string, def float64) float64 { 263 | return o.GetFloat64Default(key, def) 264 | } 265 | 266 | // GetFloat64Default return an float64 value from Onion, if the value is not exists or if the value is not 267 | // float64 then return the default 268 | func (o *Onion) GetFloat64Default(key string, def float64) float64 { 269 | v, ok := o.Get(key) 270 | if !ok { 271 | return def 272 | } 273 | 274 | switch nv := v.(type) { 275 | case string: 276 | // Env is not typed and always is String, so try to convert it to int 277 | // if possible 278 | f, err := strconv.ParseFloat(nv, 64) 279 | if err != nil { 280 | return def 281 | } 282 | return f 283 | case int: 284 | return float64(nv) 285 | case int64: 286 | return float64(nv) 287 | case float32: 288 | return float64(nv) 289 | case float64: 290 | return nv 291 | default: 292 | return def 293 | } 294 | 295 | } 296 | 297 | // GetFloat64 return the float64 value from config, if its not there, return zero 298 | func GetFloat64(key string) float64 { 299 | return o.GetFloat64(key) 300 | } 301 | 302 | // GetFloat64 return the float64 value from config, if its not there, return zero 303 | func (o *Onion) GetFloat64(key string) float64 { 304 | return o.GetFloat64Default(key, 0) 305 | } 306 | 307 | // GetStringDefault get a string from Onion. if the value is not exists or if tha value is not 308 | // string, return the default 309 | func GetStringDefault(key string, def string) string { 310 | return o.GetStringDefault(key, def) 311 | } 312 | 313 | // GetStringDefault get a string from Onion. if the value is not exists or if tha value is not 314 | // string, return the default 315 | func (o *Onion) GetStringDefault(key string, def string) string { 316 | v, ok := o.Get(key) 317 | if !ok { 318 | return def 319 | } 320 | 321 | s, ok := v.(string) 322 | if !ok { 323 | return def 324 | } 325 | 326 | return s 327 | } 328 | 329 | // GetString is for getting an string from conig. if the key is not 330 | func GetString(key string) string { 331 | return o.GetString(key) 332 | } 333 | 334 | // GetString is for getting an string from conig. if the key is not 335 | func (o *Onion) GetString(key string) string { 336 | return o.GetStringDefault(key, "") 337 | } 338 | 339 | // GetBoolDefault return bool value from Onion. if the value is not exists or if tha value is not 340 | // boolean, return the default 341 | func GetBoolDefault(key string, def bool) bool { 342 | return o.GetBoolDefault(key, def) 343 | } 344 | 345 | // GetBoolDefault return bool value from Onion. if the value is not exists or if tha value is not 346 | // boolean, return the default 347 | func (o *Onion) GetBoolDefault(key string, def bool) bool { 348 | v, ok := o.Get(key) 349 | if !ok { 350 | return def 351 | } 352 | 353 | switch nv := v.(type) { 354 | case string: 355 | // Env is not typed and always is String, so try to convert it to boolean 356 | // if possible 357 | i, err := strconv.ParseBool(nv) 358 | if err != nil { 359 | return def 360 | } 361 | return i 362 | case bool: 363 | return nv 364 | default: 365 | return def 366 | } 367 | } 368 | 369 | // GetBool is used to get a boolean value fro config, with false as default 370 | func GetBool(key string) bool { 371 | return o.GetBool(key) 372 | } 373 | 374 | // GetBool is used to get a boolean value fro config, with false as default 375 | func (o *Onion) GetBool(key string) bool { 376 | return o.GetBoolDefault(key, false) 377 | } 378 | 379 | // GetDurationDefault is a function to get duration from config. it support both 380 | // string duration (like 1h3m2s) and integer duration 381 | func GetDurationDefault(key string, def time.Duration) time.Duration { 382 | return o.GetDurationDefault(key, def) 383 | } 384 | 385 | // GetDurationDefault is a function to get duration from config. it support both 386 | // string duration (like 1h3m2s) and integer duration 387 | func (o *Onion) GetDurationDefault(key string, def time.Duration) time.Duration { 388 | v, ok := o.Get(key) 389 | if !ok { 390 | return def 391 | } 392 | 393 | switch nv := v.(type) { 394 | case string: 395 | d, err := time.ParseDuration(nv) 396 | if err != nil { 397 | return def 398 | } 399 | return d 400 | case int: 401 | return time.Duration(nv) 402 | case int64: 403 | return time.Duration(nv) 404 | case time.Duration: 405 | return nv 406 | default: 407 | return def 408 | } 409 | } 410 | 411 | // GetDuration is for getting duration from config, it cast both int and string 412 | // to duration 413 | func GetDuration(key string) time.Duration { 414 | return o.GetDuration(key) 415 | } 416 | 417 | // GetDuration is for getting duration from config, it cast both int and string 418 | // to duration 419 | func (o *Onion) GetDuration(key string) time.Duration { 420 | return o.GetDurationDefault(key, 0) 421 | } 422 | 423 | func (o *Onion) getSlice(key string) (interface{}, bool) { 424 | v, ok := o.Get(key) 425 | if !ok { 426 | return nil, false 427 | } 428 | 429 | if reflect.TypeOf(v).Kind() != reflect.Slice { // Not good 430 | return nil, false 431 | } 432 | 433 | return v, true 434 | } 435 | 436 | // GetStringSlice try to get a slice from the config, also it support comma separated value 437 | // if there is no array at the key. 438 | func GetStringSlice(key string) []string { 439 | return o.GetStringSlice(key) 440 | } 441 | 442 | // GetStringSlice try to get a slice from the config, also it support comma separated value 443 | // if there is no array at the key. 444 | func (o *Onion) GetStringSlice(key string) []string { 445 | var ok bool 446 | v, ok := o.getSlice(key) 447 | if !ok { 448 | if v := o.GetString(key); len(v) > 0 { 449 | return strings.Split(v, ",") 450 | } 451 | return nil 452 | } 453 | 454 | switch nv := v.(type) { 455 | case []string: 456 | return nv 457 | case []interface{}: 458 | res := make([]string, len(nv)) 459 | for i := range nv { 460 | if res[i], ok = nv[i].(string); !ok { 461 | return nil 462 | } 463 | } 464 | return res 465 | } 466 | 467 | return nil 468 | } 469 | 470 | // LayersData is used to get all layers data at once, useful for test and also 471 | // used in the config writer 472 | func (o *Onion) LayersData() []map[string]interface{} { 473 | o.lock.RLock() 474 | defer o.lock.RUnlock() 475 | 476 | res := make([]map[string]interface{}, 0, len(o.ll)) 477 | for i := range o.ll { 478 | l := o.ll[i] 479 | res = append(res, o.data[l]) 480 | } 481 | 482 | return res 483 | } 484 | 485 | // NewContext return a new Onion, context is used for watch 486 | func NewContext(ctx context.Context, layers ...Layer) *Onion { 487 | o := &Onion{} 488 | 489 | o.AddLayersContext(ctx, layers...) 490 | return o 491 | } 492 | 493 | // New returns a new onion 494 | func New(layers ...Layer) *Onion { 495 | return NewContext(context.Background(), layers...) 496 | } 497 | -------------------------------------------------------------------------------- /onion_test.go: -------------------------------------------------------------------------------- 1 | package onion 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "reflect" 7 | "testing" 8 | "time" 9 | 10 | . "github.com/smartystreets/goconvey/convey" 11 | ) 12 | 13 | func getMap(prefix string, s ...interface{}) map[string]interface{} { 14 | tmp := make(map[string]interface{}) 15 | for i := range s { 16 | tmp[fmt.Sprintf("%s%d", prefix, i)] = s[i] 17 | } 18 | return tmp 19 | } 20 | 21 | type dummyWatch struct { 22 | data map[string]interface{} 23 | c chan map[string]interface{} 24 | } 25 | 26 | func (d *dummyWatch) Load() map[string]interface{} { 27 | return d.data 28 | } 29 | 30 | func (d *dummyWatch) Watch() <-chan map[string]interface{} { 31 | return d.c 32 | } 33 | 34 | func newDummy(data map[string]interface{}) *dummyWatch { 35 | return &dummyWatch{ 36 | data: data, 37 | c: make(chan map[string]interface{}), 38 | } 39 | } 40 | 41 | func TestOnion(t *testing.T) { 42 | Convey("Onion basic functionality", t, func() { 43 | data := getMap("key", 42, "universe", "answer", true, float32(20.88), float64(200.123), int64(100)) 44 | data["nested"] = getMap("n", "a", 99, true) 45 | 46 | t1 := make(map[interface{}]interface{}) 47 | t1["str1"] = 1 48 | t1["str2"] = "hi" 49 | t1["other"] = struct{}{} 50 | t1["what"] = getMap("n", "a") 51 | t2 := make(map[interface{}]interface{}) 52 | for k, v := range t1 { 53 | t2[k] = v 54 | } 55 | t1["nested"] = t2 56 | t2[true] = false 57 | 58 | data["yes"] = t1 59 | data["slice1"] = []string{"a", "b", "c"} 60 | data["slice2"] = []interface{}{"a", "b", "c"} 61 | data["slice3"] = []interface{}{"a", "b", true} 62 | data["slice4"] = []int{1, 2, 3} 63 | data["ignored"] = "ignore me" 64 | 65 | data["dur"] = time.Minute 66 | data["durstring"] = "1h2m3s" 67 | data["durstringinvalid"] = "ertyuiop" 68 | data["durint"] = 100000000 69 | data["durint64"] = int64(100000000) 70 | data["booldur"] = true 71 | 72 | lm := NewMapLayer(data) 73 | AddLayers(lm) 74 | Convey("Get direct variable", func() { 75 | So(GetInt("key0"), ShouldEqual, 42) 76 | So(GetString("key1"), ShouldEqual, "universe") 77 | So(GetString("key2"), ShouldEqual, "answer") 78 | So(GetBool("key3"), ShouldBeTrue) 79 | So(GetInt("key4"), ShouldEqual, 20) 80 | So(GetFloat32("key4"), ShouldEqual, 20.88) 81 | So(GetInt("key5"), ShouldEqual, 200) 82 | So(GetFloat64("key5"), ShouldEqual, 200.123) 83 | So(GetInt("key6"), ShouldEqual, 100) 84 | 85 | So(GetInt64("key0"), ShouldEqual, 42) 86 | So(GetInt64("key4"), ShouldEqual, 20) 87 | So(GetInt64("key5"), ShouldEqual, 200) 88 | So(GetInt64("key6"), ShouldEqual, 100) 89 | d, _ := time.ParseDuration("1h2m3s") 90 | So(GetDuration("durstring"), ShouldEqual, d) 91 | So(GetDuration("durstringinvalid"), ShouldEqual, 0) 92 | So(GetDuration("not-set-value"), ShouldEqual, 0) 93 | So(GetDuration("durint"), ShouldEqual, time.Duration(100000000)) 94 | So(GetDuration("durint64"), ShouldEqual, time.Duration(100000000)) 95 | So(GetDuration("booldur"), ShouldEqual, 0) 96 | So(GetDuration("dur"), ShouldEqual, time.Minute) 97 | }) 98 | 99 | Convey("Get default value", func() { 100 | So(GetIntDefault("key1", 0), ShouldEqual, 0) 101 | So(GetIntDefault("nokey1", 0), ShouldEqual, 0) 102 | So(GetStringDefault("key0", ""), ShouldEqual, "") 103 | So(GetStringDefault("nokey0", ""), ShouldEqual, "") 104 | So(GetBoolDefault("key0", false), ShouldBeFalse) 105 | So(GetBoolDefault("nokey0", false), ShouldBeFalse) 106 | So(GetInt64Default("key1", 0), ShouldEqual, 0) 107 | So(GetInt64Default("nokey1", 0), ShouldEqual, 0) 108 | 109 | So(GetInt64Default("", 0), ShouldEqual, 0) // Empty key 110 | So(GetInt64Default("key3", 10000), ShouldEqual, 10000) 111 | So(GetFloat32Default("", 0), ShouldEqual, 0) // Empty key 112 | So(GetFloat32Default("key3", 10000), ShouldEqual, 10000) 113 | So(GetFloat64Default("", 0.123), ShouldEqual, 0.123) // Empty key 114 | So(GetFloat64Default("key3", 10000.123), ShouldEqual, 10000.123) 115 | 116 | So(GetDurationDefault("not-set-value", time.Minute), ShouldEqual, time.Minute) 117 | }) 118 | 119 | Convey("Get nested variable", func() { 120 | So(GetStringDefault("nested.n0", ""), ShouldEqual, "a") 121 | So(GetInt64Default("nested.n1", 0), ShouldEqual, 99) 122 | So(GetIntDefault("nested.n1", 0), ShouldEqual, 99) 123 | So(GetBoolDefault("nested.n2", false), ShouldEqual, true) 124 | So(GetIntDefault("yes.str1", 0), ShouldEqual, 1) 125 | So(GetStringDefault("yes.str2", ""), ShouldEqual, "hi") 126 | So(GetStringDefault("yes.nested.str2", ""), ShouldEqual, "hi") 127 | So(GetStringDefault("yes.what.n0", ""), ShouldEqual, "a") 128 | }) 129 | 130 | Convey("Get nested default variable", func() { 131 | So(GetStringDefault("nested.n01", ""), ShouldEqual, "") 132 | So(GetStringDefault("key0.n01", ""), ShouldEqual, "") 133 | So(GetInt64Default("nested.n11", 0), ShouldEqual, 0) 134 | So(GetIntDefault("nested.n11", 0), ShouldEqual, 0) 135 | So(GetBoolDefault("nested.n21", false), ShouldEqual, false) 136 | 137 | So(GetStringDefault("yes.nested.no", "def"), ShouldEqual, "def") 138 | So(GetStringDefault("yes.nested.other.key", "def"), ShouldEqual, "def") 139 | So(GetStringDefault("yes.what.no", "def"), ShouldEqual, "def") 140 | }) 141 | 142 | Convey("change delimiter", func() { 143 | So(GetDelimiter(), ShouldEqual, ".") 144 | SetDelimiter("/") 145 | So(GetDelimiter(), ShouldEqual, "/") 146 | Convey("get with modified delimiter", func() { 147 | So(GetStringDefault("nested/n0", ""), ShouldEqual, "a") 148 | So(GetInt64Default("nested/n1", 0), ShouldEqual, 99) 149 | So(GetIntDefault("nested/n1", 0), ShouldEqual, 99) 150 | So(GetBoolDefault("nested/n2", false), ShouldEqual, true) 151 | So(GetStringDefault("nested.n0", ""), ShouldEqual, "") 152 | So(GetInt64Default("nested.n1", 0), ShouldEqual, 0) 153 | So(GetIntDefault("nested.n1", 0), ShouldEqual, 0) 154 | So(GetBoolDefault("nested.n2", false), ShouldEqual, false) 155 | So(GetStringDefault("key0/n01", ""), ShouldEqual, "") 156 | }) 157 | 158 | SetDelimiter("") 159 | So(GetDelimiter(), ShouldEqual, ".") 160 | }) 161 | 162 | Convey("slice test", func() { 163 | So(reflect.DeepEqual(GetStringSlice("slice1"), []string{"a", "b", "c"}), ShouldBeTrue) 164 | So(reflect.DeepEqual(GetStringSlice("slice2"), []string{"a", "b", "c"}), ShouldBeTrue) 165 | So(GetStringSlice("slice3"), ShouldBeNil) 166 | So(GetStringSlice("notslice3"), ShouldBeNil) 167 | So(GetStringSlice("yes.str1"), ShouldBeNil) 168 | So(GetStringSlice("slice4"), ShouldBeNil) 169 | }) 170 | }) 171 | 172 | Convey("Test layer overwrite", t, func() { 173 | lm1 := NewMapLayer(getMap("test", 1, true)) 174 | lm2 := NewMapLayer(getMap("test", 2, false)) 175 | os.Setenv("TEST0", "3") 176 | os.Setenv("TEST1", "True") 177 | os.Setenv("TEST2", "INVALIDBOOL") 178 | lm3 := NewEnvLayer("_", "TEST0", "TEST1", "TEST2") 179 | 180 | o := New(lm1) 181 | So(o.GetInt64Default("test0", 0), ShouldEqual, 1) 182 | So(o.GetBoolDefault("test1", false), ShouldBeTrue) 183 | o.AddLayers(lm2) 184 | So(o.GetInt64Default("test0", 0), ShouldEqual, 2) 185 | So(o.GetBoolDefault("test1", true), ShouldBeFalse) 186 | o.AddLayers(lm3) // Special case in ENV loader 187 | So(o.GetInt64Default("test0", 0), ShouldEqual, 3) 188 | So(o.GetBoolDefault("test1", false), ShouldBeTrue) 189 | So(o.GetBoolDefault("test2", false), ShouldBeFalse) 190 | }) 191 | 192 | Convey("test direct creation", t, func() { 193 | o := &Onion{} 194 | So(o.GetIntDefault("empty", 1000), ShouldEqual, 1000) 195 | lm := NewMapLayer(getMap("test", 1, true)) 196 | o1 := &Onion{} 197 | o1.AddLayers(lm) 198 | So(o1.GetIntDefault("test0", 0), ShouldEqual, 1) 199 | }) 200 | } 201 | 202 | func BenchmarkOion(b *testing.B) { 203 | for i := 0; i < b.N; i++ { 204 | benconion.GetInt("key0") 205 | benconion.GetString("key1") 206 | benconion.GetString("key2") 207 | benconion.GetBool("key3") 208 | benconion.GetInt("key4") 209 | benconion.GetInt("key5") 210 | benconion.GetInt("key6") 211 | 212 | benconion.GetInt64("key0") 213 | benconion.GetInt64("key4") 214 | benconion.GetInt64("key5") 215 | benconion.GetInt64("key6") 216 | benconion.GetDuration("durstring") 217 | benconion.GetDuration("durstringinvalid") 218 | benconion.GetDuration("not-set-value") 219 | benconion.GetDuration("durint") 220 | benconion.GetDuration("durint64") 221 | benconion.GetDuration("booldur") 222 | benconion.GetDuration("dur") 223 | } 224 | } 225 | 226 | var benconion = New() 227 | 228 | func init() { 229 | 230 | data := getMap("key", 42, "universe", "answer", true, float32(20.88), float64(200), int64(100)) 231 | data["nested"] = getMap("n", "a", 99, true) 232 | t1 := make(map[interface{}]interface{}) 233 | t1["str1"] = 1 234 | t1["str2"] = "hi" 235 | t1["other"] = struct{}{} 236 | t1["what"] = getMap("n", "a") 237 | t2 := make(map[interface{}]interface{}) 238 | for k, v := range t1 { 239 | t2[k] = v 240 | } 241 | t1["nested"] = t2 242 | t2[true] = false 243 | 244 | data["yes"] = t1 245 | data["slice1"] = []string{"a", "b", "c"} 246 | data["slice2"] = []interface{}{"a", "b", "c"} 247 | data["slice3"] = []interface{}{"a", "b", true} 248 | data["slice4"] = []int{1, 2, 3} 249 | data["ignored"] = "ignore me" 250 | 251 | data["dur"] = time.Minute 252 | data["durstring"] = "1h2m3s" 253 | data["durstringinvalid"] = "ertyuiop" 254 | data["durint"] = 100000000 255 | data["durint64"] = int64(100000000) 256 | data["booldur"] = true 257 | 258 | benconion.AddLayers(NewMapLayer(data)) 259 | } 260 | 261 | func TestLayersData(t *testing.T) { 262 | Convey("Test merge", t, func() { 263 | o := New() 264 | l1 := NewMapLayer(map[string]interface{}{ 265 | "key1": 1, 266 | "key2": 2, 267 | }) 268 | l2 := NewMapLayer(map[string]interface{}{ 269 | "key1": 10, 270 | "key3": 3, 271 | }) 272 | 273 | o.AddLayers(l1, l2) 274 | 275 | ret := []map[string]interface{}{ 276 | map[string]interface{}{ 277 | "key1": 1, 278 | "key2": 2, 279 | }, 280 | map[string]interface{}{ 281 | "key1": 10, 282 | "key3": 3, 283 | }, 284 | } 285 | 286 | So(o.LayersData(), ShouldResemble, ret) 287 | }) 288 | } 289 | 290 | func TestWatch(t *testing.T) { 291 | Convey("Test watch", t, func() { 292 | data := map[string]interface{}{ 293 | "k1": "10.0", 294 | } 295 | 296 | l := newDummy(data) 297 | o := New(l) 298 | ch := o.ReloadWatch() 299 | So(o.GetFloat32("k1"), ShouldEqual, 10.0) 300 | data["k1"] = "100.0" 301 | l.c <- data 302 | <-ch 303 | So(o.GetFloat32("k1"), ShouldEqual, 100.0) 304 | }) 305 | } 306 | -------------------------------------------------------------------------------- /onionwriter/onion_writer.go: -------------------------------------------------------------------------------- 1 | package onionwriter 2 | 3 | import ( 4 | "encoding/json" 5 | "io" 6 | 7 | "github.com/goraz/onion" 8 | "github.com/mitchellh/mapstructure" 9 | ) 10 | 11 | // SerializeOnion try to serialize the onion into a json stream. 12 | func SerializeOnion(o *onion.Onion, w io.Writer) error { 13 | data := o.LayersData() 14 | 15 | mergedData := onion.NewMapLayer(data...) 16 | 17 | enc := json.NewEncoder(w) 18 | enc.SetIndent("", " ") 19 | return enc.Encode(mergedData.Load()) 20 | } 21 | 22 | // MergeLayersOnion is used to get all layers data merged into one 23 | // Latest added overwrite previous ones. 24 | func MergeLayersOnion(o *onion.Onion) map[string]interface{} { 25 | layersData := o.LayersData() 26 | 27 | return onion.NewMapLayer(layersData...).Load() 28 | } 29 | 30 | // DecodeOnion try to convert merged layers in the output structure. 31 | // output must be a pointer to a map or struct. 32 | func DecodeOnion(o *onion.Onion, output interface{}) error { 33 | merged := MergeLayersOnion(o) 34 | 35 | return mapstructure.Decode(merged, &output) 36 | } 37 | -------------------------------------------------------------------------------- /onionwriter/onion_writer_test.go: -------------------------------------------------------------------------------- 1 | package onionwriter 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "os" 8 | "testing" 9 | 10 | "github.com/goraz/onion" 11 | . "github.com/smartystreets/goconvey/convey" 12 | ) 13 | 14 | func getMap(prefix string, s ...interface{}) map[string]interface{} { 15 | tmp := make(map[string]interface{}) 16 | for i := range s { 17 | tmp[fmt.Sprintf("%s%d", prefix, i)] = s[i] 18 | } 19 | return tmp 20 | } 21 | 22 | func TestSerializeOnion(t *testing.T) { 23 | Convey("Serialize the onion into json", t, func() { 24 | m1 := map[string]interface{}{ 25 | "key1": "HALLO", 26 | "key2": "JA", 27 | "key3": 100, 28 | "key4": []string{}, 29 | } 30 | 31 | m2 := map[string]interface{}{ 32 | "key1": "HELLO", 33 | "key4": []string{"a", "b", "c"}, 34 | } 35 | 36 | o := onion.New(onion.NewMapLayer(m1), onion.NewMapLayer(m2)) 37 | buf := &bytes.Buffer{} 38 | So(SerializeOnion(o, buf), ShouldBeNil) 39 | data := make(map[string]interface{}) 40 | So(json.Unmarshal(buf.Bytes(), &data), ShouldBeNil) 41 | o2 := onion.New(onion.NewMapLayer(data)) 42 | 43 | So(o2.GetString("key1"), ShouldEqual, o.GetString("key1")) 44 | So(o2.GetString("key2"), ShouldEqual, o.GetString("key2")) 45 | So(o2.GetInt("key3"), ShouldEqual, o.GetInt("key3")) 46 | So(o2.GetStringSlice("key4"), ShouldResemble, o.GetStringSlice("key4")) 47 | }) 48 | 49 | Convey("Test layers merge", t, func() { 50 | lm1 := onion.NewMapLayer(getMap("test", 1, true)) 51 | lm2 := onion.NewMapLayer(getMap("test", 2, false)) 52 | os.Setenv("TEST0", "3") 53 | os.Setenv("TEST1", "True") 54 | os.Setenv("TEST2", "INVALIDBOOL") 55 | lm3 := onion.NewEnvLayer("_", "TEST0", "TEST1", "TEST2") 56 | 57 | o := onion.New(lm1) 58 | o.AddLayers(lm2) 59 | 60 | mergedLayers := MergeLayersOnion(o) 61 | So(mergedLayers["test0"], ShouldEqual, 2) 62 | So(mergedLayers["test1"], ShouldBeFalse) 63 | 64 | o.AddLayers(lm3) // Special case in ENV loader 65 | mergedLayers = MergeLayersOnion(o) 66 | So(mergedLayers["test0"], ShouldEqual, "3") 67 | So(mergedLayers["test1"], ShouldEqual, "True") 68 | So(mergedLayers["test2"], ShouldEqual, "INVALIDBOOL") 69 | }) 70 | 71 | Convey("Test layers merge and decode to struct", t, func() { 72 | type Config struct { 73 | Test0 int64 74 | Test1 bool 75 | Test2 bool 76 | Test3 string 77 | } 78 | 79 | lm1 := onion.NewMapLayer(getMap("test", 1, true)) 80 | lm2 := onion.NewMapLayer(getMap("test", 2, false)) 81 | 82 | o := onion.New(lm1) 83 | o.AddLayers(lm2) 84 | 85 | var conf, conf2 Config 86 | 87 | So(DecodeOnion(o, &conf), ShouldBeNil) 88 | So(conf.Test0, ShouldEqual, 2) 89 | So(conf.Test1, ShouldBeFalse) 90 | 91 | os.Setenv("TEST3", "ALongStringInSnakeCase") 92 | 93 | lm3 := onion.NewEnvLayer("_", "TEST3") 94 | o.AddLayers(lm3) // Special case in ENV loader 95 | 96 | So(DecodeOnion(o, &conf2), ShouldBeNil) 97 | So(conf2.Test0, ShouldEqual, 2) 98 | So(conf2.Test1, ShouldBeFalse) 99 | So(conf2.Test2, ShouldBeFalse) 100 | So(conf2.Test3, ShouldEqual, "ALongStringInSnakeCase") 101 | }) 102 | 103 | } 104 | -------------------------------------------------------------------------------- /stream_layer.go: -------------------------------------------------------------------------------- 1 | package onion 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "io" 9 | "log" 10 | "os" 11 | "path/filepath" 12 | "strings" 13 | "sync" 14 | ) 15 | 16 | var ( 17 | decLock sync.RWMutex 18 | decoders = map[string]Decoder{ 19 | "json": &jsonDecoder{}, 20 | } 21 | ) 22 | 23 | // Cipher is used to decrypt data on loading 24 | type Cipher interface { 25 | Decrypt(io.Reader) ([]byte, error) 26 | } 27 | 28 | // Decoder is a stream decoder to convert a stream into a map of config keys, json is supported out of 29 | // the box 30 | type Decoder interface { 31 | Decode(context.Context, io.Reader) (map[string]interface{}, error) 32 | } 33 | 34 | type jsonDecoder struct { 35 | } 36 | 37 | func decrypt(c Cipher, r io.Reader) (io.Reader, error) { 38 | if c == nil { 39 | return r, nil 40 | } 41 | b, err := c.Decrypt(r) 42 | if err != nil { 43 | return nil, err 44 | } 45 | 46 | return bytes.NewReader(b), nil 47 | } 48 | 49 | func (jd *jsonDecoder) Decode(_ context.Context, r io.Reader) (map[string]interface{}, error) { 50 | var data map[string]interface{} 51 | if err := json.NewDecoder(r).Decode(&data); err != nil { 52 | return nil, err 53 | } 54 | 55 | return data, nil 56 | } 57 | 58 | // RegisterDecoder add a new decoder to the system, json is registered out of the box 59 | func RegisterDecoder(dec Decoder, formats ...string) { 60 | decLock.Lock() 61 | defer decLock.Unlock() 62 | 63 | for _, format := range formats { 64 | format := strings.ToLower(format) 65 | 66 | _, alreadyExists := decoders[format] 67 | if alreadyExists { 68 | log.Fatalf("decoder for format %q is already registered: you can have only one", format) 69 | } 70 | 71 | decoders[format] = dec 72 | } 73 | } 74 | 75 | // GetDecoder returns the decoder based on its name, it may returns nil if the decoder is not 76 | // registered 77 | func GetDecoder(format string) Decoder { 78 | decLock.RLock() 79 | defer decLock.RUnlock() 80 | 81 | return decoders[strings.ToLower(format)] 82 | } 83 | 84 | type streamLayer struct { 85 | c chan map[string]interface{} 86 | cipher Cipher 87 | } 88 | 89 | func (sl *streamLayer) Load() map[string]interface{} { 90 | return <-sl.c 91 | } 92 | 93 | func (sl *streamLayer) Watch() <-chan map[string]interface{} { 94 | return sl.c 95 | } 96 | 97 | func (sl *streamLayer) Reload(ctx context.Context, r io.Reader, format string) error { 98 | dec := GetDecoder(format) 99 | if dec == nil { 100 | return fmt.Errorf("format %q is not registered", format) 101 | } 102 | dr, err := decrypt(sl.cipher, r) 103 | if err != nil { 104 | return err 105 | } 106 | 107 | data, err := dec.Decode(ctx, dr) 108 | if err != nil { 109 | return err 110 | } 111 | 112 | go func() { 113 | select { 114 | case sl.c <- data: 115 | case <-ctx.Done(): 116 | } 117 | }() 118 | 119 | return nil 120 | } 121 | 122 | // NewStreamLayerContext try to create a layer based on a stream, the format should be a registered 123 | // format (see RegisterDecoder) and if the Cipher is not nil, it pass data to cipher first. 124 | // A nil cipher is accepted as plain cipher 125 | func NewStreamLayerContext(ctx context.Context, r io.Reader, format string, c Cipher) (Layer, error) { 126 | if r == nil { 127 | return nil, fmt.Errorf("nil stream") 128 | } 129 | sl := &streamLayer{ 130 | c: make(chan map[string]interface{}), 131 | cipher: c, 132 | } 133 | 134 | err := sl.Reload(ctx, r, format) 135 | if err != nil { 136 | return nil, err 137 | } 138 | 139 | return sl, nil 140 | } 141 | 142 | // NewStreamLayer create new stream layer, see the NewStreamLayerContext 143 | func NewStreamLayer(r io.Reader, format string, c Cipher) (Layer, error) { 144 | return NewStreamLayerContext(context.Background(), r, format, c) 145 | } 146 | 147 | // NewFileLayerContext create a new file layer. it choose the format base on the extension 148 | func NewFileLayerContext(ctx context.Context, path string, c Cipher) (Layer, error) { 149 | ext := strings.TrimPrefix(filepath.Ext(path), ".") 150 | f, err := os.Open(path) 151 | if err != nil { 152 | return nil, err 153 | } 154 | defer func() { _ = f.Close() }() 155 | 156 | return NewStreamLayerContext(ctx, f, ext, c) 157 | } 158 | 159 | // NewFileLayer create a new file layer. it choose the format base on the extension 160 | func NewFileLayer(path string, c Cipher) (Layer, error) { 161 | return NewFileLayerContext(context.Background(), path, c) 162 | } 163 | -------------------------------------------------------------------------------- /stream_layer_test.go: -------------------------------------------------------------------------------- 1 | package onion 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "io" 7 | "testing" 8 | 9 | . "github.com/smartystreets/goconvey/convey" 10 | ) 11 | 12 | const validJSON = ` 13 | { 14 | "string" : "str", 15 | "number" : 100, 16 | "nested" : { 17 | "bool" : "true" 18 | } 19 | } 20 | ` 21 | 22 | func TestNewStreamLayer(t *testing.T) { 23 | Convey("Stream layer test", t, func() { 24 | buf := bytes.NewBufferString(validJSON) 25 | l, err := NewStreamLayer(buf, "json", nil) 26 | So(err, ShouldBeNil) 27 | o := New(l) 28 | So(o.GetString("string"), ShouldEqual, "str") 29 | So(o.GetInt("number"), ShouldEqual, 100) 30 | So(o.GetBool("nested.bool"), ShouldBeTrue) 31 | }) 32 | } 33 | 34 | type dummyDecoder struct { 35 | data map[string]interface{} 36 | err error 37 | } 38 | 39 | func (d *dummyDecoder) Decode(context.Context, io.Reader) (map[string]interface{}, error) { 40 | return d.data, d.err 41 | } 42 | 43 | func TestRegisterDecoder(t *testing.T) { 44 | Convey("Test dummy decoder", t, func() { 45 | RegisterDecoder(&dummyDecoder{ 46 | data: map[string]interface{}{ 47 | "hi": 10, 48 | }, 49 | }, "dummy") 50 | l, err := NewStreamLayer(bytes.NewReader(nil), "dummy", nil) 51 | So(err, ShouldBeNil) 52 | o := New(l) 53 | So(o.GetInt("hi"), ShouldEqual, 10) 54 | }) 55 | 56 | Convey("Fail decode", t, func() { 57 | _, err := NewStreamLayer(nil, "hi_i_am_not_a_format", nil) 58 | So(err, ShouldBeError) 59 | 60 | buf := bytes.NewBufferString(`{INVALID}`) 61 | _, err = NewStreamLayer(buf, "json", nil) 62 | So(err, ShouldBeError) 63 | 64 | _, err = NewStreamLayer(nil, "json", nil) 65 | So(err, ShouldNotBeNil) 66 | }) 67 | } 68 | -------------------------------------------------------------------------------- /utils.go: -------------------------------------------------------------------------------- 1 | package onion 2 | 3 | // MergeLayersData is an helper function to merge more layers in one. 4 | // Following slice order, a previous layer key is overriden by an equal key in 5 | // next layer. 6 | func mergeLayersData(layers ...map[string]interface{}) map[string]interface{} { 7 | if len(layers) == 0 { 8 | return map[string]interface{}{} 9 | } 10 | 11 | mergedLayer := layers[len(layers)-1] 12 | layers = layers[:len(layers)-1] 13 | 14 | for i := len(layers) - 1; i >= 0; i-- { 15 | mergedLayer = mergeKeys(mergedLayer, layers[i]) 16 | } 17 | 18 | return mergedLayer 19 | } 20 | 21 | // mergeKeys recursively merge right into left, never replacing any key that already exists in left 22 | func mergeKeys(left, right map[string]interface{}) map[string]interface{} { 23 | if left == nil { 24 | return right 25 | } 26 | 27 | for key, rightVal := range right { 28 | 29 | if _, present := left[key]; !present { 30 | left[key] = rightVal 31 | continue 32 | } 33 | 34 | leftMap, isLeftValAMap := left[key].(map[string]interface{}) 35 | rightMap, isRightValAMap := rightVal.(map[string]interface{}) 36 | if isLeftValAMap && isRightValAMap { 37 | left[key] = mergeKeys(leftMap, rightMap) 38 | } 39 | } 40 | 41 | return left 42 | } 43 | -------------------------------------------------------------------------------- /utils_test.go: -------------------------------------------------------------------------------- 1 | package onion 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/smartystreets/goconvey/convey" 7 | ) 8 | 9 | func TestMergeLayer(t *testing.T) { 10 | Convey("Test merge data", t, func() { 11 | data1 := map[string]interface{}{ 12 | "1": 10, 13 | "2": 20, 14 | "3": 30, 15 | } 16 | 17 | data2 := map[string]interface{}{ 18 | "1": 100, 19 | } 20 | 21 | data3 := map[string]interface{}{ 22 | "1": 1000, 23 | "3": 3000, 24 | } 25 | 26 | merged := mergeLayersData(data1, data2, data3) 27 | So(merged["1"], ShouldEqual, 1000) 28 | So(merged["2"], ShouldEqual, 20) 29 | So(merged["3"], ShouldEqual, 3000) 30 | 31 | merged = mergeLayersData(data3, data2, data1) 32 | So(merged["1"], ShouldEqual, 10) 33 | So(merged["2"], ShouldEqual, 20) 34 | So(merged["3"], ShouldEqual, 30) 35 | 36 | merged = mergeLayersData(data3, data1, data2) 37 | So(merged["1"], ShouldEqual, 100) 38 | So(merged["2"], ShouldEqual, 20) 39 | So(merged["3"], ShouldEqual, 30) 40 | 41 | merged = mergeLayersData(data1) 42 | So(merged["1"], ShouldEqual, 10) 43 | So(merged["2"], ShouldEqual, 20) 44 | So(merged["3"], ShouldEqual, 30) 45 | 46 | merged = mergeLayersData() 47 | So(len(merged), ShouldEqual, 0) 48 | }) 49 | } 50 | --------------------------------------------------------------------------------