├── .gitignore ├── .gitmodules ├── .travis.yml ├── AUTHORS ├── CONTRIBUTING ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── cmd └── remco │ ├── config.go │ ├── config_test.go │ ├── main.go │ ├── supervisor.go │ ├── supervisor_test.go │ └── version.go ├── docs ├── config.toml ├── content │ ├── config │ │ ├── configuration-options.md │ │ ├── environment-variables.md │ │ ├── index.md │ │ ├── sample-config.md │ │ └── sample-resource.md │ ├── details │ │ ├── backends.md │ │ ├── commands.md │ │ ├── exec-mode.md │ │ ├── index.md │ │ ├── plugins.md │ │ ├── process-lifecycle.md │ │ ├── telemetry.md │ │ ├── template-resource.md │ │ └── zombie-reaping.md │ ├── examples │ │ ├── haproxy.md │ │ └── index.md │ ├── plugins │ │ ├── consul-plugin-example.md │ │ ├── env-plugin-example.md │ │ └── index.md │ └── template │ │ ├── index.md │ │ ├── template-filters.md │ │ └── template-functions.md ├── images │ └── Remco-overview.svg ├── layouts │ ├── index.html │ └── partials │ │ ├── logo.html │ │ └── style.html └── static │ ├── .gitkeep │ └── style.css ├── go.mod ├── go.sum ├── integration ├── consul │ ├── consul.toml │ └── test.sh ├── env │ ├── env.toml │ └── test.sh ├── etcdv2 │ ├── etcd.toml │ └── test.sh ├── etcdv3 │ ├── etcd.toml │ └── test.sh ├── file │ ├── config.yml │ ├── file.toml │ └── test.sh ├── redis │ ├── redis.toml │ └── test.sh ├── templates │ └── basic.conf.tmpl ├── vault │ ├── test.sh │ └── vault.toml └── zookeeper │ ├── test.sh │ └── zookeeper.toml ├── pkg ├── backends │ ├── config_types.go │ ├── consul.go │ ├── env.go │ ├── error │ │ └── error.go │ ├── etcd.go │ ├── file.go │ ├── mock.go │ ├── nats.go │ ├── plugin │ │ └── plugin.go │ ├── redis.go │ ├── vault.go │ └── zookeeper.go ├── log │ ├── log.go │ └── log_test.go ├── telemetry │ ├── inmem.go │ ├── prometheus.go │ ├── statsd.go │ ├── statsite.go │ ├── telemetry.go │ └── telemetry_test.go └── template │ ├── backend.go │ ├── executor.go │ ├── executor_test.go │ ├── fileutil │ ├── fileStat_posix.go │ ├── fileStat_windows.go │ ├── fileutil.go │ └── fileutil_test.go │ ├── renderer.go │ ├── resource.go │ ├── resource_test.go │ ├── template_filters.go │ ├── template_filters_test.go │ ├── template_funcs.go │ ├── template_funcs_test.go │ └── util.go ├── scripts └── deploy-ghpages.sh └── test /.gitignore: -------------------------------------------------------------------------------- 1 | bin/ 2 | .idea/ 3 | .DS_Store -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "hugo-theme-learn"] 2 | path = docs/themes/hugo-theme-learn 3 | url = https://github.com/matcornic/hugo-theme-learn.git -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | before_install: 4 | - go get honnef.co/go/tools/cmd/staticcheck 5 | 6 | # install consul 7 | - wget https://releases.hashicorp.com/consul/1.5.3/consul_1.5.3_linux_amd64.zip 8 | - unzip consul_1.5.3_linux_amd64.zip 9 | - sudo mv consul /bin/ 10 | - consul agent -server -bootstrap-expect 1 -data-dir /tmp/consul -bind 127.0.0.1 & 11 | # install etcd 12 | - curl -L https://github.com/etcd-io/etcd/releases/download/v3.3.13/etcd-v3.3.13-linux-amd64.tar.gz -o etcd-v3.3.13-linux-amd64.tar.gz 13 | - tar xzf etcd-v3.3.13-linux-amd64.tar.gz 14 | - sudo mv etcd-v3.3.13-linux-amd64/etcd /bin/ 15 | - sudo mv etcd-v3.3.13-linux-amd64/etcdctl /bin/ 16 | - etcd & 17 | # Install vault 18 | - wget https://releases.hashicorp.com/vault/1.2.1/vault_1.2.1_linux_amd64.zip 19 | - unzip vault_1.2.1_linux_amd64.zip 20 | - sudo mv vault /bin/ 21 | - vault server -dev & 22 | # Install zookeeper 23 | - wget https://archive.apache.org/dist/zookeeper/zookeeper-3.4.9/zookeeper-3.4.9.tar.gz 24 | - tar xzf zookeeper-3.4.9.tar.gz 25 | - echo "tickTime=2000" > zookeeper-3.4.9/conf/zoo.cfg 26 | - echo "dataDir=/tmp/zookeeper" >> zookeeper-3.4.9/conf/zoo.cfg 27 | - echo "clientPort=2181" >> zookeeper-3.4.9/conf/zoo.cfg 28 | - mkdir /tmp/zookeeper 29 | - zookeeper-3.4.9/bin/zkServer.sh start 30 | # https://github.com/travis-ci/travis-ci/issues/8229 31 | - export GOROOT=$(go env GOROOT) 32 | 33 | install: 34 | - make build 35 | - sudo make install 36 | 37 | go: 38 | - 1.12.x 39 | - 1.13.x 40 | 41 | env: 42 | - VAULT_ADDR='http://127.0.0.1:8200' GO111MODULE=on 43 | 44 | services: 45 | - redis 46 | 47 | before_script: 48 | - go vet $(go list ./... | grep -v /vendor/) 49 | - staticcheck $(go list ./... | grep -v /vendor/) 50 | 51 | script: 52 | - ./test 53 | - bash integration/consul/test.sh 54 | - bash integration/etcdv2/test.sh 55 | - bash integration/etcdv3/test.sh 56 | - bash integration/file/test.sh 57 | - bash integration/vault/test.sh 58 | - bash integration/redis/test.sh 59 | - bash integration/env/test.sh 60 | - bash integration/zookeeper/test.sh 61 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | List of remco contributors and authors; these are the copyright holders 2 | for remco, referred to as The Remco Authors. 3 | 4 | Rene Kaufmann 5 | Rashit Azizbaev 6 | Stefan Seide 7 | -------------------------------------------------------------------------------- /CONTRIBUTING: -------------------------------------------------------------------------------- 1 | 1. First check to make sure if an issue for the problem you're addressing, 2 | or feature you're adding, has already been filed. If not, file one here: 3 | 4 | https://github.com/HeavyHorst/remco/issues 5 | 6 | Please indicate in the description of the issue that you're working on 7 | the issue, so we don't duplicate effort. 8 | 9 | 2. By submitting code to the project, you are asserting that the work is 10 | either your own, or that you are authorized to submit it to this project. 11 | Further, you are asserting that the project may continue to use, modify, 12 | and redistribute your contribution under the terms in the LICENSE file. 13 | 14 | 3. All code must pass go vet, and be go fmt compliant. 15 | 16 | 4. Feel free to add your name to the end of the AUTHORS file. (Please do this, particularly for non-trivial 17 | changes!) 18 | 19 | 5. Submit a github pull request. 20 | 21 | Thank you for your contributions! 22 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:buster AS build 2 | 3 | WORKDIR /src 4 | COPY . . 5 | RUN make 6 | 7 | FROM scratch AS bin 8 | COPY --from=build /src/bin/remco /remco 9 | 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright © 2013 Kelsey Hightower 2 | Copyright © 2016 The Remco Authors 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | this software and associated documentation files (the "Software"), to deal in 6 | the Software without restriction, including without limitation the rights to 7 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 8 | of the Software, and to permit persons to whom the Software is furnished to do 9 | so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in all 12 | copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 20 | SOFTWARE. 21 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: build clean test help default tag fmt vendor vet install release 2 | 3 | BIN_NAME := bin/remco 4 | 5 | GOARCH ?= amd64 6 | 7 | VERSION := 0.12.4 8 | GIT_COMMIT := $(shell git rev-parse HEAD) 9 | GIT_DIRTY := $(shell test -n "`git status --porcelain`" && echo "+CHANGES" || true) 10 | BUILD_DATE := $(shell date '+%Y-%m-%d-%H:%M:%S') 11 | 12 | SYSCONFDIR := /etc/ 13 | PREFIX := /usr/local 14 | BINDIR := ${PREFIX}/bin 15 | 16 | DEFAULTCONFDIR := ${SYSCONFDIR}/remco/ 17 | DEFAULTCONF := ${DEFAULTCONFDIR}/config 18 | 19 | 20 | GO_SRC := $(shell find ./ -type f -name '*.go' -and -not -name '*_test.go') 21 | GO_TEST_SRC := $(shell find ./ -type f -name '*_test.go') 22 | 23 | GO := go 24 | 25 | GO_OPTS := -mod=mod 26 | 27 | OS_LIST := linux darwin windows 28 | 29 | OUT_RELEASE_ZIP := $(addsuffix _$(GOARCH).zip, $(addprefix bin/remco_$(VERSION)_, $(OS_LIST))) 30 | 31 | default: build 32 | 33 | help: 34 | @echo 'Management commands for remco:' 35 | @echo 36 | @echo 'Usage:' 37 | @echo ' make build Compile the project.' 38 | @echo ' make release Create all the releases for [$(OS_LIST)]' 39 | @echo ' make test Run the unit tests.' 40 | @echo ' make vendor Recover the deps (put them in /vendor)' 41 | @echo ' make fmt use go fmt on the code.' 42 | @echo ' make vet use go vet on the code.' 43 | @echo ' make clean Clean the directory tree.' 44 | @echo 45 | 46 | build: ${BIN_NAME} 47 | 48 | ${BIN_NAME}: $(GO_SRC) 49 | @echo "building ${BIN_NAME} ${VERSION}" 50 | $(GO) build -a -tags netgo -ldflags "-s -w -X main.version=${VERSION} \ 51 | -X main.buildDate=${BUILD_DATE} \ 52 | -X main.commit=${GIT_COMMIT}${GIT_DIRTY}" \ 53 | -o ${BIN_NAME} ${GO_OPTS} ./cmd/remco/ 54 | 55 | vendor: 56 | @echo "recovering/vendoring the dependencies" 57 | $(GO) mod vendor 58 | 59 | clean: 60 | @echo "Cleaning up" 61 | @test ! -e ${BIN_NAME} || rm ${BIN_NAME} 62 | @test ! -e coverage.out || rm coverage.out 63 | @test ! -e coverage.html || rm coverage.html 64 | @rm -f bin/*.zip 65 | @rm -f bin/remco_* 66 | 67 | test-browser-cov: test 68 | $(GO) tool cover -html=coverage.out 69 | 70 | test: coverage.out 71 | 72 | fmt: 73 | $(GO) fmt ... 74 | 75 | vet: 76 | $(GO) vet ... 77 | 78 | coverage.out: $(GO_SRC) $(GO_TEST_SRC) build 79 | @echo "Running the test" 80 | $(GO) test ./... -race ${GO_OPTS} -coverprofile=coverage.out 81 | $(GO) tool cover -html=coverage.out -o coverage.html 82 | @echo "report available in coverage.html" 83 | 84 | install: build 85 | @echo "Installing remco" 86 | @mkdir -p ${DESTDIR}/${BINDIR} 87 | @mkdir -p ${DESTDIR}/${DEFAULTCONFDIR} 88 | @install -m 755 ${BIN_NAME} ${DESTDIR}/${BINDIR} 89 | @if ! [ -e "${DESTDIR}/${DEFAULTCONF}" ];\ 90 | then \ 91 | install -m 640 ./integration/file/file.toml ${DESTDIR}/${DEFAULTCONF};\ 92 | else \ 93 | echo "conf file '${DESTDIR}/${DEFAULTCONF}' already present";\ 94 | fi 95 | 96 | tag: 97 | git tag -a v${VERSION} -m "version ${VERSION}" 98 | git push origin v${VERSION} 99 | 100 | release: $(OUT_RELEASE_ZIP) 101 | 102 | $(OUT_RELEASE_ZIP): $(GO_SRC) 103 | GOARCH=$(GOARCH) CGO_ENABLED=0 GOOS=$(subst bin/remco_${VERSION}_,,$(subst _$(GOARCH).zip,,$@)) \ 104 | $(MAKE) build \ 105 | BIN_NAME=$(subst ${VERSION}_,,$(subst _$(GOARCH).zip,,$@)) 106 | cd bin && zip -r $(shell basename $@) $(shell basename $(subst ${VERSION}_,,$(subst _$(GOARCH).zip,,$@))) 107 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/HeavyHorst/remco.svg?branch=master)](https://travis-ci.org/HeavyHorst/remco) [![Go Report Card](https://goreportcard.com/badge/github.com/HeavyHorst/remco)](https://goreportcard.com/report/github.com/HeavyHorst/remco) [![MIT licensed](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/HeavyHorst/remco/master/LICENSE) 2 | 3 | # Remco 4 | 5 | remco is a lightweight configuration management tool. It's highly influenced by [confd](https://github.com/kelseyhightower/confd). 6 | Remcos main purposes are (like confd's): 7 | 8 | - keeping local configuration files up-to-date using data stored in a key/value store like etcd or consul and processing template resources. 9 | - reloading applications to pick up new config file changes 10 | 11 | ## Differences between remco and confd 12 | 13 | - Multiple source/destination pairs per template resource - useful for programs that need more than one config file 14 | - Multiple backends per template resource - get normal config values from etcd and secrets from vault 15 | - [Pongo2](https://github.com/flosch/pongo2) template engine instead of go's text/template 16 | - Zombie reaping support (if remco runs as pid 1) 17 | - Additional backends can be provided as plugins. 18 | - Create your own custom template filters easily with JavaScript. 19 | - [Exec](https://heavyhorst.github.io/remco/details/exec-mode/) mode similar to consul-template. 20 | 21 | ## Overview 22 | 23 | ![remco overview](https://cdn.rawgit.com/HeavyHorst/remco/master/docs/images/Remco-overview.svg) 24 | 25 | ## Documentation 26 | 27 | See: https://heavyhorst.github.io/remco/ 28 | 29 | ## Installation 30 | ### Building from source 31 | 32 | ```shell 33 | $ go get github.com/HeavyHorst/remco/cmd/remco 34 | $ go install github.com/HeavyHorst/remco/cmd/remco 35 | ``` 36 | 37 | You should now have `remco` in your `$GOPATH/bin` directory 38 | 39 | ### Building from the repository 40 | 41 | ```shell 42 | $ git clone https://github.com/HeavyHorst/remco 43 | $ cd remco 44 | $ make 45 | $ ls bin/ 46 | remco 47 | ``` 48 | 49 | ### Building a given release 50 | 51 | ```shell 52 | $ export VERSION=v0.12.2 53 | $ git checkout ${VERSION} 54 | $ make release -j 55 | $ ls bin/ 56 | remco_0.12.2_darwin_amd64.zip remco_0.12.2_linux_amd64.zip remco_0.12.2_windows_amd64.zip remco_darwin remco_linux remco_windows 57 | ``` 58 | 59 | ### Using a pre-built release 60 | 61 | Download the releases and extract the binary. 62 | 63 | ```shell 64 | $ REMCO_VER=0.12.2 65 | $ wget https://github.com/HeavyHorst/remco/releases/download/v${REMCO_VER}/remco_${REMCO_VER}_linux_amd64.zip 66 | $ unzip remco_${REMCO_VER}_linux_amd64.zip 67 | ``` 68 | 69 | Optionally move the binary to your PATH 70 | 71 | ```shell 72 | $ mv remco_linux /usr/local/bin/remco 73 | ``` 74 | 75 | Now you can run the remco command! 76 | 77 | ## Execution 78 | 79 | run remco from local dir, configuration is read as default from `/etc/remco/config` 80 | 81 | Command line params: 82 | 83 | | parameter | description | 84 | | --- | --- | 85 | | -config | path to the configuration file | 86 | | -onetime | flag to one run templating once, overriding "Onetime" flag for all backend resources | 87 | | -version | print version and exit | 88 | 89 | ## Contributing 90 | 91 | See [Contributing](https://github.com/HeavyHorst/remco/blob/master/CONTRIBUTING) for details on submitting patches. 92 | -------------------------------------------------------------------------------- /cmd/remco/config.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2016 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | package main 10 | 11 | import ( 12 | "io/ioutil" 13 | "os" 14 | "path/filepath" 15 | "strings" 16 | 17 | "github.com/BurntSushi/toml" 18 | "github.com/HeavyHorst/remco/pkg/backends" 19 | "github.com/HeavyHorst/remco/pkg/backends/plugin" 20 | "github.com/HeavyHorst/remco/pkg/log" 21 | "github.com/HeavyHorst/remco/pkg/telemetry" 22 | "github.com/HeavyHorst/remco/pkg/template" 23 | "github.com/pkg/errors" 24 | ) 25 | 26 | // BackendConfigs holds every individually backend config. 27 | // The values are filled with data from the configuration file. 28 | type BackendConfigs struct { 29 | Etcd *backends.EtcdConfig 30 | File *backends.FileConfig 31 | Env *backends.EnvConfig 32 | Consul *backends.ConsulConfig 33 | Vault *backends.VaultConfig 34 | Redis *backends.RedisConfig 35 | Zookeeper *backends.ZookeeperConfig 36 | Nats *backends.NatsConfig 37 | Mock *backends.MockConfig 38 | Plugin []plugin.Plugin 39 | } 40 | 41 | func (c *BackendConfigs) Copy() BackendConfigs { 42 | newC := BackendConfigs{} 43 | if c.Etcd != nil { 44 | newC.Etcd = new(backends.EtcdConfig) 45 | *newC.Etcd = *c.Etcd 46 | } 47 | if c.File != nil { 48 | newC.File = new(backends.FileConfig) 49 | *newC.File = *c.File 50 | } 51 | if c.Env != nil { 52 | newC.Env = new(backends.EnvConfig) 53 | *newC.Env = *c.Env 54 | } 55 | if c.Consul != nil { 56 | newC.Consul = new(backends.ConsulConfig) 57 | *newC.Consul = *c.Consul 58 | } 59 | if c.Vault != nil { 60 | newC.Vault = new(backends.VaultConfig) 61 | *newC.Vault = *c.Vault 62 | } 63 | if c.Redis != nil { 64 | newC.Redis = new(backends.RedisConfig) 65 | *newC.Redis = *c.Redis 66 | } 67 | if c.Zookeeper != nil { 68 | newC.Zookeeper = new(backends.ZookeeperConfig) 69 | *newC.Zookeeper = *c.Zookeeper 70 | } 71 | if c.Nats != nil { 72 | newC.Nats = new(backends.NatsConfig) 73 | *newC.Nats = *c.Nats 74 | } 75 | if c.Mock != nil { 76 | newC.Mock = new(backends.MockConfig) 77 | *newC.Mock = *c.Mock 78 | } 79 | return newC 80 | } 81 | 82 | // GetBackends returns a slice with all BackendConfigs for easy iteration. 83 | func (c *BackendConfigs) GetBackends() []template.BackendConnector { 84 | bc := []template.BackendConnector{ 85 | c.Etcd, 86 | c.File, 87 | c.Env, 88 | c.Consul, 89 | c.Vault, 90 | c.Redis, 91 | c.Zookeeper, 92 | c.Mock, 93 | c.Nats, 94 | } 95 | 96 | for _, v := range c.Plugin { 97 | bc = append(bc, &v) 98 | } 99 | 100 | return bc 101 | } 102 | 103 | // Configuration is the representation of an config file 104 | type Configuration struct { 105 | LogLevel string `toml:"log_level"` 106 | LogFormat string `toml:"log_format"` 107 | IncludeDir string `toml:"include_dir"` 108 | FilterDir string `toml:"filter_dir"` 109 | PidFile string `toml:"pid_file"` 110 | LogFile string `toml:"log_file"` 111 | Resource []Resource 112 | Telemetry telemetry.Telemetry 113 | } 114 | 115 | type DefaultBackends struct { 116 | Backends BackendConfigs `toml:"default_backends"` 117 | Resource []Resource 118 | } 119 | 120 | // Resource is the representation of an resource configuration 121 | type Resource struct { 122 | Exec template.ExecConfig 123 | StartCmd string `toml:"start_cmd" json:"start_cmd"` 124 | ReloadCmd string `toml:"reload_cmd" json:"reload_cmd"` 125 | Template []*template.Renderer 126 | Backends BackendConfigs `toml:"backend"` 127 | 128 | // defaults to the filename of the resource 129 | Name string 130 | } 131 | 132 | func readFileAndExpandEnv(path string) ([]byte, error) { 133 | buf, err := ioutil.ReadFile(path) 134 | if err != nil { 135 | return buf, errors.Wrap(err, "read file failed") 136 | } 137 | // expand the environment variables 138 | buf = []byte(os.ExpandEnv(string(buf))) 139 | return buf, nil 140 | } 141 | 142 | // NewConfiguration reads the file at `path`, expand the environment variables 143 | // and unmarshals it to a new configuration struct. 144 | // It returns an error if any. 145 | func NewConfiguration(path string) (Configuration, error) { 146 | var c Configuration 147 | var dbc DefaultBackends 148 | 149 | buf, err := readFileAndExpandEnv(path) 150 | if err != nil { 151 | return c, err 152 | } 153 | 154 | if err := toml.Unmarshal(buf, &dbc); err != nil { 155 | return c, errors.Wrapf(err, "toml unmarshal failed: %s", path) 156 | } 157 | 158 | c.Resource = dbc.Resource 159 | for i := 0; i < len(c.Resource); i++ { 160 | c.Resource[i].Backends = dbc.Backends.Copy() 161 | } 162 | 163 | // Set defaults as in go-metrics DefaultConfig 164 | c.Telemetry.EnableHostname = true 165 | c.Telemetry.EnableRuntimeMetrics = true 166 | 167 | if err := toml.Unmarshal(buf, &c); err != nil { 168 | return c, errors.Wrapf(err, "toml unmarshal failed: %s", path) 169 | } 170 | 171 | for _, v := range c.Resource { 172 | if v.Name == "" { 173 | v.Name = filepath.Base(path) 174 | } 175 | } 176 | 177 | if c.IncludeDir != "" { 178 | files, err := ioutil.ReadDir(c.IncludeDir) 179 | if err != nil { 180 | return c, err 181 | } 182 | for _, file := range files { 183 | if strings.HasSuffix(file.Name(), ".toml") { 184 | fp := filepath.Join(c.IncludeDir, file.Name()) 185 | 186 | log.WithFields( 187 | "path", fp, 188 | ).Info("loading resource configuration") 189 | 190 | buf, err := readFileAndExpandEnv(fp) 191 | if err != nil { 192 | return c, err 193 | } 194 | r := Resource{ 195 | Backends: dbc.Backends.Copy(), 196 | } 197 | if err := toml.Unmarshal(buf, &r); err != nil { 198 | return c, errors.Wrapf(err, "toml unmarshal failed: %s", fp) 199 | } 200 | // don't add empty resources 201 | if len(r.Template) > 0 { 202 | if r.Name == "" { 203 | r.Name = file.Name() 204 | } 205 | c.Resource = append(c.Resource, r) 206 | } 207 | } 208 | } 209 | } 210 | 211 | if c.FilterDir != "" { 212 | if err := template.RegisterCustomJsFilters(c.FilterDir); err != nil { 213 | return c, err 214 | } 215 | } 216 | 217 | c.configureLogger() 218 | 219 | return c, nil 220 | } 221 | 222 | // configureLogger configures the global logger. 223 | // It sets the log level and log formatting. 224 | func (c *Configuration) configureLogger() { 225 | log.InitializeLogging(c.LogFormat, c.LogLevel) 226 | } 227 | -------------------------------------------------------------------------------- /cmd/remco/config_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2016 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | package main 10 | 11 | import ( 12 | "io/ioutil" 13 | "os" 14 | "testing" 15 | 16 | "github.com/HeavyHorst/remco/pkg/backends" 17 | "github.com/HeavyHorst/remco/pkg/telemetry" 18 | "github.com/HeavyHorst/remco/pkg/template" 19 | 20 | . "gopkg.in/check.v1" 21 | ) 22 | 23 | const ( 24 | testFile string = ` 25 | log_level = "debug" 26 | log_format = "text" 27 | include_dir = "/tmp/resource.d/" 28 | 29 | [default_backends] 30 | [default_backends.mock] 31 | onetime = false 32 | prefix = "Hallo" 33 | 34 | 35 | [[resource]] 36 | name = "haproxy" 37 | [[resource.template]] 38 | src = "/tmp/test12345.tmpl" 39 | dst = "/tmp/test12345.cfg" 40 | checkCmd = "" 41 | reloadCmd = "" 42 | mode = "0644" 43 | [resource.backend] 44 | [resource.backend.mock] 45 | keys = ["/"] 46 | watchKeys = ["/"] 47 | watch = false 48 | interval = 1 49 | 50 | [telemetry] 51 | enabled = true 52 | enable_hostname = false 53 | enable_hostname_label = false 54 | enable_runtime_metrics = false 55 | [telemetry.sinks.prometheus] 56 | addr = ":2112" 57 | expiration = 600 58 | ` 59 | resourceFile string = ` 60 | [[template]] 61 | src = "/tmp/test12345.tmpl" 62 | dst = "/tmp/test12345.cfg" 63 | checkCmd = "" 64 | reloadCmd = "" 65 | mode = "0644" 66 | [backend] 67 | [backend.mock] 68 | keys = ["/"] 69 | watchKeys = ["/"] 70 | watch = false 71 | interval = 1 72 | ` 73 | ) 74 | 75 | var expectedTemplates = []*template.Renderer{ 76 | { 77 | Src: "/tmp/test12345.tmpl", 78 | Dst: "/tmp/test12345.cfg", 79 | Mode: "0644", 80 | }, 81 | } 82 | 83 | var expectedBackend = BackendConfigs{ 84 | Mock: &backends.MockConfig{ 85 | Backend: template.Backend{ 86 | Watch: false, 87 | Keys: []string{"/"}, 88 | WatchKeys: []string{"/"}, 89 | Interval: 1, 90 | Onetime: false, 91 | Prefix: "Hallo", 92 | }, 93 | }, 94 | } 95 | 96 | var expected = Configuration{ 97 | LogLevel: "debug", 98 | LogFormat: "text", 99 | IncludeDir: "/tmp/resource.d/", 100 | Resource: []Resource{ 101 | { 102 | Name: "haproxy", 103 | Template: expectedTemplates, 104 | Backends: expectedBackend, 105 | }, 106 | { 107 | Name: "test.toml", 108 | Template: expectedTemplates, 109 | Backends: expectedBackend, 110 | }, 111 | }, 112 | Telemetry: telemetry.Telemetry{ 113 | Enabled: true, 114 | ServiceName: "", 115 | HostName: "", 116 | EnableHostname: false, 117 | EnableHostnameLabel: false, 118 | EnableRuntimeMetrics: false, 119 | Sinks: telemetry.Sinks{ 120 | Prometheus: &telemetry.PrometheusSink{ 121 | Addr: ":2112", 122 | Expiration: 600, 123 | }, 124 | }, 125 | }, 126 | } 127 | 128 | // Hook up gocheck into the "go test" runner. 129 | func Test(t *testing.T) { TestingT(t) } 130 | 131 | type FilterSuite struct { 132 | cfgPath string 133 | } 134 | 135 | var _ = Suite(&FilterSuite{}) 136 | 137 | func (s *FilterSuite) SetUpSuite(t *C) { 138 | err := os.Mkdir("/tmp/resource.d", 0755) 139 | if err != nil { 140 | t.Error(err) 141 | } 142 | 143 | err = ioutil.WriteFile("/tmp/resource.d/test.toml", []byte(resourceFile), 0644) 144 | if err != nil { 145 | t.Error(err) 146 | } 147 | 148 | f3, err := ioutil.TempFile("/tmp", "") 149 | if err != nil { 150 | t.Error(err) 151 | } 152 | defer f3.Close() 153 | _, err = f3.WriteString(testFile) 154 | if err != nil { 155 | t.Error(err) 156 | } 157 | s.cfgPath = f3.Name() 158 | } 159 | 160 | func (s *FilterSuite) TearDownSuite(t *C) { 161 | err := os.Remove(s.cfgPath) 162 | if err != nil { 163 | t.Log(err) 164 | } 165 | err = os.RemoveAll("/tmp/resource.d") 166 | if err != nil { 167 | t.Log(err) 168 | } 169 | } 170 | 171 | func (s *FilterSuite) TestNewConf(t *C) { 172 | cfg, err := NewConfiguration(s.cfgPath) 173 | if err != nil { 174 | t.Error(err) 175 | } 176 | t.Check(cfg, DeepEquals, expected) 177 | } 178 | -------------------------------------------------------------------------------- /cmd/remco/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2016 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | package main 10 | 11 | import ( 12 | "flag" 13 | "fmt" 14 | "os" 15 | "os/signal" 16 | "reflect" 17 | "sync" 18 | "syscall" 19 | 20 | "github.com/HeavyHorst/remco/pkg/log" 21 | "github.com/hashicorp/consul-template/signals" 22 | "github.com/hashicorp/go-reap" 23 | ) 24 | 25 | var ( 26 | configPath string 27 | printVersionAndExit bool 28 | onetime bool 29 | ) 30 | 31 | func init() { 32 | const defaultConfig = "/etc/remco/config" 33 | flag.StringVar(&configPath, "config", defaultConfig, "path to the configuration file") 34 | flag.BoolVar(&printVersionAndExit, "version", false, "print version and exit") 35 | flag.BoolVar(&onetime, "onetime", false, "run templating process once and exit") 36 | } 37 | 38 | func run() int32 { 39 | // catch all signals 40 | signalChan := make(chan os.Signal, 1) 41 | signal.Notify(signalChan) 42 | 43 | done := make(chan struct{}) 44 | reapLock := &sync.RWMutex{} 45 | 46 | cfg, err := NewConfiguration(configPath) 47 | if err != nil { 48 | log.Fatal("failed to read config", err) 49 | } 50 | 51 | if onetime { 52 | for _, res := range cfg.Resource { 53 | for _, b := range res.Backends.GetBackends() { 54 | if !reflect.ValueOf(b).IsZero() { 55 | backend := b.GetBackend() 56 | backend.Onetime = true 57 | } 58 | } 59 | } 60 | } 61 | 62 | run := NewSupervisor(cfg, reapLock, done) 63 | defer run.Stop() 64 | 65 | // reap zombies if pid is 1 66 | pidReapChan := make(reap.PidCh, 1) 67 | errorReapChan := make(reap.ErrorCh, 1) 68 | if os.Getpid() == 1 { 69 | if !reap.IsSupported() { 70 | log.Warning("the pid is 1 but zombie reaping is not supported on this platform") 71 | } else { 72 | go reap.ReapChildren(pidReapChan, errorReapChan, done, reapLock) 73 | } 74 | } 75 | 76 | for { 77 | select { 78 | case s := <-signalChan: 79 | switch s { 80 | case syscall.SIGHUP: 81 | log.WithFields( 82 | "file", configPath, 83 | ).Info("loading new config") 84 | newConf, err := NewConfiguration(configPath) 85 | if err != nil { 86 | log.Error("failed to read config", err) 87 | continue 88 | } 89 | run.Reload(newConf) 90 | case signals.SignalLookup["SIGCHLD"]: 91 | case os.Interrupt, syscall.SIGTERM: 92 | log.Info(fmt.Sprintf("Captured %v. Exiting...", s)) 93 | return 0 94 | default: 95 | run.SendSignal(s) 96 | } 97 | case pid := <-pidReapChan: 98 | log.Debug(fmt.Sprintf("Reaped child process %d", pid)) 99 | case err := <-errorReapChan: 100 | log.Error(fmt.Sprintf("Error reaping child process %v", err)) 101 | case <-done: 102 | return run.getNumResourceErrors() 103 | } 104 | } 105 | } 106 | 107 | func main() { 108 | flag.Parse() 109 | 110 | if printVersionAndExit { 111 | printVersion() 112 | return 113 | } 114 | rc := run() 115 | // be on the safe side for portability and lots of backend resources 116 | if rc > 125 { 117 | rc = 125 118 | } 119 | os.Exit(int(rc)) 120 | } 121 | -------------------------------------------------------------------------------- /cmd/remco/supervisor.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2016 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | package main 10 | 11 | import ( 12 | "context" 13 | "fmt" 14 | "io/ioutil" 15 | "math/rand" 16 | "os" 17 | "sync" 18 | "sync/atomic" 19 | "time" 20 | 21 | "github.com/HeavyHorst/remco/pkg/log" 22 | "github.com/HeavyHorst/remco/pkg/telemetry" 23 | "github.com/HeavyHorst/remco/pkg/template" 24 | "github.com/pborman/uuid" 25 | "github.com/pkg/errors" 26 | ) 27 | 28 | type reloadSignal struct { 29 | c Configuration 30 | reloaded chan<- struct{} 31 | } 32 | 33 | // Supervisor runs 34 | type Supervisor struct { 35 | stopChan chan struct{} 36 | reloadChan chan reloadSignal 37 | wg sync.WaitGroup 38 | 39 | signalChans map[string]chan os.Signal 40 | signalChansMutex sync.RWMutex 41 | 42 | pidFile string 43 | telemetry telemetry.Telemetry 44 | 45 | reapLock *sync.RWMutex 46 | 47 | resourcesWithError int32 48 | } 49 | 50 | // NewSupervisor creates a new Supervisor 51 | func NewSupervisor(cfg Configuration, reapLock *sync.RWMutex, done chan struct{}) *Supervisor { 52 | w := &Supervisor{ 53 | stopChan: make(chan struct{}), 54 | reloadChan: make(chan reloadSignal), 55 | signalChans: make(map[string]chan os.Signal), 56 | reapLock: reapLock, 57 | } 58 | 59 | w.pidFile = cfg.PidFile 60 | w.telemetry = cfg.Telemetry 61 | pid := os.Getpid() 62 | err := w.writePid(pid) 63 | if err != nil { 64 | log.WithFields("pid_file", w.pidFile).Error("failed to write pidfile", err) 65 | } 66 | 67 | stopChan := make(chan struct{}) 68 | stoppedChan := make(chan struct{}) 69 | 70 | _, err = w.telemetry.Init() 71 | if err != nil { 72 | log.Error(fmt.Sprintf("error starting telemetry: %v", err)) 73 | } 74 | go w.runResource(cfg.Resource, stopChan, stoppedChan) 75 | w.wg.Add(1) 76 | go func() { 77 | defer w.wg.Done() 78 | // close the done channel 79 | // this signals the main function that all work is done. 80 | // for example all backends are configured with onetime=true 81 | defer close(done) 82 | for { 83 | select { 84 | case rs := <-w.reloadChan: 85 | // write a new pidfile if the pid filepath has changed 86 | if rs.c.PidFile != w.pidFile { 87 | err := w.deletePid() 88 | if err != nil { 89 | log.WithFields("pid_file", w.pidFile).Error("failed to delete pidfile", err) 90 | } 91 | w.pidFile = rs.c.PidFile 92 | err = w.writePid(pid) 93 | if err != nil { 94 | log.WithFields("pid_file", w.pidFile).Error("failed to write pidfile", err) 95 | } 96 | } 97 | err = w.telemetry.Stop() 98 | if err != nil { 99 | log.Error(fmt.Sprintf("error stopping telemetry: %v", err)) 100 | } 101 | _, err = rs.c.Telemetry.Init() 102 | if err != nil { 103 | log.Error(fmt.Sprintf("error starting telemetry: %v", err)) 104 | } 105 | stopChan <- struct{}{} 106 | <-stoppedChan 107 | go w.runResource(rs.c.Resource, stopChan, stoppedChan) 108 | rs.reloaded <- struct{}{} 109 | case <-stoppedChan: 110 | return 111 | case <-w.stopChan: 112 | stopChan <- struct{}{} 113 | <-stoppedChan 114 | return 115 | } 116 | } 117 | }() 118 | 119 | return w 120 | } 121 | 122 | func (ru *Supervisor) getNumResourceErrors() int32 { 123 | return atomic.LoadInt32(&ru.resourcesWithError) 124 | } 125 | 126 | func (ru *Supervisor) incResourceError() { 127 | atomic.AddInt32(&ru.resourcesWithError, 1) 128 | } 129 | 130 | func (ru *Supervisor) writePid(pid int) error { 131 | if ru.pidFile == "" { 132 | return nil 133 | } 134 | 135 | log.Info(fmt.Sprintf("creating pid file at %q", ru.pidFile)) 136 | 137 | err := ioutil.WriteFile(ru.pidFile, []byte(fmt.Sprintf("%d", pid)), 0666) 138 | if err != nil { 139 | return errors.Wrap(err, "couldn't write pid file") 140 | } 141 | return nil 142 | } 143 | 144 | func (ru *Supervisor) deletePid() error { 145 | if ru.pidFile == "" { 146 | return nil 147 | } 148 | 149 | log.Debug(fmt.Sprintf("removing pid file at %q", ru.pidFile)) 150 | 151 | stat, err := os.Stat(ru.pidFile) 152 | if err != nil { 153 | return errors.Wrap(err, "couldn't get file stats") 154 | } 155 | 156 | if stat.IsDir() { 157 | return fmt.Errorf("the pid file path seems to be a directory") 158 | } 159 | 160 | return os.Remove(ru.pidFile) 161 | } 162 | 163 | func (ru *Supervisor) addSignalChan(id string, sigchan chan os.Signal) { 164 | ru.signalChansMutex.Lock() 165 | defer ru.signalChansMutex.Unlock() 166 | ru.signalChans[id] = sigchan 167 | } 168 | 169 | func (ru *Supervisor) removeSignalChan(id string) { 170 | ru.signalChansMutex.Lock() 171 | defer ru.signalChansMutex.Unlock() 172 | delete(ru.signalChans, id) 173 | } 174 | 175 | // SendSignal forwards the given Signal to all child processes 176 | func (ru *Supervisor) SendSignal(s os.Signal) { 177 | ru.signalChansMutex.RLock() 178 | defer ru.signalChansMutex.RUnlock() 179 | // try to send the signal to all child processes 180 | // we don't block here if the signal can't be send 181 | for _, v := range ru.signalChans { 182 | select { 183 | case v <- s: 184 | default: 185 | } 186 | } 187 | } 188 | 189 | func (ru *Supervisor) runResource(r []Resource, stop, stopped chan struct{}) { 190 | defer func() { 191 | if stopped != nil { 192 | stopped <- struct{}{} 193 | } 194 | }() 195 | 196 | ctx, cancel := context.WithCancel(context.Background()) 197 | defer cancel() 198 | done := make(chan struct{}) 199 | 200 | wait := sync.WaitGroup{} 201 | for _, v := range r { 202 | wait.Add(1) 203 | go func(r Resource) { 204 | defer wait.Done() 205 | 206 | rsc := template.ResourceConfig{ 207 | Exec: r.Exec, 208 | Template: r.Template, 209 | Name: r.Name, 210 | StartCmd: r.StartCmd, 211 | ReloadCmd: r.ReloadCmd, 212 | Connectors: r.Backends.GetBackends(), 213 | } 214 | res, err := template.NewResourceFromResourceConfig(ctx, ru.reapLock, rsc) 215 | if err != nil { 216 | log.Error("failed to create new resource", err) 217 | ru.incResourceError() 218 | return 219 | } 220 | defer res.Close() 221 | 222 | id := uuid.New() 223 | ru.addSignalChan(id, res.SignalChan) 224 | defer ru.removeSignalChan(id) 225 | 226 | restartChan := make(chan struct{}, 1) 227 | restartChan <- struct{}{} 228 | 229 | for { 230 | select { 231 | case <-ctx.Done(): 232 | return 233 | case <-restartChan: 234 | res.Monitor(ctx) 235 | if res.Failed && res.OnetimeOnly { 236 | ru.incResourceError() 237 | return 238 | } else if res.Failed { 239 | go func() { 240 | // try to restart the resource after a random amount of time 241 | rn := rand.Int63n(30) 242 | log.WithFields( 243 | "resource", r.Name, 244 | "restartDelay", rn, 245 | ).Error("resource execution failed, restarting after delay") 246 | time.Sleep(time.Duration(rn) * time.Second) 247 | select { 248 | case <-ctx.Done(): 249 | return 250 | default: 251 | restartChan <- struct{}{} 252 | } 253 | }() 254 | } else { 255 | return 256 | } 257 | } 258 | } 259 | }(v) 260 | } 261 | 262 | go func() { 263 | // If there is no goroutine left - quit 264 | // this is necessary for the onetime mode 265 | wait.Wait() 266 | close(done) 267 | }() 268 | 269 | for { 270 | select { 271 | case <-stop: 272 | cancel() 273 | wait.Wait() 274 | return 275 | case <-done: 276 | return 277 | } 278 | } 279 | } 280 | 281 | // Reload with the new configuration. 282 | func (ru *Supervisor) Reload(cfg Configuration) { 283 | reloaded := make(chan struct{}) 284 | ru.reloadChan <- reloadSignal{ 285 | c: cfg, 286 | reloaded: reloaded, 287 | } 288 | <-reloaded 289 | } 290 | 291 | // Stop stops the Supervisor gracefully. 292 | func (ru *Supervisor) Stop() int32 { 293 | close(ru.stopChan) 294 | // wait for the main routine to exit 295 | ru.wg.Wait() 296 | 297 | // remove the pidfile 298 | err := ru.deletePid() 299 | if err != nil { 300 | log.WithFields("pid_file", ru.pidFile).Error("failed to delete pidfile", err) 301 | } 302 | return ru.getNumResourceErrors() 303 | } 304 | -------------------------------------------------------------------------------- /cmd/remco/supervisor_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2016 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | package main 10 | 11 | import ( 12 | "os" 13 | 14 | "github.com/HeavyHorst/remco/pkg/backends" 15 | "github.com/HeavyHorst/remco/pkg/telemetry" 16 | "github.com/HeavyHorst/remco/pkg/template" 17 | 18 | . "gopkg.in/check.v1" 19 | ) 20 | 21 | var exampleTemplates = []*template.Renderer{ 22 | { 23 | Src: "/tmp/test12345.tmpl", 24 | Dst: "/tmp/test12345.cfg", 25 | Mode: "0644", 26 | }, 27 | } 28 | 29 | var exampleBackend = BackendConfigs{ 30 | Mock: &backends.MockConfig{ 31 | Backend: template.Backend{ 32 | Watch: false, 33 | Keys: []string{"/"}, 34 | Interval: 1, 35 | Onetime: false, 36 | }, 37 | }, 38 | } 39 | 40 | var exampleConfiguration = Configuration{ 41 | LogLevel: "debug", 42 | LogFormat: "text", 43 | IncludeDir: "/tmp/resource.d/", 44 | PidFile: "/tmp/remco_test.pid", 45 | Resource: []Resource{ 46 | { 47 | Name: "test.toml", 48 | Template: exampleTemplates, 49 | Backends: exampleBackend, 50 | }, 51 | }, 52 | Telemetry: telemetry.Telemetry{ 53 | Enabled: true, 54 | ServiceName: "test", 55 | Sinks: telemetry.Sinks{}, 56 | }, 57 | } 58 | 59 | type RunnerTestSuite struct { 60 | runner *Supervisor 61 | } 62 | 63 | var _ = Suite(&RunnerTestSuite{}) 64 | 65 | func (s *RunnerTestSuite) SetUpSuite(t *C) { 66 | s.runner = NewSupervisor(exampleConfiguration, nil, make(chan struct{})) 67 | } 68 | 69 | func (s *RunnerTestSuite) TestNew(t *C) { 70 | t.Check(s.runner.stopChan, NotNil) 71 | t.Check(s.runner.reloadChan, NotNil) 72 | t.Check(s.runner.signalChans, NotNil) 73 | t.Check(s.runner.reapLock, IsNil) 74 | t.Check(s.runner.pidFile, Equals, "/tmp/remco_test.pid") 75 | t.Check(s.runner.telemetry, DeepEquals, exampleConfiguration.Telemetry) 76 | } 77 | 78 | func (s *RunnerTestSuite) TestWritePid(t *C) { 79 | err := s.runner.writePid(os.Getpid()) 80 | t.Check(err, IsNil) 81 | } 82 | 83 | func (s *RunnerTestSuite) TestDeletePid(t *C) { 84 | err := s.runner.deletePid() 85 | t.Check(err, IsNil) 86 | } 87 | 88 | func (s *RunnerTestSuite) TestSignalChan(t *C) { 89 | c := make(chan os.Signal, 1) 90 | s.runner.addSignalChan("id", c) 91 | s.runner.SendSignal(os.Interrupt) 92 | t.Check(<-c, Equals, os.Interrupt) 93 | 94 | // channel is full, should not block 95 | c <- os.Interrupt 96 | s.runner.SendSignal(os.Interrupt) 97 | 98 | s.runner.removeSignalChan("id") 99 | } 100 | 101 | func (s *RunnerTestSuite) TestReload(t *C) { 102 | new := exampleConfiguration 103 | new.PidFile = "/tmp/remco_test2.pid" 104 | new.Telemetry.ServiceName = "test2" 105 | s.runner.Reload(new) 106 | } 107 | 108 | func (s *RunnerTestSuite) TearDownSuite(t *C) { 109 | s.runner.Stop() 110 | t.Check(s.runner.signalChans, HasLen, 0) 111 | } 112 | -------------------------------------------------------------------------------- /cmd/remco/version.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2016 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | package main 10 | 11 | import ( 12 | "fmt" 13 | "runtime" 14 | ) 15 | 16 | // values set with linker flags 17 | // don't you dare modifying this values! 18 | var version string 19 | var buildDate string 20 | var commit string 21 | 22 | func printVersion() { 23 | fmt.Println("remco Version: " + version) 24 | fmt.Println("UTC Build Time: " + buildDate) 25 | fmt.Println("Git Commit Hash: " + commit) 26 | fmt.Println("Go Version: " + runtime.Version()) 27 | fmt.Printf("Go OS/Arch: %s/%s\n", runtime.GOOS, runtime.GOARCH) 28 | } 29 | -------------------------------------------------------------------------------- /docs/config.toml: -------------------------------------------------------------------------------- 1 | baseurl = "https://heavyhorst.github.io/remco/" 2 | languageCode = "en-us" 3 | title = "Remco Docs" 4 | theme = "hugo-theme-learn" 5 | metadataformat = "yaml" 6 | canonifyurls = true 7 | relativeURLs = true 8 | 9 | [params] 10 | menu = ["details", "config", "template", "plugins", "examples"] 11 | 12 | # General information 13 | author = "The remco authors" 14 | description = "remco is a lightweight configuration management tool" 15 | copyright = "Released under the MIT license" 16 | 17 | # Repository 18 | provider = "GitHub" 19 | repo_url = "https://github.com/HeavyHorst/remco" 20 | 21 | version = "v0.8.0" 22 | logo = "images/logo.png" 23 | favicon = "" 24 | 25 | permalink = "#" 26 | 27 | # Custom assets 28 | custom_css = [] 29 | custom_js = [] 30 | 31 | # Syntax highlighting theme 32 | highlight_css = "" 33 | 34 | [params.palette] 35 | primary = "teal" 36 | accent = "green" 37 | 38 | [params.font] 39 | text = "Ubuntu" 40 | code = "Ubuntu Mono" 41 | 42 | 43 | [social] 44 | twitter = "" 45 | github = "heavyhorst" 46 | 47 | [blackfriday] 48 | smartypants = true 49 | fractions = true 50 | smartDashes = true 51 | plainIDAnchors = true 52 | -------------------------------------------------------------------------------- /docs/content/config/configuration-options.md: -------------------------------------------------------------------------------- 1 | --- 2 | date: 2016-12-03T14:57:13+01:00 3 | next: /config/sample-config/ 4 | prev: /config/environment-variables/ 5 | title: configuration options 6 | toc: true 7 | weight: 10 8 | --- 9 | 10 | 11 | ## Global configuration options 12 | - **log_level(string):** 13 | - Valid levels are panic, fatal, error, warn, info and debug. Default is info. 14 | - **log_format(string):** 15 | - The format of the log messages. Valid formats are *text* and *json*. 16 | - **include_dir(string):** 17 | - Specify an entire directory of resource configuration files to include. Data from files will be imported directly into `resource` array. 18 | - **filter_dir(string):** 19 | - A folder with custom JavaScript template filters. 20 | - **pid_file(string):** 21 | - A filename to write the process-id to. 22 | - **log_file(string):** 23 | - Specify the log file name. The empty string means to log to stdout. 24 | 25 | ## Resource configuration options 26 | - **name(string, optional):** 27 | - You can give the resource a name which is added to the logs as field *resource*. Default is the name of the resource file. 28 | - **start_cmd(string, optional)** 29 | - An optional command which is executed once all templates have been processed successfully. 30 | - **reload_cmd(string, optional)** 31 | - An optional command which is executed as soon as a template belonging to the resource has been successfully recreated. 32 | 33 | ## Exec configuration options 34 | - **command(string):** 35 | - This is the command to exec as a child process. Note that the child process must remain in the foreground. 36 | - **kill_signal(string):** 37 | - This defines the signal sent to the child process when remco is gracefully shutting down. The application needs to exit before the `kill_timeout`, 38 | it will be terminated otherwise (like kill -9). The default value is "SIGTERM". 39 | - **kill_timeout(int):** 40 | - the maximum amount of time (seconds) to wait for the child process to gracefully terminate. Default is 10. 41 | - **reload_signal(string):** 42 | - This defines the signal sent to the child process when some configuration data is changed. If no signal is specified the child process will be killed (gracefully) and started again. 43 | - **splay(int):** 44 | - A random splay to wait before killing the command. May be useful in large clusters to prevent all child processes to reload at the same time when configuration changes occur. Default is 0. 45 | 46 | ## Template configuration options 47 | - **src(string):** 48 | - The path of the template that will be used to render the application's configuration file. 49 | - **dst(string):** 50 | - The location to place the rendered configuration file. 51 | - **make_directories(bool, optional):** 52 | - make parent directories for the dst path as needed. Default is false. 53 | - **check_cmd(string, optional):** 54 | - An optional command to check the rendered source template before writing it to the destination. If this command returns non-zero, the destination will not be overwritten by the rendered source template. We can use `{{.src}}` here to reference the rendered source template. 55 | - **reload_cmd(string, optional):** 56 | - An optional command to run after the destination is updated. We can use `{{.dst}}` here to reference the destination. 57 | - **mode(string, optional):** 58 | - The permission mode of the file. Default is "0644". 59 | - **UID(int, optional):** 60 | - The UID that should own the file. Defaults to the effective uid. 61 | - **GID(int, optional):** 62 | - The GID that should own the file. Defaults to the effective gid. 63 | 64 | ## Backend configuration options 65 | 66 | See the example configuration to see how global default values can be set for individual backends. 67 | 68 |
69 | **valid in every backend** 70 | 71 | - **keys([]string):** 72 | - The backend keys that the template requires to be rendered correctly. The child keys are also loaded. 73 | - **watch(bool, optional):** 74 | - Enable watch support. Default is false. 75 | - **prefix(string, optional):** 76 | - Key path prefix. Default is "". 77 | - **watchKeys([]string, optional):** 78 | - Keys list to watch. Default is same as keys 79 | - **interval(int, optional):** 80 | - The backend polling interval. Can be used as a reconciliation loop for watch or standalone. 81 | - **onetime(bool, optional):** 82 | - Render the config file and quit. Default is false. 83 |
84 | 85 |
86 | **etcd** 87 | 88 | - **nodes([]string):** 89 | - List of backend nodes. 90 | - **srv_record(string, optional):** 91 | - A DNS server record to discover the etcd nodes. 92 | - **scheme(string, optional):** 93 | - The backend URI scheme (http or https). This is only used when the nodes are discovered via DNS srv records and the api level is 2. Default is http. 94 | - **client_cert(string, optional):** 95 | - The client cert file. 96 | - **client_key(string, optional):** 97 | - The client key file. 98 | - **client_ca_keys(string, optional):** 99 | - The client CA key file. 100 | - **username(string, optional):** 101 | - The username for the basic_auth authentication. 102 | - **password(string, optional):** 103 | - The password for the basic_auth authentication. 104 | - **version(uint, optional):** 105 | - The etcd api-level to use (2 or 3). Default is 2. 106 |
107 | 108 |
109 | **nats** 110 | 111 | - **nodes([]string, optional):** 112 | - List of backend nodes. If none is provided nats.DefaultURL is used. 113 | - **bucket(string):** 114 | - The nats kv bucket where your config keys are stored 115 | - **username(string, optional):** 116 | - The username for the basic_auth authentication. 117 | - **password(string, optional):** 118 | - The password for the basic_auth authentication. 119 | - **token(string, optional):** 120 | - The athentication token for the nats server 121 | - **creds(string, optional):** 122 | - The path to an NATS 2.0 and NATS NGS compatible user credentials file 123 |
124 | 125 |
126 | **consul** 127 | 128 | - **nodes([]string):** 129 | - List of backend nodes. 130 | - **srv_record(string, optional):** 131 | - A DNS server record to discover the consul nodes. 132 | - **scheme(string):** 133 | - The backend URI scheme (http or https). 134 | - **client_cert(string, optional):** 135 | - The client cert file. 136 | - **client_key(string, optional):** 137 | - The client key file. 138 | - **client_ca_keys(string, optional):** 139 | - The client CA key file. 140 |
141 | 142 |
143 | **file** 144 | 145 | - **filepath(string):** 146 | - The filepath to a yaml or json file containing the key-value pairs. This can be a local file or a remote http/https location. 147 | - **httpheaders(map[string]string):** 148 | - Optional HTTP-headers to append to the request if the file path is a remote http/https location. 149 |
150 | 151 |
152 | **redis** 153 | 154 | - **nodes([]string):** 155 | - List of backend nodes. 156 | - **srv_record(string), optional:** 157 | - A DNS server record to discover the redis nodes. 158 | - **password(string, optional):** 159 | - The redis password. 160 | - **database(int, optional):** 161 | - The redis database. 162 |
163 | 164 |
165 | **vault** 166 | 167 | - **node(string):** 168 | - The backend node. 169 | - **auth_type(string):** 170 | - The vault authentication type. (token, approle, app-id, userpass, github, cert, kubernetes) 171 | - **auth_token(string):** 172 | - The vault authentication token. Only used with auth_type=token or github. 173 | - **role_id(string):** 174 | - The vault app role. Only used with auth_type=approle and kubernetes. 175 | - **secret_id(string):** 176 | - The vault secret id. Only used with auth_type=approle. 177 | - **app_id(string):** 178 | - The vault app ID. Only used with auth_type=app-id. 179 | - **user_id(string):** 180 | - The vault user ID. Only used with auth_type=app-id. 181 | - **username(string):** 182 | - The username for the userpass authentication. 183 | - **password(string):** 184 | - The password for the userpass authentication. 185 | - **client_cert(string, optional):** 186 | - The client cert file. 187 | - **client_key(string, optional):** 188 | - The client key file. 189 | - **client_ca_keys(string, optional):** 190 | - The client CA key file. 191 |
192 | 193 | 194 |
195 | **env** 196 |
197 | 198 |
199 | **zookeeper** 200 | 201 | - **nodes([]string):** 202 | - List of backend nodes. 203 | - **srv_record(string, optional):** 204 | - A DNS server record to discover the zookeeper nodes. 205 |
206 | 207 | ## Telemetry configuration options 208 | - **enabled(bool):** 209 | - Flag to enable telemetry. 210 | - **service_name(string):** 211 | - Service name to add to every metric name. "remco" by default 212 | - **hostname(string):** 213 | - Hostname to use. If not provided and enable_hostname, it will be os.Hostname 214 | - **enable_hostname(bool):** 215 | - Enable prefixing gauge values with hostname. `true` by default 216 | - **enable_hostname_label(bool):** 217 | - Put hostname into label instead of metric name. `false` by default 218 | - **enable_runtime_metrics(bool):** 219 | - Enables profiling of runtime metrics (GC, Goroutines, Memory). `true` by default 220 | ## Sink configuration options 221 | 222 |
223 | **inmem** 224 | 225 | - **interval(int):** 226 | - How long is each aggregation interval (seconds). 227 | - **retain(int):** 228 | - Retain controls how many metrics interval we keep. 229 |
230 | 231 |
232 | **prometheus** 233 | 234 | - **addr(string):** 235 | - Address to expose metrics on. Prometheus stats will be available at /metrics endpoint. 236 | - **expiration(int):** 237 | - Expiration is the duration a metric is valid for, after which it will be untracked. If the value is zero, a metric is never expired. 238 | 239 | {{% notice note %}} 240 | If you are using only prometheus sink you may want to disable runtime metrics with **enable_runtime_metrics** option, 241 | because they will duplicate prometheus builtin runtime metrics reporting. Also, consider using **enable_hostname_label** 242 | to put hostname in gauge metrics to label instead of metric name. 243 | {{% /notice %}} 244 |
245 | 246 |
247 | **statsd** 248 | 249 | - **addr(string):** 250 | - Statsd/Statsite server address 251 |
252 | 253 |
254 | **statsite** 255 | 256 | - **addr(string):** 257 | - Statsd/Statsite server address 258 |
259 | -------------------------------------------------------------------------------- /docs/content/config/environment-variables.md: -------------------------------------------------------------------------------- 1 | --- 2 | date: 2016-12-03T14:56:09+01:00 3 | next: /config/configuration-options/ 4 | prev: /config/ 5 | title: environment variables 6 | toc: true 7 | weight: 5 8 | --- 9 | 10 | If you wish to use environmental variables in your config files as a way 11 | to configure values, you can simply use $VARIABLE_NAME or ${VARIABLE_NAME} and the text will be replaced with the value of the environmental variable VARIABLE_NAME. 12 | -------------------------------------------------------------------------------- /docs/content/config/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | chapter: true 3 | date: 2016-12-03T14:37:29+01:00 4 | icon: 2. 5 | next: /config/environment-variables/ 6 | prev: /details/telemetry/ 7 | title: Configuration 8 | weight: 0 9 | --- 10 | 11 | ### Chapter 2 12 | 13 | # Configuration 14 | 15 | The configuration file is in TOML format.
16 | TOML looks very similar to INI configuration formats, but with slightly more rich data structures and nesting support. 17 | -------------------------------------------------------------------------------- /docs/content/config/sample-config.md: -------------------------------------------------------------------------------- 1 | --- 2 | date: 2016-12-03T14:59:49+01:00 3 | next: /config/sample-resource/ 4 | prev: /config/configuration-options/ 5 | title: sample config file 6 | toc: true 7 | weight: 15 8 | --- 9 | 10 | ``` 11 | #remco.toml 12 | ################################################################ 13 | # Global configuration 14 | ################################################################ 15 | log_level = "debug" 16 | log_format = "json" 17 | include_dir = "/etc/remco/resource.d/" 18 | pid_file = "/var/run/remco/remco.pid" 19 | log_file = "/var/log/remco.log" 20 | 21 | # default backend configurations. 22 | # these settings can be overwritten in the individual resource backend settings. 23 | [default_backends] 24 | [default_backends.file] 25 | onetime = true 26 | prefix = "/bla" 27 | 28 | ################################################################ 29 | # Resource configuration 30 | ################################################################ 31 | [[resource]] 32 | name = "haproxy" 33 | start_cmd = "echo 1" 34 | reload_cmd = "echo 1" 35 | [[resource.template]] 36 | src = "/etc/remco/templates/haproxy.cfg" 37 | dst = "/etc/haproxy/haproxy.cfg" 38 | check_cmd = "somecommand" 39 | reload_cmd = "somecommand" 40 | mode = "0644" 41 | 42 | [resource.backend] 43 | # you can use as many backends as you like 44 | # in this example vault and file 45 | [resource.backend.vault] 46 | node = "http://127.0.0.1:8200" 47 | ## Token based auth backend 48 | auth_type = "token" 49 | auth_token = "vault_token" 50 | ## AppID based auth backend 51 | # auth_type = "app-id" 52 | # app_id = "vault_app_id" 53 | # user_id = "vault_user_id" 54 | ## userpass based auth backend 55 | # auth_type = "userpass" 56 | # username = "username" 57 | # password = "password" 58 | client_cert = "/path/to/client_cert" 59 | client_key = "/path/to/client_key" 60 | client_ca_keys = "/path/to/client_ca_keys" 61 | 62 | # These values are valid in every backend 63 | watch = true 64 | prefix = "/" 65 | onetime = true 66 | interval = 1 67 | keys = ["/"] 68 | watchKeys = ["/haproxy/reload"] 69 | 70 | [resource.backend.file] 71 | httpheader = { X-Test-Token = "XXX", X-Test-Token2 = "YYY" } 72 | filepath = "/etc/remco/test.yml" 73 | watch = true 74 | keys = ["/prefix"] 75 | 76 | ################################################################ 77 | # Telemetry configuration 78 | ################################################################ 79 | [telemetry] 80 | enabled = true 81 | [telemetry.sinks.prometheus] 82 | addr = ":2112" 83 | expiration = 600 84 | ``` 85 | 86 | -------------------------------------------------------------------------------- /docs/content/config/sample-resource.md: -------------------------------------------------------------------------------- 1 | --- 2 | date: 2016-12-03T15:01:23+01:00 3 | next: /template/ 4 | prev: /config/sample-config/ 5 | title: sample resource 6 | toc: true 7 | weight: 20 8 | --- 9 | 10 | ``` 11 | [exec] 12 | command = "/path/to/program" 13 | kill_signal = "SIGTERM" 14 | reload_signal = "SIGHUP" 15 | kill_timeout = 10 16 | splay = 10 17 | 18 | 19 | [[template]] 20 | src = "/etc/remco/templates/haproxy.cfg" 21 | dst = "/etc/haproxy/haproxy.cfg" 22 | reload_cmd = "haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -D -sf `cat /var/run/haproxy.pid`" 23 | mode = "0644" 24 | 25 | [backend] 26 | [backend.etcd] 27 | nodes = ["http://localhost:2379"] 28 | keys = ["/service-registry"] 29 | watchKeys = ["/haproxy/reload"] 30 | watch = true 31 | interval = 60 32 | version = 3 33 | 34 | ``` 35 | 36 | -------------------------------------------------------------------------------- /docs/content/details/backends.md: -------------------------------------------------------------------------------- 1 | --- 2 | date: 2016-12-03T14:31:14+01:00 3 | next: /details/plugins/ 4 | prev: /details/zombie-reaping/ 5 | title: backends 6 | toc: true 7 | weight: 30 8 | --- 9 | 10 | Remco can fetch configuration data from a bunch of different kv-stores. 11 | Some backends can be configured to watch for changes in the store to immediately react to these changes. 12 | The other way is to provide a backend polling interval. 13 | These two modes are not mutual exclusive, you can watch for changes and run the interval processor as a reconciliation loop. 14 | 15 | Every Backend needs to implement the [easyKV](https://github.com/HeavyHorst/easyKV) interface. 16 | This is also the repository where the current implementations live. 17 | 18 | 19 | Currently supported are: 20 | 21 | - **etcd 2 and 3** (interval and watch) 22 | - **consul** (interval and watch) 23 | - **nats kv** (interval and watch) 24 | - **zookeeper** (interval and watch) 25 | - **redis** (only interval) 26 | - **vault** (only interval) 27 | - **environment** (only interval) 28 | - **yaml/json files** (interval and watch) 29 | 30 | The different configuration parameters can be found here: [backend configuration](/config/configuration-options/#backend-configuration-options). 31 | -------------------------------------------------------------------------------- /docs/content/details/commands.md: -------------------------------------------------------------------------------- 1 | --- 2 | date: 2016-12-03T14:25:24+01:00 3 | next: /details/zombie-reaping/ 4 | prev: /details/exec-mode/ 5 | title: commands 6 | toc: true 7 | weight: 15 8 | --- 9 | 10 | Each template can have its own reload and check command. Both commands are executed in a sh-shell which means that operations like environment variable substitution or pipes should work correctly. 11 | 12 | In the check command its additionally possible to reference to the rendered source template with {{ .src }}. 13 | 14 | The check command must exit with status code 0 so that: 15 | 16 | - the reload command runs 17 | - the child process gets reloaded 18 | 19 | The template configuration parameters can be found here: [template configuration](/config/configuration-options/#template-configuration-options). 20 | -------------------------------------------------------------------------------- /docs/content/details/exec-mode.md: -------------------------------------------------------------------------------- 1 | --- 2 | date: 2016-12-03T14:23:28+01:00 3 | next: /details/commands/ 4 | prev: /details/template-resource/ 5 | title: exec mode 6 | toc: true 7 | weight: 10 8 | --- 9 | 10 | Remco has the ability to run one arbitary child process per template resource. 11 | When any of the provided templates change and the check command (if any) succeeds, remco will send the configurable reload signal to the child process. 12 | Remco will kill and restart the child process if no reload signal is provided. 13 | Additionally, every signal that remco receives will be forwarded to the child process. 14 | 15 | The template resource will fail if the child process dies. It will be automatically restarted after a random amount of time (0-30s). 16 | This also means that the child needs to remain in the foreground, otherwise the template resource will be restarted endlessly. 17 | 18 | The exec configuration parameters can be found here: [exec configuration](/config/configuration-options/#exec-configuration-options). -------------------------------------------------------------------------------- /docs/content/details/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | chapter: true 3 | date: 2016-12-03T14:19:23+01:00 4 | icon: 1. 5 | next: /details/template-resource/ 6 | prev: # 7 | title: Details 8 | weight: 0 9 | --- 10 | 11 | ### Chapter 1 12 | 13 | # Details 14 | 15 | 16 | -------------------------------------------------------------------------------- /docs/content/details/plugins.md: -------------------------------------------------------------------------------- 1 | --- 2 | date: 2016-12-03T14:32:50+01:00 3 | next: /details/process-lifecycle/ 4 | prev: /details/backends/ 5 | title: plugins 6 | toc: true 7 | weight: 35 8 | --- 9 | 10 | Remco supports backends as plugins. 11 | There is no requirement that plugins be written in Go. 12 | Every language that can provide a JSON-RPC API is ok. 13 | 14 | Example: [env plugin](/plugins/env-plugin-example/). 15 | -------------------------------------------------------------------------------- /docs/content/details/process-lifecycle.md: -------------------------------------------------------------------------------- 1 | --- 2 | date: 2016-12-03T14:33:41+01:00 3 | next: /details/telemetry/ 4 | prev: /details/plugins/ 5 | title: process lifecycle 6 | toc: true 7 | weight: 40 8 | --- 9 | 10 | Remcos lifecycle can be controlled with several syscalls. 11 | 12 | - os.Interrupt(SIGINT on linux) and SIGTERM: remco will gracefully shut down 13 | - SIGHUP: remco will reload all configuration files. 14 | -------------------------------------------------------------------------------- /docs/content/details/telemetry.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Telemetry" 3 | date: 2020-09-05T21:12:31+03:00 4 | next: /config/ 5 | prev: /details/process-lifecycle/ 6 | toc: true 7 | weight: 50 8 | --- 9 | 10 | Remco can expose different metrics about it's state using [go-metrics](https://github.com/armon/go-metrics). 11 | You can configure any type of sink supported by go-metric using configuration file. 12 | All the configured sinks will be aggregated using FanoutSink. 13 | 14 | 15 | Currently supported sinks are: 16 | 17 | - **inmem** 18 | - **prometheus** 19 | - **statsd** 20 | - **statsite** 21 | 22 | The different coniguration parameters can be found here: [telemetry configuration](/config/configuration-options/#telemetry-configuration-options). 23 | 24 | Exposed metrics: 25 | - **files.template_execution_duration** 26 | - Duration of template execution 27 | - **files.check_command_duration** 28 | - Duration of check_command execution 29 | - **files.reload_command_duration** 30 | - Duration of reload_command execution 31 | - **files.stage_errors_total** 32 | - Total number of errors in file staging action 33 | - **files.staged_total** 34 | - Total number of successfully files staged 35 | - **files.sync_errors_total** 36 | - Total number of errors in file syncing action 37 | - **files.synced_total** 38 | - Total number of successfully files synced 39 | - **backends.sync_errors_total** 40 | - Total errors in backend sync action 41 | - **backends.synced_total** 42 | - Total number of successfully synced backends 43 | -------------------------------------------------------------------------------- /docs/content/details/template-resource.md: -------------------------------------------------------------------------------- 1 | --- 2 | date: 2016-12-03T14:20:57+01:00 3 | next: /details/exec-mode 4 | prev: /details/ 5 | title: template resource 6 | toc: true 7 | weight: 5 8 | --- 9 | 10 | A template resource in remco consists of the following parts: 11 | 12 | - **one optional exec command.** 13 | - **one or many templates.** 14 | - **one or many backends.** 15 | 16 | {{% notice note %}} 17 | Please note that it is not possible to use the same backend more than once per template resource. 18 | It is for example not possible to use two different redis servers. 19 | {{% /notice %}} -------------------------------------------------------------------------------- /docs/content/details/zombie-reaping.md: -------------------------------------------------------------------------------- 1 | --- 2 | date: 2016-12-03T14:30:27+01:00 3 | next: /details/backends/ 4 | prev: /details/commands/ 5 | title: zombie reaping 6 | toc: true 7 | weight: 20 8 | --- 9 | 10 | See: https://blog.phusion.nl/2015/01/20/docker-and-the-pid-1-zombie-reaping-problem/ 11 | 12 | If Remco detects that it runs as pid 1 (for example in a Docker container) it will automatically reap zombie processes. 13 | No additional init system is needed. 14 | -------------------------------------------------------------------------------- /docs/content/examples/haproxy.md: -------------------------------------------------------------------------------- 1 | --- 2 | date: 2016-12-03T14:56:09+01:00 3 | next: # 4 | prev: /examples/ 5 | title: Dynamic haproxy configuration with docker, registrator and etcd 6 | toc: true 7 | weight: 5 8 | --- 9 | 10 | ## The haproxy template 11 | 12 | We expect [registrator](http://gliderlabs.github.io/registrator/latest/) to write the service data in this format to etcd: 13 | 14 | /services// = : 15 | 16 | The scheme (tcp, http) and the host_port of the service is configurable over the following keys: 17 | 18 | /config//scheme 19 | /config//host_port 20 | 21 | We create the template for the haproxy configuration file first. 22 | Create the file **haproxy.tmpl** and add the following config blocks: 23 | 24 | Some default configuration parameters: 25 | ``` 26 | global 27 | daemon 28 | maxconn 2048 29 | 30 | defaults 31 | timeout connect 5000ms 32 | timeout client 500000ms 33 | timeout server 500000ms 34 | log global 35 | 36 | frontend name_resolver_http 37 | bind *:80 38 | mode http 39 | ``` 40 | 41 | 42 |
43 | 44 | This block creates the *http* acl's. 45 | We itarate over all directories under /config (the services) and create a **url_beg** and a **hdr_beg(host)** acl if the service has a scheme configured. 46 | Note that we sort the services by length and iterate in reversed order (longest services first). That way we can have services with the same prefix, for example redis_test, and redis. 47 | 48 | ``` 49 | {% for dir in lsdir("/config") | sortByLength reversed %} 50 | {% if exists(printf("/config/%s/scheme", dir)) %} 51 | {% if getv(printf("/config/%s/scheme", dir)) == "http" %} 52 | {% if ls(printf("/services/%s", dir)) %} 53 | acl is_{{ dir }} url_beg /{{ dir }} 54 | acl is_{{ dir }} hdr_beg(host) {{ dir }} 55 | use_backend {{ dir }}_servers if is_{{ dir }} 56 | {% endif %} 57 | {% endif %} 58 | {% endif %} 59 | {% endfor %} 60 | ``` 61 | 62 | If we had one service named redis we would get: 63 | 64 | ``` 65 | acl is_redis url_beg /redis 66 | acl is_redis hdr_beg(host) redis 67 | use_backend redis_servers if is_redis 68 | ``` 69 | 70 |
71 | 72 | Optional template block to expose a service on an host port: 73 | We iterate over all services under /config, test if the scheme and host_port is configured and create the host port configuration. 74 | 75 | 76 | ``` 77 | {% for dir in lsdir("/config") %} 78 | {% if exists (printf ("/config/%s/scheme", dir )) %} 79 | {% if exists (printf("/config/%s/host_port", dir )) %} 80 | {% if ls(printf("/services/%s", dir)) %} 81 | frontend {{ dir }}_port 82 | mode {{ getv (printf("/config/%s/scheme", dir)) }} 83 | bind *:{{ getv (printf ("/config/%s/host_port", dir)) }} 84 | default_backend {{ dir }}_servers 85 | {% endif %} 86 | {% endif %} 87 | {% endif %} 88 | {% endfor %} 89 | ``` 90 | 91 | If we had one service named redis with scheme=tcp and host_port=6379 we would get: 92 | 93 | ``` 94 | frontend redis_port 95 | mode tcp 96 | bind *:6379 97 | default_backend redis_servers 98 | ``` 99 | 100 |
101 | 102 | The last block creates the haproxy backends. 103 | We iterate over all services and, if a scheme is set, create the backend *{service_name}_servers*. 104 | 105 | ``` 106 | {% for dir in lsdir("/services") %} 107 | {% if exists(printf("/config/%s/scheme", dir)) %} 108 | backend {{ dir }}_servers 109 | mode {{ getv (printf ("/config/%s/scheme", dir)) }} 110 | {% for i in gets (printf("/services/%s/*", dir)) %} 111 | server server_{{ dir }}_{{ base (i.Key) }} {{ i.Value }} 112 | {% endfor %} 113 | {% endif %} 114 | {% endfor %} 115 | ``` 116 | 117 | If we had one service named redis with scheme=tcp we could get for example: 118 | 119 | ``` 120 | backend redis_servers 121 | mode tcp 122 | server server_redis_1 192.168.0.10:32012 123 | server server_redis_2 192.168.0.10:35013 124 | ``` 125 | 126 |
127 | 128 | ## The remco configuration file 129 | 130 | We also need to create the remco configuration file. 131 | Create a file named **config** and insert the following toml configuration. 132 | 133 | ```toml 134 | ################################################################ 135 | # Global configuration 136 | ################################################################ 137 | log_level = "debug" 138 | log_format = "text" 139 | 140 | [[resource]] 141 | name = "haproxy" 142 | 143 | [[resource.template]] 144 | src = "/etc/remco/templates/haproxy.tmpl" 145 | dst = "/etc/haproxy/haproxy.cfg" 146 | reload_cmd = "haproxy -f {{.dst}} -p /var/run/haproxy.pid -D -sf `cat /var/run/haproxy.pid`" 147 | 148 | [resource.backend] 149 | [resource.backend.etcd] 150 | nodes = ["${ETCD_NODE}"] 151 | keys = ["/services", "/config"] 152 | watchKeys = ["/haproxy/reload"] 153 | watch = true 154 | interval = 60 155 | ``` 156 | 157 | ## The Dockerfile 158 | 159 | ``` 160 | FROM alpine:3.4 161 | 162 | ENV REMCO_VER 0.8.0 163 | 164 | RUN apk --update add --no-cache haproxy bash ca-certificates 165 | RUN wget https://github.com/HeavyHorst/remco/releases/download/v${REMCO_VER}/remco_${REMCO_VER}_linux_amd64.zip && \ 166 | unzip remco_${REMCO_VER}_linux_amd64.zip && rm remco_${REMCO_VER}_linux_amd64.zip && \ 167 | mv remco_linux /bin/remco 168 | 169 | COPY config /etc/remco/config 170 | COPY haproxy.tmpl /etc/remco/templates/haproxy.tmpl 171 | 172 | ENTRYPOINT ["remco"] 173 | ``` 174 | 175 | ## Build and Run the container 176 | 177 | You should have three files at this point: 178 | 179 | ``` 180 | . 181 | ├── config 182 | ├── Dockerfile 183 | └── haproxy.tmpl 184 | ``` 185 | 186 | ### Build the docker container: 187 | 188 | ```bash 189 | sudo docker build -t remcohaproxy . 190 | ``` 191 | 192 | ### Optionally test the container: 193 | 194 | #### put some data into etcd: 195 | 196 | ```bash 197 | etcdctl set /services/exampleService/1 someip:port 198 | etcdctl set /config/exampleService/scheme http 199 | etcdctl set /config/exampleService/host_port 1234 200 | ``` 201 | 202 | 203 | In this example we connect to a local etcd cluster. 204 | 205 | ```bash 206 | sudo docker run --rm -ti --net=host -e ETCD_NODE=http://localhost:2379 remcohaproxy 207 | ``` 208 | 209 | You should see something like this: 210 | 211 | ``` 212 | [Dec 16 18:26:20] INFO remco[1]: Target config out of sync config=/etc/haproxy/haproxy.cfg resource=haproxy source=resource.go:66 213 | [Dec 16 18:26:20] DEBUG remco[1]: Overwriting target config config=/etc/haproxy/haproxy.cfg resource=haproxy source=resource.go:66 214 | [Dec 16 18:26:20] DEBUG remco[1]: Running haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -D -sf `cat /var/run/haproxy.pid` resource=haproxy source=resource.go:66 215 | [Dec 16 18:26:20] DEBUG remco[1]: "" resource=haproxy source=resource.go:66 216 | [Dec 16 18:26:20] INFO remco[1]: Target config has been updated config=/etc/haproxy/haproxy.cfg resource=haproxy source=resource.go:66 217 | [Dec 16 18:26:20] DEBUG remco[1]: [Reaped child process 60] source=main.go:87 218 | [Dec 16 18:26:24] DEBUG remco[1]: Retrieving keys backend=etcd key_prefix= resource=haproxy source=resource.go:66 219 | [Dec 16 18:26:24] DEBUG remco[1]: Compiling source template resource=haproxy source=resource.go:66 template=/etc/remco/templates/haproxy.tmpl 220 | [Dec 16 18:26:24] DEBUG remco[1]: Comparing staged and dest config files dest=/etc/haproxy/haproxy.cfg resource=haproxy source=resource.go:66 staged=.haproxy.cfg389124299 221 | [Dec 16 18:26:24] DEBUG remco[1]: Target config in sync config=/etc/haproxy/haproxy.cfg resource=haproxy source=resource.go:66 222 | ``` 223 | 224 | ### Run registrator 225 | 226 | ``` 227 | sudo docker run -d \ 228 | --name=registrator \ 229 | --net=host \ 230 | --volume=/var/run/docker.sock:/tmp/docker.sock \ 231 | gliderlabs/registrator:latest \ 232 | etcd://localhost:2379/services 233 | ``` 234 | 235 | Now every container get automatically registered under /services. 236 | You can then configure the scheme and optionally the host_port of each service that you want to expose. 237 | -------------------------------------------------------------------------------- /docs/content/examples/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | chapter: true 3 | date: 2016-12-03T14:19:23+01:00 4 | icon: 5. 5 | next: /examples/haproxy/ 6 | prev: /plugins/consul-plugin-example/ 7 | title: Examples 8 | weight: 0 9 | --- 10 | 11 | ### Chapter 5 12 | 13 | # Examples 14 | 15 | This is a collection of different examples. -------------------------------------------------------------------------------- /docs/content/plugins/consul-plugin-example.md: -------------------------------------------------------------------------------- 1 | --- 2 | date: 2016-12-03T15:14:31+01:00 3 | next: # 4 | prev: /plugins/env-plugin-example/ 5 | title: consul plugin example 6 | toc: true 7 | weight: 10 8 | --- 9 | 10 | Here is another simple example plugin that speaks to the consul service endpoint instead of the consul kv-store like the built in consul backend. 11 | 12 | ```go 13 | package main 14 | 15 | import ( 16 | "encoding/json" 17 | "fmt" 18 | "log" 19 | "net/rpc/jsonrpc" 20 | "path" 21 | "strconv" 22 | 23 | "github.com/HeavyHorst/easyKV" 24 | "github.com/HeavyHorst/remco/backends/plugin" 25 | consul "github.com/hashicorp/consul/api" 26 | "github.com/natefinch/pie" 27 | ) 28 | 29 | func NewConsulClient(addr string) (*consul.Client, error) { 30 | config := consul.DefaultConfig() 31 | config.Address = addr 32 | c, err := consul.NewClient(config) 33 | if err != nil { 34 | return nil, err 35 | } 36 | return c, nil 37 | } 38 | 39 | type ConsulRPCServer struct { 40 | client *consul.Client 41 | } 42 | 43 | func main() { 44 | p := pie.NewProvider() 45 | if err := p.RegisterName("Plugin", &ConsulRPCServer{}); err != nil { 46 | log.Fatalf("failed to register Plugin: %s", err) 47 | } 48 | p.ServeCodec(jsonrpc.NewServerCodec) 49 | } 50 | 51 | func (c *ConsulRPCServer) Init(args map[string]string, resp *bool) error { 52 | var err error 53 | if addr, ok := args["addr"]; ok { 54 | c.client, err = NewConsulClient(addr) 55 | if err != nil { 56 | return err 57 | } 58 | *resp = true 59 | return nil 60 | } 61 | return fmt.Errorf("I need an Address !") 62 | } 63 | 64 | func (c *ConsulRPCServer) GetValues(args []string, resp *map[string]string) error { 65 | r := make(map[string]string) 66 | passingOnly := true 67 | for _, v := range args { 68 | addrs, _, err := c.client.Health().Service(v, "", passingOnly, nil) 69 | if len(addrs) == 0 && err == nil { 70 | log.Printf("service ( %s ) was not found", v) 71 | } 72 | if err != nil { 73 | return err 74 | } 75 | 76 | for idx, addr := range addrs { 77 | key := path.Join("/", "_consul", "service", addr.Service.Service, strconv.Itoa(idx)) 78 | service_json, _ := json.Marshal(addr) 79 | r[key] = string(service_json) 80 | } 81 | } 82 | *resp = r 83 | return nil 84 | } 85 | 86 | func (c *ConsulRPCServer) Close(args interface{}, resp *interface{}) error { 87 | // consul client doesn't need to be closed 88 | return nil 89 | } 90 | 91 | func (c *ConsulRPCServer) WatchPrefix(args plugin.WatchConfig, resp *uint64) error { 92 | return easyKV.ErrWatchNotSupported 93 | } 94 | ``` 95 | The config backend section could look like this: 96 | 97 | ``` 98 | [backend] 99 | [[backend.plugin]] 100 | path = "/etc/remco/plugins/consul-service" 101 | keys = ["consul"] 102 | interval = 60 103 | onetime = false 104 | [backend.plugin.config] 105 | addr = "localhost:8500" 106 | ``` 107 | 108 | -------------------------------------------------------------------------------- /docs/content/plugins/env-plugin-example.md: -------------------------------------------------------------------------------- 1 | --- 2 | date: 2016-12-03T15:13:50+01:00 3 | next: /plugins/consul-plugin-example/ 4 | prev: /plugins/ 5 | title: env plugin example 6 | toc: true 7 | weight: 5 8 | --- 9 | 10 | This is the env backend as a plugin. 11 | If you want to try it yourself, then 12 | just compile it and move the executable to /etc/remco/plugins. 13 | 14 | ```go 15 | package main 16 | 17 | import ( 18 | "context" 19 | "log" 20 | "net/rpc/jsonrpc" 21 | 22 | "github.com/HeavyHorst/easyKV" 23 | "github.com/HeavyHorst/easyKV/env" 24 | "github.com/HeavyHorst/remco/backends/plugin" 25 | "github.com/natefinch/pie" 26 | ) 27 | 28 | func main() { 29 | p := pie.NewProvider() 30 | if err := p.RegisterName("Plugin", &EnvRPCServer{}); err != nil { 31 | log.Fatalf("failed to register Plugin: %s", err) 32 | } 33 | p.ServeCodec(jsonrpc.NewServerCodec) 34 | } 35 | 36 | type EnvRPCServer struct { 37 | // This is the real implementation 38 | Impl easyKV.ReadWatcher 39 | } 40 | 41 | func (e *EnvRPCServer) Init(args map[string]interface{}, resp *bool) error { 42 | // use the data in args to create the ReadWatcher 43 | // env var doesn't need any data 44 | 45 | var err error 46 | e.Impl, err = env.New() 47 | return err 48 | } 49 | 50 | func (e *EnvRPCServer) GetValues(args []string, resp *map[string]string) error { 51 | erg, err := e.Impl.GetValues(args) 52 | if err != nil { 53 | return err 54 | } 55 | *resp = erg 56 | return nil 57 | } 58 | 59 | func (e *EnvRPCServer) Close(args interface{}, resp *interface{}) error { 60 | e.Impl.Close() 61 | return nil 62 | } 63 | 64 | func (e EnvRPCServer) WatchPrefix(args plugin.WatchConfig, resp *uint64) error { 65 | var err error 66 | *resp, err = e.Impl.WatchPrefix(context.Background(), args.Prefix, easyKV.WithKeys(args.Opts.Keys), easyKV.WithWaitIndex(args.Opts.WaitIndex)) 67 | return err 68 | } 69 | ``` 70 | 71 | Then create a config file with this backend section. 72 | 73 | ```toml 74 | [backend] 75 | [[backend.plugin]] 76 | path = "/etc/remco/plugins/env" 77 | keys = ["/"] 78 | interval = 60 79 | watch = false 80 | [backend.plugin.config] 81 | # these parameters are not used in the env backend plugin 82 | # but other plugins may need some data (password, prefix ...) 83 | a = "hallo" 84 | b = "moin" 85 | ``` 86 | -------------------------------------------------------------------------------- /docs/content/plugins/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | chapter: true 3 | date: 2016-12-03T15:11:07+01:00 4 | icon: 4. 5 | next: /plugins/env-plugin-example/ 6 | prev: /template/template-filters/ 7 | title: Plugins 8 | weight: 0 9 | --- 10 | 11 | ### Chapter 4 12 | 13 | # Plugins 14 | 15 | -------------------------------------------------------------------------------- /docs/content/template/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | chapter: true 3 | date: 2016-12-03T15:02:45+01:00 4 | icon: 3. 5 | next: /template/template-functions/ 6 | prev: /config/sample-resource/ 7 | title: Template 8 | weight: 0 9 | --- 10 | 11 | ### Chapter 3 12 | 13 | # Templates 14 | 15 | Templates are written in flosch's [`pongo2`](https://github.com/flosch/pongo2) template engine. 16 | 17 | {{% notice tip %}} 18 | For a documentation on how the templating language works you can [head over to the Django documentation](https://docs.djangoproject.com/en/dev/topics/templates/). pongo2 aims to be compatible with it. 19 | {{% /notice %}} -------------------------------------------------------------------------------- /docs/content/template/template-filters.md: -------------------------------------------------------------------------------- 1 | --- 2 | date: 2016-12-03T15:07:41+01:00 3 | next: /plugins/ 4 | prev: /template/template-functions/ 5 | title: template filters 6 | toc: true 7 | weight: 20 8 | --- 9 | 10 | ## Builtin filters 11 | 12 |
13 | **parseInt** -- Takes the given string and parses it as a base-10 integer (64bit) 14 | 15 | ``` 16 | {{ "12000" | parseInt }} 17 | ``` 18 |
19 | 20 |
21 | **parseFloat** -- Takes the given string and parses it as a float64 22 | 23 | ``` 24 | {{ "12000.45" | parseFloat }} 25 | ``` 26 |
27 | 28 |
29 | **base64** -- Encodes a string as base64 30 | 31 | ``` 32 | {{ "somestring" | base64}} 33 | ``` 34 |
35 | 36 |
37 | **base** -- Alias for the [path.Base](https://golang.org/pkg/path/#Base) function. 38 | 39 | ``` 40 | {{ "/home/user/test" | base }} 41 | ``` 42 |
43 | 44 |
45 | **dir** -- Alias for the [path.Dir](https://golang.org/pkg/path/#Dir) function. 46 | 47 | ``` 48 | {{ "/home/user/test" | dir }} 49 | ``` 50 |
51 | 52 |
53 | **split** -- Alias for the [strings.Split](https://golang.org/pkg/strings/#Split) function. 54 | 55 | ``` 56 | {% for i in ("/home/user/test" | split:"/") %} 57 | {{i}} 58 | {% endfor %} 59 | ``` 60 |
61 | 62 |
63 | **mapValue** -- Returns an map element by key 64 | 65 | ``` 66 | {{ getv("/some_yaml_config") | parseYAML | mapValue:"key" }} 67 | ``` 68 |
69 | 70 |
71 | **index** -- Returns an array element by index 72 | 73 | ``` 74 | {{ "/home/user/test" | split:"/" | index:"1" }} 75 | ``` 76 |
77 | 78 |
79 | **parseYAML** -- Returns an interface{} of the yaml value. 80 | 81 | ``` 82 | {% for value in getvs("/cache1/domains/*") %} 83 | {% set data = value | parseYAML %} 84 | {{ data.type }} {{ data.name }} {{ data.addr }} 85 | {% endfor %} 86 | ``` 87 |
88 | 89 |
90 | **parseJSON** -- Returns an interface{} of the json value. 91 | 92 | ``` 93 | {% for value in getvs("/cache1/domains/*") %} 94 | {% set data = value | parseJSON %} 95 | {{ data.type }} {{ data.name }} {{ data.addr }} 96 | {% endfor %} 97 | ``` 98 |
99 | 100 |
101 | **toJSON** -- Converts data, for example the result of gets or lsdir, into an JSON object. 102 | 103 | ``` 104 | {{ gets("/myapp/database/*") | toJson}} 105 | ``` 106 |
107 | 108 |
109 | **toPrettyJSON** -- Converts data, for example the result of gets or lsdir, into an pretty-printed JSON object, indented by four spaces. 110 | 111 | ``` 112 | {{ gets("/myapp/database/*") | toPrettyJson}} 113 | ``` 114 |
115 | 116 |
117 | **toYAML** -- Converts data, for example the result of gets or lsdir, into a YAML string. 118 | 119 | ``` 120 | {{ gets("/myapp/database/*") | toYAML}} 121 | ``` 122 |
123 | 124 |
125 | **sortByLength** - Returns the sorted array. 126 | 127 | Works with []string and []KVPair. 128 | ``` 129 | {% for dir in lsdir("/config") | sortByLength %} 130 | {{dir}} 131 | {% endfor %} 132 | ``` 133 |
134 | 135 | ## Custom filters 136 | 137 | It is possible to create custom filters in JavaScript. 138 | If you want to create a 'toEnv' filter, which transforms file system paths to environment variables, you must create the file 'toEnv.js' in the configurable filter directory. 139 | 140 | The filter code could look like: 141 | ```javascript 142 | In.split("/").join("_").substr(1).toUpperCase(); 143 | ``` 144 | 145 | There are two predefined variables: 146 | 147 | - In: the filter input (string) 148 | - Param: the optional filter parameter (string) 149 | 150 | As the parameter one string is possible only, the parameter string is added with a double-colon to the filter name (""yadda" | filter:"paramstr"). 151 | When the filter function needs multiple parameter all of them must be put into this one string and parsed inside the filter to extract all 152 | parameter from this string (example "replace" filter below). 153 | 154 | Remark: 155 | 156 | * "console" object for logging does not exist, therefore no output (for debugging and similar) possible. 157 | * variable declaration must use "var" as other keywords like "const" or "let" are not defined 158 | * the main script must not use "return" keyword, last output is the filter result. 159 | 160 | ### Examples 161 | **reverse filter** 162 | 163 | Put file `reverse.js` into the configured "filter_dir" with following content: 164 | 165 | ```javascript 166 | function reverse(s) { 167 | var o = ""; 168 | for (var i = s.length - 1; i >= 0; i--) 169 | o += s[i]; 170 | return o; 171 | } 172 | 173 | reverse(In); 174 | ``` 175 | Call this filter inside your template (e,g, "my-reverse-template.tmpl") with 176 | 177 | ``` 178 | {% set myString = "hip-hip-hooray" %} 179 | myString is {{ myString }} 180 | reversed myString is {{ myString | reverse }} 181 | ``` 182 | 183 | Output is: 184 | 185 | ```text 186 | myString is hip-hip-hooray 187 | reversed myString is yarooh-pih-pih 188 | ``` 189 | 190 | **replace filter** 191 | 192 | Put file `replace.js` into the configured "filter_dir" with following content: 193 | 194 | ```javascript 195 | function replace(str, p) { 196 | var params = [' ','_']; // default: replace all spaces with underscore 197 | if (p) { 198 | params = p.split(','); // split all params given at comma 199 | } 200 | // if third param is a "g" like "global" change search string to regexp 201 | if (params.length > 2 && params[2] == 'g') { 202 | params[0] = new RegExp(params[0], params[2]); 203 | } 204 | // javascript string.replace replaces first occurence only if search param is a string 205 | // need regexp object to replace all occurences 206 | return str.replace(params[0], params[1]); 207 | } 208 | replace(In, Param) 209 | ``` 210 | 211 | Use this inside the template as: 212 | 213 | ``` 214 | {% set myString = "hip-hip-hooray" %} 215 | myString is {{ myString }} 216 | replace with default params (spaces): {{ myString | replace }} 217 | only replace first "-" with underscore is {{ myString | replace:"-,_" }} 218 | replace all "-" with underscore is {{ myString | replace:"-,_,g" }} 219 | ``` 220 | 221 | Output is: 222 | 223 | ```text 224 | myString is hip-hip-hooray 225 | replace with default params (spaces): hip-hip-hooray 226 | only replace first "-" with underscore is hip_hip-hooray 227 | replace all "-" with underscore is hip_hip_hooray 228 | ``` 229 | -------------------------------------------------------------------------------- /docs/content/template/template-functions.md: -------------------------------------------------------------------------------- 1 | --- 2 | date: 2016-12-03T15:06:33+01:00 3 | next: /template/template-filters/ 4 | prev: /template/ 5 | title: template functions 6 | toc: true 7 | weight: 5 8 | --- 9 | 10 |
11 | **exists** -- Checks if the key exists. Return false if key is not found. 12 | 13 | ``` 14 | {% if exists("/key") %} 15 | value: {{ getv ("/key") }} 16 | {% endif %} 17 | ``` 18 |
19 | 20 |
21 | **get** -- Returns the KVPair where key matches its argument. 22 | 23 | ``` 24 | {% with get("/key") as dat %} 25 | key: {{dat.Key}} 26 | value: {{dat.Value}} 27 | {% endwith %} 28 | ``` 29 |
30 | 31 |
32 | **gets** -- Returns all KVPair, []KVPair, where key matches its argument. 33 | 34 | ``` 35 | {% for i in gets("/*") %} 36 | key: {{i.Key}} 37 | value: {{i.Value}} 38 | {% endfor %} 39 | ``` 40 |
41 | 42 |
43 | **getv** -- Returns the value as a string where key matches its argument or an optional default value. 44 | 45 | ``` 46 | value: {{ getv("/key") }} 47 | ``` 48 | #### With a default value 49 | ``` 50 | value: {{ getv("/key", "default_value") }} 51 | ``` 52 |
53 | 54 |
55 | **getvs** -- Returns all values, []string, where key matches its argument. 56 | 57 | ``` 58 | {% for value in getvs("/*") %} 59 | value: {{value}} 60 | {% endfor %} 61 | ``` 62 |
63 | 64 |
65 | **getenv** -- Retrieves the value of the environment variable named by the key. It returns the value, which will be empty if the variable is not present. Optionally, you can give a default value that will be returned if the key is not present. 66 | 67 | ``` 68 | export HOSTNAME=`hostname` 69 | ``` 70 | ``` 71 | hostname: {{getenv("HOSTNAME")}} 72 | ``` 73 | #### With a default value 74 | ``` 75 | ipaddr: {{ getenv("HOST_IP", "127.0.0.1") }} 76 | ``` 77 |
78 | 79 |
80 | **ls** -- Returns all subkeys, []string, where path matches its argument. Returns an empty list if path is not found. 81 | 82 | ``` 83 | {% for i in ls("/deis/services") %} 84 | value: {{i}} 85 | {% endfor %} 86 | ``` 87 |
88 | 89 |
90 | **lsdir** -- Returns all subkeys, []string, where path matches its argument. It only returns subkeys that also have subkeys. Returns an empty list if path is not found. 91 | 92 | ``` 93 | {% for dir in lsdir("/deis/services") %} 94 | value: {{dir}} 95 | {% endfor %} 96 | ``` 97 |
98 | 99 |
100 | **replace** -- Alias for the [strings.Replace](https://golang.org/pkg/strings/#Replace) function. 101 | 102 | ``` 103 | backend = {{ replace(getv("/services/backend/nginx"), "-", "_", -1) }} 104 | ``` 105 |
106 | 107 |
108 | **contains** -- Alias for the [strings.Contains](https://golang.org/pkg/strings/#Contains) function. 109 | 110 | ``` 111 | {% if contains(getv("/services/backend/nginx"), "something") %} 112 | something 113 | {% endif %} 114 | ``` 115 |
116 | 117 |
118 | **printf** -- Alias for the [fmt.Sprintf](https://golang.org/pkg/fmt/#Sprintf) function. 119 | 120 | ``` 121 | {{ getv (printf ("/config/%s/host_port", dir)) }} 122 | ``` 123 |
124 | 125 |
126 | **unixTS** -- Wrapper for [time.Now().Unix()](https://golang.org/pkg/time/#Unix). 127 | 128 | ``` 129 | {{ unixTS }} 130 | ``` 131 |
132 | 133 |
134 | **dateRFC3339** -- Wrapper for [time.Now().Format(time.RFC3339)](https://golang.org/pkg/time/). 135 | 136 | ``` 137 | {{ dateRFC3339 }} 138 | ``` 139 |
140 | 141 |
142 | **lookupIP** -- Wrapper for the [net.LookupIP](https://golang.org/pkg/net/#LookupIP) function. The wrapper returns the IP addresses in alphabetical order. 143 | 144 | ``` 145 | {% for ip in lookupIP("kube-master") %} 146 | {{ ip }} 147 | {% endfor %} 148 | ``` 149 |
150 | 151 |
152 | **lookupSRV** -- Wrapper for the [net.LookupSRV](https://golang.org/pkg/net/#LookupSRV) function. The wrapper returns the SRV records in alphabetical order. 153 | 154 | ``` 155 | {% for srv in lookupSRV("xmpp-server", "tcp", "google.com") %} 156 | target: {{ srv.Target }} 157 | port: {{ srv.Port }} 158 | priority: {{ srv.Priority }} 159 | weight: {{ srv.Weight }} 160 | {% endfor %} 161 | ``` 162 |
163 | 164 |
165 | **createMap** -- create a hashMap to store values at runtime. This can be useful if you want to generate json/yaml files. 166 | 167 | ``` 168 | {% set map = createMap() %} 169 | {{ map.Set("Moin", "Hallo2") }} 170 | {{ map.Set("Test", 105) }} 171 | {{ map | toYAML }} 172 | 173 | {% set map2 = createMap() %} 174 | {{ map2.Set("Moin", "Hallo") }} 175 | {{ map2.Set("Test", 300) }} 176 | {{ map2.Set("anotherMap", map) }} 177 | {{ map2 | toYAML }} 178 | ``` 179 | 180 | The hashmap supports the following methods: 181 | * `m.Set("key", value)` adds a new value of arbitrary type referenced by "key" to the map 182 | * `m.Get("key")` get the value for the given "key" 183 | * `m.Remove("key")` removes the key and value from the map 184 |
185 | 186 |
187 | **createSet** -- create a Set to store values at runtime. This can be useful if you want to generate json/yaml files. 188 | 189 | ``` 190 | {% set s = createSet() %} 191 | {{ s.Append("Moin") }} 192 | {{ s.Append("Moin") }} 193 | {{ s.Append("Hallo") }} 194 | {{ s.Append(1) }} 195 | {{ s.Remove("Hallo") }} 196 | {{ s | toYAML }} 197 | ``` 198 | The set created supports the following methods: 199 | * `s.Append("string")` adds a new string to the set. Attention - the set is not 200 | sorted or the order of appended elements guaranteed. 201 | * `s.Remove("string")` removes the given element from the set. 202 | * `s.Contains("string")` check if the given string is part of the set, returns 203 | true or false otherwise 204 | * `s.SortedSet()` return a new list where all elements are sorted in increasing 205 | order. This method should be used inside the template with a for-in loop to generate 206 | a stable output file not changing order of elements on every run. 207 | 208 | ``` 209 | {% set s = createSet() %} 210 | {% s.Append("Moin") %} 211 | {% s.Append("Hi") %} 212 | {% s.Append("Hallo") %} 213 | 214 | {% for greeting in s %} 215 | {{ geeting }} 216 | {% endfor %} 217 | 218 | {% for greeting in s.SortedSet() %} 219 | {{ geeting }} 220 | {% endfor %} 221 | ``` 222 | 223 | The output of the first loop is not defined, it can be in every order (like `Moin Hallo Hi` or `Hi Hallo Moin` and so on) 224 | The second loop returns every time `Hallo Hi Moin` (items sorted as string in increasing order) 225 |
226 | -------------------------------------------------------------------------------- /docs/layouts/index.html: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/layouts/partials/logo.html: -------------------------------------------------------------------------------- 1 | REMCO -------------------------------------------------------------------------------- /docs/layouts/partials/style.html: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /docs/static/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HeavyHorst/remco/fe0db414ca99ce5055ceb316f52a13948b213d36/docs/static/.gitkeep -------------------------------------------------------------------------------- /docs/static/style.css: -------------------------------------------------------------------------------- 1 | body, html { 2 | font-family: 'Inconsolata',monospace 3 | } -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/HeavyHorst/remco 2 | 3 | go 1.23.0 4 | 5 | toolchain go1.24.0 6 | 7 | require ( 8 | github.com/BurntSushi/toml v1.2.1 9 | github.com/HeavyHorst/easykv v1.2.12 10 | github.com/HeavyHorst/memkv v1.0.2 11 | github.com/HeavyHorst/pongo2 v3.3.0+incompatible 12 | github.com/armon/go-metrics v0.4.1 13 | github.com/dlclark/regexp2 v1.11.5 // indirect 14 | github.com/dop251/goja v0.0.0-20250305215736-311323fba474 15 | github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect 16 | github.com/hashicorp/consul-template v0.30.0 17 | github.com/hashicorp/go-hclog v1.6.3 18 | github.com/hashicorp/go-reap v0.0.0-20170704170343-bf58d8a43e7b 19 | github.com/juju/errors v0.0.0-20190930114154-d42613fe1ab9 // indirect 20 | github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 // indirect 21 | github.com/juju/testing v0.0.0-20191001232224-ce9dec17d28b // indirect 22 | github.com/mattn/go-shellwords v1.0.6 23 | github.com/natefinch/pie v0.0.0-20170715172608-9a0d72014007 24 | github.com/pborman/uuid v1.2.0 25 | github.com/pkg/errors v0.9.1 26 | github.com/prometheus/client_golang v1.14.0 27 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 28 | gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect 29 | gopkg.in/yaml.v3 v3.0.1 30 | sigs.k8s.io/yaml v1.2.0 31 | ) 32 | 33 | require ( 34 | github.com/armon/go-radix v1.0.0 // indirect 35 | github.com/beorn7/perks v1.0.1 // indirect 36 | github.com/cenkalti/backoff/v4 v4.3.0 // indirect 37 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 38 | github.com/coreos/go-semver v0.3.0 // indirect 39 | github.com/coreos/go-systemd/v22 v22.3.2 // indirect 40 | github.com/fatih/color v1.16.0 // indirect 41 | github.com/fsnotify/fsnotify v1.5.1 // indirect 42 | github.com/garyburd/redigo v1.6.2 // indirect 43 | github.com/go-jose/go-jose/v4 v4.0.5 // indirect 44 | github.com/gogo/protobuf v1.3.2 // indirect 45 | github.com/golang/protobuf v1.5.3 // indirect 46 | github.com/google/pprof v0.0.0-20250302191652-9094ed2288e7 // indirect 47 | github.com/google/uuid v1.4.0 // indirect 48 | github.com/hashicorp/consul/api v1.20.0 // indirect 49 | github.com/hashicorp/errwrap v1.1.0 // indirect 50 | github.com/hashicorp/go-cleanhttp v0.5.2 // indirect 51 | github.com/hashicorp/go-immutable-radix v1.3.1 // indirect 52 | github.com/hashicorp/go-multierror v1.1.1 // indirect 53 | github.com/hashicorp/go-retryablehttp v0.7.7 // indirect 54 | github.com/hashicorp/go-rootcerts v1.0.2 // indirect 55 | github.com/hashicorp/go-secure-stdlib/parseutil v0.1.9 // indirect 56 | github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect 57 | github.com/hashicorp/go-sockaddr v1.0.7 // indirect 58 | github.com/hashicorp/golang-lru v0.5.4 // indirect 59 | github.com/hashicorp/hcl v1.0.0 // indirect 60 | github.com/hashicorp/serf v0.10.1 // indirect 61 | github.com/hashicorp/vault/api v1.16.0 // indirect 62 | github.com/json-iterator/go v1.1.12 // indirect 63 | github.com/klauspost/compress v1.18.0 // indirect 64 | github.com/kr/pretty v0.3.0 // indirect 65 | github.com/kr/text v0.2.0 // indirect 66 | github.com/mattn/go-colorable v0.1.13 // indirect 67 | github.com/mattn/go-isatty v0.0.20 // indirect 68 | github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect 69 | github.com/mitchellh/go-homedir v1.1.0 // indirect 70 | github.com/mitchellh/mapstructure v1.5.0 // indirect 71 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 72 | github.com/modern-go/reflect2 v1.0.2 // indirect 73 | github.com/nats-io/nats.go v1.39.1 // indirect 74 | github.com/nats-io/nkeys v0.4.10 // indirect 75 | github.com/nats-io/nuid v1.0.1 // indirect 76 | github.com/prometheus/client_model v0.3.0 // indirect 77 | github.com/prometheus/common v0.37.0 // indirect 78 | github.com/prometheus/procfs v0.8.0 // indirect 79 | github.com/rogpeppe/go-internal v1.6.1 // indirect 80 | github.com/ryanuber/go-glob v1.0.0 // indirect 81 | github.com/tevino/go-zookeeper v0.0.0-20170512024026-c218ec636bef // indirect 82 | go.etcd.io/etcd/api/v3 v3.5.4 // indirect 83 | go.etcd.io/etcd/client/pkg/v3 v3.5.4 // indirect 84 | go.etcd.io/etcd/client/v2 v2.305.4 // indirect 85 | go.etcd.io/etcd/client/v3 v3.5.4 // indirect 86 | go.uber.org/atomic v1.10.0 // indirect 87 | go.uber.org/multierr v1.8.0 // indirect 88 | go.uber.org/zap v1.22.0 // indirect 89 | golang.org/x/crypto v0.36.0 // indirect 90 | golang.org/x/net v0.37.0 // indirect 91 | golang.org/x/sys v0.31.0 // indirect 92 | golang.org/x/text v0.23.0 // indirect 93 | golang.org/x/time v0.11.0 // indirect 94 | google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac // indirect 95 | google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917 // indirect 96 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe // indirect 97 | google.golang.org/grpc v1.61.0 // indirect 98 | google.golang.org/protobuf v1.33.0 // indirect 99 | gopkg.in/yaml.v2 v2.4.0 // indirect 100 | ) 101 | -------------------------------------------------------------------------------- /integration/consul/consul.toml: -------------------------------------------------------------------------------- 1 | #remco.toml 2 | ################################################################ 3 | # Global configuration 4 | ################################################################ 5 | log_level = "debug" 6 | log_format = "text" 7 | 8 | 9 | ################################################################ 10 | # Resource configuration 11 | ################################################################ 12 | [[resource]] 13 | [[resource.template]] 14 | src = "./integration/templates/basic.conf.tmpl" 15 | dst = "/tmp/remco-basic-test.conf" 16 | 17 | [resource.backend] 18 | [resource.backend.consul] 19 | nodes = ["127.0.0.1:8500"] 20 | prefix = "/appdata" 21 | onetime = true 22 | interval = 1 23 | keys = ["/"] 24 | -------------------------------------------------------------------------------- /integration/consul/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Configure consul 4 | 5 | curl -X PUT http://127.0.0.1:8500/v1/kv/appdata/database/host -d '127.0.0.1' 6 | curl -X PUT http://127.0.0.1:8500/v1/kv/appdata/database/password -d 'p@sSw0rd' 7 | curl -X PUT http://127.0.0.1:8500/v1/kv/appdata/database/port -d '3306' 8 | curl -X PUT http://127.0.0.1:8500/v1/kv/appdata/database/username -d 'remco' 9 | curl -X PUT http://127.0.0.1:8500/v1/kv/appdata/upstream/app1 -d '10.0.1.10:8080' 10 | curl -X PUT http://127.0.0.1:8500/v1/kv/appdata/upstream/app2 -d '10.0.1.11:8080' 11 | 12 | remco --config integration/consul/consul.toml 13 | cat /tmp/remco-basic-test.conf 14 | -------------------------------------------------------------------------------- /integration/env/env.toml: -------------------------------------------------------------------------------- 1 | #remco.toml 2 | ################################################################ 3 | # Global configuration 4 | ################################################################ 5 | log_level = "debug" 6 | log_format = "text" 7 | 8 | 9 | ################################################################ 10 | # Resource configuration 11 | ################################################################ 12 | [[resource]] 13 | [[resource.template]] 14 | src = "./integration/templates/basic.conf.tmpl" 15 | dst = "/tmp/remco-basic-test.conf" 16 | 17 | [resource.backend] 18 | [resource.backend.env] 19 | prefix = "/appdata" 20 | onetime = true 21 | interval = 1 22 | keys = ["/"] 23 | -------------------------------------------------------------------------------- /integration/env/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export APPDATA_DATABASE_HOST="127.0.0.1" 4 | export APPDATA_DATABASE_PASSWORD="p@sSw0rd" 5 | export APPDATA_DATABASE_PORT="3306" 6 | export APPDATA_DATABASE_USERNAME="remco" 7 | export APPDATA_DATABASE_APP1="10.0.1.10:8080" 8 | export APPDATA_DATABASE_APP2="10.0.1.11:8080" 9 | 10 | remco --config integration/env/env.toml 11 | cat /tmp/remco-basic-test.conf 12 | -------------------------------------------------------------------------------- /integration/etcdv2/etcd.toml: -------------------------------------------------------------------------------- 1 | #remco.toml 2 | ################################################################ 3 | # Global configuration 4 | ################################################################ 5 | log_level = "debug" 6 | log_format = "text" 7 | 8 | 9 | ################################################################ 10 | # Resource configuration 11 | ################################################################ 12 | [[resource]] 13 | [[resource.template]] 14 | src = "./integration/templates/basic.conf.tmpl" 15 | dst = "/tmp/remco-basic-test-etcdv2.conf" 16 | 17 | [resource.backend] 18 | [resource.backend.etcd] 19 | nodes = ["http://127.0.0.1:2379"] 20 | prefix = "/appdata" 21 | version = 2 22 | onetime = true 23 | keys = ["/"] 24 | -------------------------------------------------------------------------------- /integration/etcdv2/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | etcdctl --endpoints=http://127.0.0.1:2379 set /appdata/database/host 127.0.0.1 4 | etcdctl --endpoints=http://127.0.0.1:2379 set /appdata/database/password p@sSw0rd 5 | etcdctl --endpoints=http://127.0.0.1:2379 set /appdata/database/port 3306 6 | etcdctl --endpoints=http://127.0.0.1:2379 set /appdata/database/username remco 7 | etcdctl --endpoints=http://127.0.0.1:2379 set /appdata/upstream/app1 10.0.1.10:8080 8 | etcdctl --endpoints=http://127.0.0.1:2379 set /appdata/upstream/app2 10.0.1.11:8080 9 | 10 | remco --config integration/etcdv2/etcd.toml 11 | cat /tmp/remco-basic-test-etcdv2.conf 12 | -------------------------------------------------------------------------------- /integration/etcdv3/etcd.toml: -------------------------------------------------------------------------------- 1 | #remco.toml 2 | ################################################################ 3 | # Global configuration 4 | ################################################################ 5 | log_level = "debug" 6 | log_format = "text" 7 | 8 | 9 | ################################################################ 10 | # Resource configuration 11 | ################################################################ 12 | [[resource]] 13 | [[resource.template]] 14 | src = "./integration/templates/basic.conf.tmpl" 15 | dst = "/tmp/remco-basic-test-etcdv3.conf" 16 | 17 | [resource.backend] 18 | [resource.backend.etcd] 19 | nodes = ["127.0.0.1:2379"] 20 | prefix = "/appdata" 21 | version = 3 22 | onetime = true 23 | keys = ["/"] 24 | -------------------------------------------------------------------------------- /integration/etcdv3/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export ETCDCTL_API=3 4 | 5 | etcdctl --endpoints=127.0.0.1:2379 put /appdata/database/host 127.0.0.1 6 | etcdctl --endpoints=127.0.0.1:2379 put /appdata/database/password p@sSw0rd 7 | etcdctl --endpoints=127.0.0.1:2379 put /appdata/database/port 3306 8 | etcdctl --endpoints=127.0.0.1:2379 put /appdata/database/username remco 9 | etcdctl --endpoints=127.0.0.1:2379 put /appdata/upstream/app1 10.0.1.10:8080 10 | etcdctl --endpoints=127.0.0.1:2379 put /appdata/upstream/app2 10.0.1.11:8080 11 | 12 | remco --config integration/etcdv3/etcd.toml 13 | cat /tmp/remco-basic-test-etcdv3.conf 14 | -------------------------------------------------------------------------------- /integration/file/config.yml: -------------------------------------------------------------------------------- 1 | database: 2 | host: "127.0.0.1" 3 | password: "p@sSw0rd" 4 | port: "3306" 5 | username: "remco" 6 | 7 | upstream: 8 | - app1: 10.0.1.10:8080 9 | - app2: 10.0.1.11:8080 -------------------------------------------------------------------------------- /integration/file/file.toml: -------------------------------------------------------------------------------- 1 | #remco.toml 2 | ################################################################ 3 | # Global configuration 4 | ################################################################ 5 | log_level = "debug" 6 | log_format = "text" 7 | 8 | 9 | ################################################################ 10 | # Resource configuration 11 | ################################################################ 12 | [[resource]] 13 | [[resource.template]] 14 | src = "./integration/templates/basic.conf.tmpl" 15 | dst = "/tmp/remco-basic-test.conf" 16 | 17 | [resource.backend] 18 | [resource.backend.file] 19 | onetime = true 20 | filepath = "./integration/file/config.yml" 21 | httpheaders = { X-Test-Token = "XXX", X-Test-Token2 = "YYY" } 22 | keys = ["/"] 23 | -------------------------------------------------------------------------------- /integration/file/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | remco --config integration/file/file.toml 4 | cat /tmp/remco-basic-test.conf 5 | -------------------------------------------------------------------------------- /integration/redis/redis.toml: -------------------------------------------------------------------------------- 1 | #remco.toml 2 | ################################################################ 3 | # Global configuration 4 | ################################################################ 5 | log_level = "debug" 6 | log_format = "text" 7 | 8 | 9 | ################################################################ 10 | # Resource configuration 11 | ################################################################ 12 | [[resource]] 13 | [[resource.template]] 14 | src = "./integration/templates/basic.conf.tmpl" 15 | dst = "/tmp/remco-basic-test.conf" 16 | 17 | [resource.backend] 18 | [resource.backend.redis] 19 | nodes = ["127.0.0.1:6379"] 20 | prefix = "/appdata" 21 | onetime = true 22 | interval = 1 23 | keys = ["/"] 24 | -------------------------------------------------------------------------------- /integration/redis/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Configure redis 4 | 5 | redis-cli set /appdata/database/host 127.0.0.1 6 | redis-cli set /appdata/database/password p@sSw0rd 7 | redis-cli set /appdata/database/port 3306 8 | redis-cli set /appdata/database/username remco 9 | redis-cli set /appdata/upstream/app1 10.0.1.10:8080 10 | redis-cli set /appdata/upstream/app2 10.0.1.11:8080 11 | 12 | remco --config integration/redis/redis.toml 13 | cat /tmp/remco-basic-test.conf 14 | -------------------------------------------------------------------------------- /integration/templates/basic.conf.tmpl: -------------------------------------------------------------------------------- 1 | {{ getallkvs() | toPrettyJSON }} 2 | -------------------------------------------------------------------------------- /integration/vault/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | ROOT_TOKEN=$(vault read -field id auth/token/lookup-self) 4 | sed -i -- "s/§§token§§/${ROOT_TOKEN}/g" integration/vault/vault.toml 5 | 6 | vault secrets enable -path database kv 7 | vault secrets enable -path upstream kv 8 | 9 | vault write database/host value=127.0.0.1 10 | vault write database/port value=3306 11 | vault write database/username value=remco 12 | vault write database/password value=p@sSw0rd 13 | vault write upstream/app1 value=10.0.1.10:8080 14 | vault write upstream/app2 value=10.0.1.11:8080 15 | 16 | remco --config integration/vault/vault.toml 17 | cat /tmp/remco-basic-test.conf 18 | -------------------------------------------------------------------------------- /integration/vault/vault.toml: -------------------------------------------------------------------------------- 1 | #remco.toml 2 | ################################################################ 3 | # Global configuration 4 | ################################################################ 5 | log_level = "debug" 6 | log_format = "text" 7 | 8 | 9 | ################################################################ 10 | # Resource configuration 11 | ################################################################ 12 | [[resource]] 13 | [[resource.template]] 14 | src = "./integration/templates/basic.conf.tmpl" 15 | dst = "/tmp/remco-basic-test.conf" 16 | 17 | [resource.backend] 18 | [resource.backend.vault] 19 | onetime = true 20 | auth_type = "token" 21 | auth_token = "§§token§§" 22 | node = "http://127.0.0.1:8200" 23 | keys = [ 24 | "/database/host", 25 | "/database/password", 26 | "/database/port", 27 | "/database/username", 28 | "/upstream/app1", 29 | "/upstream/app2", 30 | ] 31 | -------------------------------------------------------------------------------- /integration/zookeeper/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | zookeeper-3.4.9/bin/zkCli.sh create /appdata "" 4 | zookeeper-3.4.9/bin/zkCli.sh create /appdata/database "" 5 | zookeeper-3.4.9/bin/zkCli.sh create /appdata/upstream "" 6 | 7 | zookeeper-3.4.9/bin/zkCli.sh create /appdata/database/password "p@sSw0rd" 8 | zookeeper-3.4.9/bin/zkCli.sh create /appdata/database/port "3306" 9 | zookeeper-3.4.9/bin/zkCli.sh create /appdata/database/username "remco" 10 | 11 | zookeeper-3.4.9/bin/zkCli.sh create /appdata/upstream/app1 "10.0.1.10:8080" 12 | zookeeper-3.4.9/bin/zkCli.sh create /appdata/upstream/app2 "10.0.1.11:8080" 13 | 14 | remco --config integration/zookeeper/zookeeper.toml 15 | cat /tmp/remco-basic-test.conf 16 | -------------------------------------------------------------------------------- /integration/zookeeper/zookeeper.toml: -------------------------------------------------------------------------------- 1 | #remco.toml 2 | ################################################################ 3 | # Global configuration 4 | ################################################################ 5 | log_level = "debug" 6 | log_format = "text" 7 | 8 | 9 | ################################################################ 10 | # Resource configuration 11 | ################################################################ 12 | [[resource]] 13 | [[resource.template]] 14 | src = "./integration/templates/basic.conf.tmpl" 15 | dst = "/tmp/remco-basic-test.conf" 16 | 17 | [resource.backend] 18 | [resource.backend.zookeeper] 19 | nodes = ["127.0.0.1:2181"] 20 | prefix = "/appdata" 21 | onetime = true 22 | interval = 1 23 | keys = ["/"] 24 | -------------------------------------------------------------------------------- /pkg/backends/config_types.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2016 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | package backends 10 | 11 | import ( 12 | "fmt" 13 | "net" 14 | "strings" 15 | ) 16 | 17 | // SRVRecord is a SRV-Record string 18 | // for example _etcd-client._tcp.example.com. 19 | type SRVRecord string 20 | 21 | // GetNodesFromSRV returns the nodes stored in the record. 22 | func (r SRVRecord) GetNodesFromSRV(scheme string) ([]string, error) { 23 | var nodes []string 24 | _, addrs, err := net.LookupSRV("", "", string(r)) 25 | if err != nil { 26 | return nodes, err 27 | } 28 | 29 | for _, srv := range addrs { 30 | port := fmt.Sprintf("%d", srv.Port) 31 | host := strings.TrimRight(srv.Target, ".") 32 | host = net.JoinHostPort(host, port) 33 | if scheme != "" { 34 | host = fmt.Sprintf("%s://%s", scheme, host) 35 | } 36 | 37 | nodes = append(nodes, host) 38 | } 39 | return nodes, nil 40 | } 41 | -------------------------------------------------------------------------------- /pkg/backends/consul.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2016 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | package backends 10 | 11 | import ( 12 | "github.com/HeavyHorst/easykv/consul" 13 | berr "github.com/HeavyHorst/remco/pkg/backends/error" 14 | "github.com/HeavyHorst/remco/pkg/log" 15 | "github.com/HeavyHorst/remco/pkg/template" 16 | ) 17 | 18 | // ConsulConfig represents the config for the consul backend. 19 | type ConsulConfig struct { 20 | // Nodes is a list of backend nodes. 21 | // A connection will only be established to the first server in the list. 22 | Nodes []string 23 | 24 | // The backend URI scheme (http or https). 25 | Scheme string 26 | 27 | // A DNS server record to discover the consul nodes. 28 | SRVRecord SRVRecord `toml:"srv_record"` 29 | 30 | // The client cert file. 31 | ClientCert string `toml:"client_cert"` 32 | 33 | // The client key file. 34 | ClientKey string `toml:"client_key"` 35 | 36 | //The client CA key file. 37 | ClientCaKeys string `toml:"client_ca_keys"` 38 | 39 | template.Backend 40 | } 41 | 42 | // Connect creates a new consulClient and fills the underlying template.Backend with the consul-Backend specific data. 43 | func (c *ConsulConfig) Connect() (template.Backend, error) { 44 | if c == nil { 45 | return template.Backend{}, berr.ErrNilConfig 46 | } 47 | c.Backend.Name = "consul" 48 | 49 | // No nodes are set but a SRVRecord is provided 50 | if len(c.Nodes) == 0 && c.SRVRecord != "" { 51 | var err error 52 | c.Nodes, err = c.SRVRecord.GetNodesFromSRV("") 53 | if err != nil { 54 | return c.Backend, err 55 | } 56 | } 57 | 58 | log.WithFields( 59 | "backend", c.Backend.Name, 60 | "nodes", c.Nodes, 61 | ).Info("set backend nodes") 62 | 63 | client, err := consul.New(c.Nodes, consul.WithScheme(c.Scheme), consul.WithTLSOptions(consul.TLSOptions{ 64 | ClientCert: c.ClientCert, 65 | ClientKey: c.ClientKey, 66 | ClientCaKeys: c.ClientCaKeys, 67 | })) 68 | 69 | if err != nil { 70 | return c.Backend, err 71 | } 72 | 73 | c.Backend.ReadWatcher = client 74 | 75 | return c.Backend, nil 76 | } 77 | 78 | // return Backend config to allow modification before connect() for onetime param or similar 79 | func (c *ConsulConfig) GetBackend() *template.Backend { 80 | return &c.Backend 81 | } 82 | -------------------------------------------------------------------------------- /pkg/backends/env.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2016 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | package backends 10 | 11 | import ( 12 | "github.com/HeavyHorst/easykv/env" 13 | berr "github.com/HeavyHorst/remco/pkg/backends/error" 14 | "github.com/HeavyHorst/remco/pkg/template" 15 | ) 16 | 17 | //EnvConfig represents the config for the env backend 18 | type EnvConfig struct { 19 | template.Backend 20 | } 21 | 22 | // Connect creates a new envClient and fills the underlying template.Backend with the file-Backend specific data. 23 | func (c *EnvConfig) Connect() (template.Backend, error) { 24 | if c == nil { 25 | return template.Backend{}, berr.ErrNilConfig 26 | } 27 | c.Backend.Name = "env" 28 | 29 | client, err := env.New() 30 | if err != nil { 31 | return c.Backend, err 32 | } 33 | 34 | c.Backend.ReadWatcher = client 35 | return c.Backend, nil 36 | } 37 | 38 | // return Backend config to allow modification before connect() for onetime param or similar 39 | func (c *EnvConfig) GetBackend() *template.Backend { 40 | return &c.Backend 41 | } 42 | -------------------------------------------------------------------------------- /pkg/backends/error/error.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2016 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | // Package error describes errors in remco backends 10 | package error 11 | 12 | import "errors" 13 | 14 | // BackendError contains an error message and the name of the backend that produced the error 15 | type BackendError struct { 16 | Backend string 17 | Message string 18 | } 19 | 20 | // Error is for the error interface 21 | func (e BackendError) Error() string { 22 | return e.Message 23 | } 24 | 25 | // ErrNilConfig is returned if Connect is called on a nil Config 26 | var ErrNilConfig = errors.New("config is nil") 27 | -------------------------------------------------------------------------------- /pkg/backends/etcd.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2016 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | package backends 10 | 11 | import ( 12 | "github.com/HeavyHorst/easykv/etcd" 13 | berr "github.com/HeavyHorst/remco/pkg/backends/error" 14 | "github.com/HeavyHorst/remco/pkg/log" 15 | "github.com/HeavyHorst/remco/pkg/template" 16 | ) 17 | 18 | // EtcdConfig represents the config for the etcd backend. 19 | type EtcdConfig struct { 20 | // Nodes is list of backend nodes. 21 | Nodes []string 22 | 23 | // The backend URI scheme (http or https). 24 | // This is only used when the nodes are discovered via DNS srv records and the api level is 2. 25 | // 26 | // The default is http. 27 | Scheme string 28 | 29 | // The client cert file. 30 | ClientCert string `toml:"client_cert"` 31 | 32 | // The client key file. 33 | ClientKey string `toml:"client_key"` 34 | 35 | // The client CA key file. 36 | ClientCaKeys string `toml:"client_ca_keys"` 37 | 38 | // A DNS server record to discover the etcd nodes. 39 | SRVRecord SRVRecord `toml:"srv_record"` 40 | 41 | // The username for the basic_auth authentication. 42 | Username string 43 | 44 | // The password for the basic_auth authentication. 45 | Password string 46 | 47 | // The etcd api-level to use (2 or 3). 48 | // 49 | // The default is 2. 50 | Version int 51 | template.Backend 52 | } 53 | 54 | // Connect creates a new etcd{2,3}Client and fills the underlying template.Backend with the etcd-Backend specific data. 55 | func (c *EtcdConfig) Connect() (template.Backend, error) { 56 | if c == nil { 57 | return template.Backend{}, berr.ErrNilConfig 58 | } 59 | 60 | // use api version 2 if no version is specified 61 | if c.Version == 0 { 62 | c.Version = 2 63 | } 64 | 65 | if c.Version == 3 { 66 | c.Backend.Name = "etcdv3" 67 | } else { 68 | c.Backend.Name = "etcd" 69 | } 70 | 71 | // No nodes are set but a SRVRecord is provided 72 | if len(c.Nodes) == 0 && c.SRVRecord != "" { 73 | var err error 74 | if c.Version != 2 { 75 | // no scheme required for etcdv3 76 | c.Scheme = "" 77 | } else if c.Scheme == "" { 78 | // etcd version is 2 and no scheme is provided 79 | // use http as default value 80 | c.Scheme = "http" 81 | } 82 | c.Nodes, err = c.SRVRecord.GetNodesFromSRV(c.Scheme) 83 | 84 | if err != nil { 85 | return c.Backend, err 86 | } 87 | } 88 | 89 | log.WithFields( 90 | "backend", c.Backend.Name, 91 | "nodes", c.Nodes, 92 | ).Info("set backend nodes") 93 | 94 | client, err := etcd.New(c.Nodes, 95 | etcd.WithBasicAuth(etcd.BasicAuthOptions{ 96 | Username: c.Username, 97 | Password: c.Password, 98 | }), 99 | etcd.WithTLSOptions(etcd.TLSOptions{ 100 | ClientCert: c.ClientCert, 101 | ClientKey: c.ClientKey, 102 | ClientCaKeys: c.ClientCaKeys, 103 | }), 104 | etcd.WithVersion(c.Version)) 105 | 106 | if err != nil { 107 | return c.Backend, err 108 | } 109 | 110 | c.Backend.ReadWatcher = client 111 | return c.Backend, nil 112 | } 113 | 114 | // return Backend config to allow modification before connect() for onetime param or similar 115 | func (c *EtcdConfig) GetBackend() *template.Backend { 116 | return &c.Backend 117 | } 118 | -------------------------------------------------------------------------------- /pkg/backends/file.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2016 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | package backends 10 | 11 | import ( 12 | "github.com/HeavyHorst/easykv/file" 13 | berr "github.com/HeavyHorst/remco/pkg/backends/error" 14 | "github.com/HeavyHorst/remco/pkg/log" 15 | "github.com/HeavyHorst/remco/pkg/template" 16 | ) 17 | 18 | // FileConfig represents the config for the file backend. 19 | type FileConfig struct { 20 | // The filepath to a yaml or json file containing the key-value pairs. 21 | // This can be a local file or a remote http/https location. 22 | Filepath string 23 | 24 | // Optional HTTP headers to append to the request if the file path is a remote http/https location. 25 | HTTPHeaders map[string]string 26 | template.Backend 27 | } 28 | 29 | // Connect creates a new fileClient and fills the underlying template.Backend with the file-Backend specific data. 30 | func (c *FileConfig) Connect() (template.Backend, error) { 31 | if c == nil { 32 | return template.Backend{}, berr.ErrNilConfig 33 | } 34 | 35 | c.Backend.Name = "file" 36 | log.WithFields( 37 | "backend", c.Backend.Name, 38 | "filepath", c.Filepath, 39 | ).Info("set file path") 40 | 41 | client, err := file.New(c.Filepath, file.WithHeaders(c.HTTPHeaders)) 42 | if err != nil { 43 | return c.Backend, err 44 | } 45 | c.Backend.ReadWatcher = client 46 | return c.Backend, nil 47 | } 48 | 49 | // return Backend config to allow modification before connect() for onetime param or similar 50 | func (c *FileConfig) GetBackend() *template.Backend { 51 | return &c.Backend 52 | } 53 | -------------------------------------------------------------------------------- /pkg/backends/mock.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2016 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | package backends 10 | 11 | import ( 12 | "github.com/HeavyHorst/easykv/mock" 13 | berr "github.com/HeavyHorst/remco/pkg/backends/error" 14 | "github.com/HeavyHorst/remco/pkg/template" 15 | ) 16 | 17 | // MockConfig represents the config for the consul backend. 18 | type MockConfig struct { 19 | Error error 20 | template.Backend 21 | } 22 | 23 | // Connect creates a new mockClient 24 | func (c *MockConfig) Connect() (template.Backend, error) { 25 | if c == nil { 26 | return template.Backend{}, berr.ErrNilConfig 27 | } 28 | c.Backend.Name = "mock" 29 | client, err := mock.New(c.Error, make(map[string]string)) 30 | if err != nil { 31 | return c.Backend, err 32 | } 33 | 34 | c.Backend.ReadWatcher = client 35 | 36 | return c.Backend, nil 37 | } 38 | 39 | // return Backend config to allow modification before connect() for onetime param or similar 40 | func (c *MockConfig) GetBackend() *template.Backend { 41 | return &c.Backend 42 | } 43 | -------------------------------------------------------------------------------- /pkg/backends/nats.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2022 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | package backends 10 | 11 | import ( 12 | "github.com/HeavyHorst/easykv/nats" 13 | berr "github.com/HeavyHorst/remco/pkg/backends/error" 14 | "github.com/HeavyHorst/remco/pkg/log" 15 | "github.com/HeavyHorst/remco/pkg/template" 16 | ) 17 | 18 | // NatsConfig represents the config for the nats backend. 19 | type NatsConfig struct { 20 | // Nodes is list of backend nodes. 21 | Nodes []string 22 | 23 | // The Nats kv bucket 24 | Bucket string 25 | 26 | // The username for the basic_auth authentication. 27 | Username string 28 | 29 | // The password for the basic_auth authentication. 30 | Password string 31 | 32 | // The token for the token auth method 33 | Token string 34 | 35 | // The path to an NATS 2.0 and NATS NGS compatible user credentials file 36 | Creds string 37 | 38 | template.Backend 39 | } 40 | 41 | // Connect creates a new etcd{2,3}Client and fills the underlying template.Backend with the etcd-Backend specific data. 42 | func (c *NatsConfig) Connect() (template.Backend, error) { 43 | if c == nil { 44 | return template.Backend{}, berr.ErrNilConfig 45 | } 46 | 47 | c.Backend.Name = "nats" 48 | 49 | log.WithFields( 50 | "backend", c.Backend.Name, 51 | "nodes", c.Nodes, 52 | ).Info("set backend nodes") 53 | 54 | client, err := nats.New(c.Nodes, c.Bucket, nats.WithBasicAuth(nats.BasicAuthOptions{ 55 | Username: c.Username, 56 | Password: c.Password, 57 | }), nats.WithToken(c.Token), nats.WithCredentials(c.Creds)) 58 | 59 | if err != nil { 60 | return c.Backend, err 61 | } 62 | 63 | c.Backend.ReadWatcher = client 64 | return c.Backend, nil 65 | } 66 | 67 | // return Backend config to allow modification before connect() for onetime param or similar 68 | func (c *NatsConfig) GetBackend() *template.Backend { 69 | return &c.Backend 70 | } 71 | -------------------------------------------------------------------------------- /pkg/backends/plugin/plugin.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2016 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | package plugin 10 | 11 | import ( 12 | "context" 13 | "net/rpc" 14 | "net/rpc/jsonrpc" 15 | "os" 16 | "path" 17 | 18 | "github.com/HeavyHorst/easykv" 19 | berr "github.com/HeavyHorst/remco/pkg/backends/error" 20 | "github.com/HeavyHorst/remco/pkg/template" 21 | "github.com/natefinch/pie" 22 | ) 23 | 24 | type plug struct { 25 | client *rpc.Client 26 | } 27 | 28 | // Plugin represents the config for a plugin. 29 | type Plugin struct { 30 | // the path to the plugin executable 31 | Path string 32 | Config map[string]interface{} 33 | template.Backend 34 | } 35 | 36 | // Connect creates the connection to the plugin and initializes it with the stored configuration. 37 | func (p *Plugin) Connect() (template.Backend, error) { 38 | if p == nil { 39 | return template.Backend{}, berr.ErrNilConfig 40 | } 41 | 42 | p.Backend.Name = path.Base(p.Path) 43 | 44 | client, err := pie.StartProviderCodec(jsonrpc.NewClientCodec, os.Stderr, p.Path) 45 | if err != nil { 46 | return p.Backend, err 47 | } 48 | 49 | plugin := &plug{client} 50 | if err := plugin.initPlugin(p.Config); err != nil { 51 | return p.Backend, err 52 | } 53 | 54 | p.Backend.ReadWatcher = plugin 55 | return p.Backend, nil 56 | } 57 | 58 | // return Backend config to allow modification before connect() for onetime param or similar 59 | func (p *Plugin) GetBackend() *template.Backend { 60 | return &p.Backend 61 | } 62 | 63 | // initPlugin sends the config map to the plugin 64 | // the plugin can then run some initialization tasks 65 | func (p *plug) initPlugin(config map[string]interface{}) error { 66 | var result bool 67 | return p.client.Call("Plugin.Init", config, &result) 68 | } 69 | 70 | // GetValues queries the plugin for keys 71 | func (p *plug) GetValues(keys []string) (result map[string]string, err error) { 72 | err = p.client.Call("Plugin.GetValues", keys, &result) 73 | return result, err 74 | } 75 | 76 | // Close closes the client connection 77 | func (p *plug) Close() { 78 | _ = p.client.Call("Plugin.Close", nil, nil) 79 | _ = p.client.Close() 80 | } 81 | 82 | // WatchConfig holds all data needed by the plugins WatchPrefix method. 83 | type WatchConfig struct { 84 | Prefix string 85 | Opts easykv.WatchOptions 86 | } 87 | 88 | func (p *plug) WatchPrefix(ctx context.Context, prefix string, opts ...easykv.WatchOption) (uint64, error) { 89 | var result uint64 90 | 91 | wc := WatchConfig{Prefix: prefix} 92 | for _, option := range opts { 93 | option(&wc.Opts) 94 | } 95 | 96 | errchan := make(chan error) 97 | go func() { 98 | select { 99 | case errchan <- p.client.Call("Plugin.WatchPrefix", wc, &result): 100 | case <-ctx.Done(): 101 | } 102 | }() 103 | 104 | select { 105 | case <-ctx.Done(): 106 | return wc.Opts.WaitIndex, nil 107 | case err := <-errchan: 108 | return result, err 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /pkg/backends/redis.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2016 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | package backends 10 | 11 | import ( 12 | "github.com/HeavyHorst/easykv/redis" 13 | berr "github.com/HeavyHorst/remco/pkg/backends/error" 14 | "github.com/HeavyHorst/remco/pkg/log" 15 | "github.com/HeavyHorst/remco/pkg/template" 16 | ) 17 | 18 | // RedisConfig represents the config for the redis backend. 19 | type RedisConfig struct { 20 | // A list of backend nodes. 21 | Nodes []string 22 | 23 | // A DNS server record to discover the redis nodes. 24 | SRVRecord SRVRecord `toml:"srv_record"` 25 | 26 | // The redis password. 27 | Password string 28 | 29 | // The redis database. 30 | Database int 31 | 32 | template.Backend 33 | } 34 | 35 | // Connect creates a new redisClient and fills the underlying template.Backend with the redis-Backend specific data. 36 | func (c *RedisConfig) Connect() (template.Backend, error) { 37 | if c == nil { 38 | return template.Backend{}, berr.ErrNilConfig 39 | } 40 | 41 | c.Backend.Name = "redis" 42 | 43 | // No nodes are set but a SRVRecord is provided 44 | if len(c.Nodes) == 0 && c.SRVRecord != "" { 45 | var err error 46 | c.Nodes, err = c.SRVRecord.GetNodesFromSRV("") 47 | if err != nil { 48 | return c.Backend, err 49 | } 50 | } 51 | 52 | log.WithFields( 53 | "backend", c.Backend.Name, 54 | "nodes", c.Nodes, 55 | ).Info("set backend nodes") 56 | 57 | client, err := redis.New(c.Nodes, redis.WithPassword(c.Password), redis.WithDatabase(c.Database)) 58 | if err != nil { 59 | return c.Backend, err 60 | } 61 | 62 | c.Backend.ReadWatcher = client 63 | 64 | if c.Backend.Watch { 65 | log.WithFields( 66 | "backend", c.Backend.Name, 67 | ).Warn("Watch is not supported, using interval instead") 68 | c.Backend.Watch = false 69 | } 70 | 71 | return c.Backend, nil 72 | } 73 | 74 | // return Backend config to allow modification before connect() for onetime param or similar 75 | func (c *RedisConfig) GetBackend() *template.Backend { 76 | return &c.Backend 77 | } 78 | -------------------------------------------------------------------------------- /pkg/backends/vault.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2016 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | package backends 10 | 11 | import ( 12 | "github.com/HeavyHorst/easykv/vault" 13 | berr "github.com/HeavyHorst/remco/pkg/backends/error" 14 | "github.com/HeavyHorst/remco/pkg/log" 15 | "github.com/HeavyHorst/remco/pkg/template" 16 | ) 17 | 18 | // VaultConfig represents the config for the vault backend. 19 | type VaultConfig struct { 20 | // The address of the vault server. 21 | Node string 22 | 23 | // The vault authentication type. 24 | // (token, approle, app-id, userpass, github, cert, kubernetes) 25 | AuthType string `toml:"auth_type"` 26 | 27 | // The vault app ID. 28 | // Only used with auth_type=app-id. 29 | AppID string `toml:"app_id"` 30 | // The vault user ID. 31 | // Only used with auth_type=app-id. 32 | UserID string `toml:"user_id"` 33 | 34 | // The vault RoleID. 35 | // Only used with auth_type=approle and kubernetes. 36 | RoleID string `toml:"role_id"` 37 | // The vault SecretID. 38 | // Only used with auth_type=approle. 39 | SecretID string `toml:"secret_id"` 40 | 41 | // The username for the userpass authentication. 42 | Username string 43 | // The password for the userpass authentication. 44 | Password string 45 | 46 | // The vault authentication token. Only used with auth_type=token and github. 47 | AuthToken string `toml:"auth_token"` 48 | 49 | ClientCert string `toml:"client_cert"` 50 | ClientKey string `toml:"client_key"` 51 | ClientCaKeys string `toml:"client_ca_keys"` 52 | template.Backend 53 | } 54 | 55 | // Connect creates a new vaultClient and fills the underlying template.Backend with the vault-Backend specific data. 56 | func (c *VaultConfig) Connect() (template.Backend, error) { 57 | if c == nil { 58 | return template.Backend{}, berr.ErrNilConfig 59 | } 60 | 61 | c.Backend.Name = "vault" 62 | log.WithFields( 63 | "backend", c.Backend.Name, 64 | "nodes", []string{c.Node}, 65 | ).Info("set backend nodes") 66 | 67 | tlsOps := vault.TLSOptions{ 68 | ClientCert: c.ClientCert, 69 | ClientKey: c.ClientKey, 70 | ClientCaKeys: c.ClientCaKeys, 71 | } 72 | 73 | authOps := vault.BasicAuthOptions{ 74 | Username: c.Username, 75 | Password: c.Password, 76 | } 77 | 78 | client, err := vault.New(c.Node, c.AuthType, 79 | vault.WithBasicAuth(authOps), 80 | vault.WithTLSOptions(tlsOps), 81 | vault.WithAppID(c.AppID), 82 | vault.WithUserID(c.UserID), 83 | vault.WithRoleID(c.RoleID), 84 | vault.WithSecretID(c.SecretID), 85 | vault.WithToken(c.AuthToken)) 86 | 87 | if err != nil { 88 | return c.Backend, err 89 | } 90 | 91 | c.Backend.ReadWatcher = client 92 | 93 | if c.Backend.Watch { 94 | log.WithFields( 95 | "backend", c.Backend.Name, 96 | ).Warn("Watch is not supported, using interval instead") 97 | c.Backend.Watch = false 98 | } 99 | 100 | return c.Backend, nil 101 | } 102 | 103 | // return Backend config to allow modification before connect() for onetime param or similar 104 | func (c *VaultConfig) GetBackend() *template.Backend { 105 | return &c.Backend 106 | } 107 | -------------------------------------------------------------------------------- /pkg/backends/zookeeper.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2016 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | package backends 10 | 11 | import ( 12 | "github.com/HeavyHorst/easykv/zookeeper" 13 | berr "github.com/HeavyHorst/remco/pkg/backends/error" 14 | "github.com/HeavyHorst/remco/pkg/log" 15 | "github.com/HeavyHorst/remco/pkg/template" 16 | ) 17 | 18 | // ZookeeperConfig represents the config for the consul backend. 19 | type ZookeeperConfig struct { 20 | // A list of zookeeper nodes. 21 | Nodes []string 22 | 23 | // A DNS server record to discover the zookeeper nodes. 24 | SRVRecord SRVRecord `toml:"srv_record"` 25 | template.Backend 26 | } 27 | 28 | // Connect creates a new zookeeperClient and fills the underlying template.Backend with the zookeeper-Backend specific data. 29 | func (c *ZookeeperConfig) Connect() (template.Backend, error) { 30 | if c == nil { 31 | return template.Backend{}, berr.ErrNilConfig 32 | } 33 | 34 | c.Backend.Name = "zookeeper" 35 | 36 | // No nodes are set but a SRVRecord is provided 37 | if len(c.Nodes) == 0 && c.SRVRecord != "" { 38 | var err error 39 | c.Nodes, err = c.SRVRecord.GetNodesFromSRV("") 40 | if err != nil { 41 | return c.Backend, err 42 | } 43 | } 44 | 45 | log.WithFields( 46 | "backend", c.Backend.Name, 47 | "nodes", c.Nodes, 48 | ).Info("set backend nodes") 49 | 50 | client, err := zookeeper.New(c.Nodes) 51 | if err != nil { 52 | return c.Backend, err 53 | } 54 | 55 | c.Backend.ReadWatcher = client 56 | return c.Backend, nil 57 | } 58 | 59 | // return Backend config to allow modification before connect() for onetime param or similar 60 | func (c *ZookeeperConfig) GetBackend() *template.Backend { 61 | return &c.Backend 62 | } 63 | -------------------------------------------------------------------------------- /pkg/log/log.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2016 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | package log 10 | 11 | import ( 12 | "fmt" 13 | "github.com/hashicorp/go-hclog" 14 | "os" 15 | "sync" 16 | ) 17 | 18 | var logger hclog.Logger 19 | var lock sync.RWMutex 20 | 21 | func init() { 22 | InitializeLogging("text", "info") 23 | } 24 | 25 | func InitializeLogging(format string, level string) { 26 | lock.Lock() 27 | defer lock.Unlock() 28 | 29 | logger = hclog. 30 | New(&hclog.LoggerOptions{JSONFormat: format == "json", Level: hclog.LevelFromString(level)}). 31 | With("prefix", fmt.Sprintf("%s[%d]", os.Args[0], os.Getpid())) 32 | } 33 | 34 | // Debug logs a message with severity DEBUG. 35 | func Debug(msg string, v ...interface{}) { 36 | logger.Debug(msg, v...) 37 | } 38 | 39 | // Error logs a message with severity ERROR. 40 | func Error(msg string, v ...interface{}) { 41 | logger.Error(msg, v...) 42 | } 43 | 44 | // Fatal logs a message with severity ERROR followed by a call to os.Exit(). 45 | func Fatal(msg string, v ...interface{}) { 46 | logger.Error(msg, v...) 47 | os.Exit(1) 48 | } 49 | 50 | // Info logs a message with severity INFO. 51 | func Info(msg string, v ...interface{}) { 52 | logger.Info(msg, v...) 53 | } 54 | 55 | // Warning logs a message with severity WARNING. 56 | func Warning(msg string, v ...interface{}) { 57 | logger.Warn(msg, v...) 58 | } 59 | 60 | func WithFields(v ...interface{}) hclog.Logger { 61 | return logger.With(v...) 62 | } 63 | -------------------------------------------------------------------------------- /pkg/log/log_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2016 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | package log 10 | 11 | import ( 12 | "bytes" 13 | "github.com/hashicorp/go-hclog" 14 | "strings" 15 | "testing" 16 | ) 17 | 18 | func TestSetLevel(t *testing.T) { 19 | text := &bytes.Buffer{} 20 | hclog.DefaultOutput = text 21 | 22 | InitializeLogging("text", "info") 23 | Warning("Warning message") 24 | Info("Info message") 25 | Debug("Debug message") 26 | 27 | lines1 := strings.Count(text.String(), "\n") 28 | 29 | text.Reset() 30 | InitializeLogging("text", "debug") 31 | 32 | Warning("Warning message") 33 | Info("Info message") 34 | Debug("Debug message") 35 | 36 | lines2 := strings.Count(text.String(), "\n") 37 | 38 | if lines1 >= lines2 { 39 | t.Error("Changing log level to debug failed") 40 | } 41 | } 42 | 43 | func TestSetFormatter(t *testing.T) { 44 | text := &bytes.Buffer{} 45 | json := &bytes.Buffer{} 46 | 47 | hclog.DefaultOutput = text 48 | InitializeLogging("text", "info") 49 | Info("Info message") 50 | Error("Error message") 51 | 52 | hclog.DefaultOutput = json 53 | InitializeLogging("json", "info") 54 | 55 | Info("Info message") 56 | Error("Error message") 57 | 58 | txtString := text.String() 59 | jsonString := json.String() 60 | if !(len(jsonString) > len(txtString)) { 61 | t.Errorf("JSON logging doesn't seem to work %s vs %s", jsonString, txtString) 62 | } 63 | //fmt.Printf("$$$$$$$$$%s vs %s \n", txtString, jsonString) 64 | InitializeLogging("text", "info") 65 | } 66 | -------------------------------------------------------------------------------- /pkg/telemetry/inmem.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2016 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | package telemetry 10 | 11 | import ( 12 | "time" 13 | 14 | "github.com/armon/go-metrics" 15 | ) 16 | 17 | // InmemSink represents inmem sink configuration 18 | type InmemSink struct { 19 | Interval int 20 | Retain int 21 | } 22 | 23 | // Creates a new inmem sink from config and registers DefaultInmemSignal signal (SIGUSR1) 24 | func (i *InmemSink) Init() (metrics.MetricSink, error) { 25 | if i == nil { 26 | return nil, ErrNilConfig 27 | } 28 | 29 | sink := metrics.NewInmemSink(time.Duration(i.Interval)*time.Second, time.Duration(i.Retain)*time.Second) 30 | metrics.DefaultInmemSignal(sink) 31 | return sink, nil 32 | } 33 | 34 | // Just returns nil 35 | func (i *InmemSink) Finalize() error { 36 | return nil 37 | } 38 | -------------------------------------------------------------------------------- /pkg/telemetry/prometheus.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2016 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | package telemetry 10 | 11 | import ( 12 | "context" 13 | "fmt" 14 | "net/http" 15 | "time" 16 | 17 | "github.com/HeavyHorst/remco/pkg/log" 18 | "github.com/armon/go-metrics" 19 | metricsPrometheus "github.com/armon/go-metrics/prometheus" 20 | "github.com/prometheus/client_golang/prometheus" 21 | "github.com/prometheus/client_golang/prometheus/promhttp" 22 | ) 23 | 24 | // PrometheusSink represents prometheus sink and prometheus stats endpoint configuration 25 | type PrometheusSink struct { 26 | Addr string 27 | Expiration int 28 | 29 | httpServer *http.Server 30 | prometheusSink *metricsPrometheus.PrometheusSink 31 | } 32 | 33 | // Creates a new prometheus sink from config and starts a goroutine with prometheus stats endpoint 34 | func (p *PrometheusSink) Init() (metrics.MetricSink, error) { 35 | if p == nil { 36 | return nil, ErrNilConfig 37 | } 38 | 39 | handler := http.NewServeMux() 40 | handler.Handle("/metrics", promhttp.Handler()) 41 | p.httpServer = &http.Server{Addr: p.Addr, Handler: handler} 42 | 43 | go func() { 44 | err := p.httpServer.ListenAndServe() 45 | if err != nil && err != http.ErrServerClosed { 46 | log.Error(fmt.Sprintf("error starting prometheus stats endpoint: %v", err)) 47 | } 48 | }() 49 | 50 | var err error 51 | p.prometheusSink, err = metricsPrometheus.NewPrometheusSinkFrom(metricsPrometheus.PrometheusOpts{ 52 | Expiration: time.Duration(p.Expiration) * time.Second, 53 | }) 54 | if err != nil { 55 | return nil, err 56 | } 57 | 58 | return p.prometheusSink, nil 59 | } 60 | 61 | // Unregisters prometheus sink and stops prometheus stats endpoint 62 | func (p *PrometheusSink) Finalize() error { 63 | if p == nil { 64 | return ErrNilConfig 65 | } 66 | 67 | prometheus.Unregister(p.prometheusSink) 68 | 69 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 70 | defer cancel() 71 | 72 | return p.httpServer.Shutdown(ctx) 73 | } 74 | -------------------------------------------------------------------------------- /pkg/telemetry/statsd.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2016 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | package telemetry 10 | 11 | import "github.com/armon/go-metrics" 12 | 13 | // StatsdSink represents statsd sink configuration 14 | type StatsdSink struct { 15 | Addr string 16 | } 17 | 18 | // Creates a new statsd sink from config 19 | func (s *StatsdSink) Init() (metrics.MetricSink, error) { 20 | if s == nil { 21 | return nil, ErrNilConfig 22 | } 23 | 24 | return metrics.NewStatsdSink(s.Addr) 25 | } 26 | 27 | // Just returns nil 28 | func (s *StatsdSink) Finalize() error { 29 | return nil 30 | } 31 | -------------------------------------------------------------------------------- /pkg/telemetry/statsite.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2016 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | package telemetry 10 | 11 | import "github.com/armon/go-metrics" 12 | 13 | // StatsiteSink represents statsite sink configuration 14 | type StatsiteSink struct { 15 | Addr string 16 | } 17 | 18 | // Creates a new statsite sink from config 19 | func (s *StatsiteSink) Init() (metrics.MetricSink, error) { 20 | if s == nil { 21 | return nil, ErrNilConfig 22 | } 23 | 24 | return metrics.NewStatsiteSink(s.Addr) 25 | } 26 | 27 | // Just returns nil 28 | func (s *StatsiteSink) Finalize() error { 29 | return nil 30 | } 31 | -------------------------------------------------------------------------------- /pkg/telemetry/telemetry.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2016 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | package telemetry 10 | 11 | import ( 12 | "errors" 13 | 14 | "github.com/HeavyHorst/remco/pkg/log" 15 | "github.com/armon/go-metrics" 16 | ) 17 | 18 | const defaultServiceName = "remco" 19 | 20 | // Every sink should implement this interface 21 | type Sink interface { 22 | Init() (metrics.MetricSink, error) 23 | Finalize() error 24 | } 25 | 26 | // ErrNilConfig is returned if Init is called on a nil Config 27 | var ErrNilConfig = errors.New("config is nil") 28 | 29 | // Telemetry represents telemetry configuration 30 | type Telemetry struct { 31 | Enabled bool 32 | ServiceName string `toml:"service_name"` 33 | HostName string 34 | EnableHostname bool `toml:"enable_hostname"` 35 | EnableHostnameLabel bool `toml:"enable_hostname_label"` 36 | EnableRuntimeMetrics bool `toml:"enable_runtime_metrics"` 37 | Sinks Sinks 38 | } 39 | 40 | // Configures metrics and adds FanoutSink with all configured sinks 41 | func (t Telemetry) Init() (*metrics.Metrics, error) { 42 | var ( 43 | m *metrics.Metrics 44 | err error 45 | ) 46 | if t.Enabled { 47 | log.Info("enabling telemetry") 48 | serviceName := defaultServiceName 49 | if t.ServiceName != "" { 50 | serviceName = t.ServiceName 51 | } 52 | metricsConf := metrics.DefaultConfig(serviceName) 53 | if t.HostName != "" { 54 | metricsConf.HostName = t.HostName 55 | } 56 | metricsConf.EnableHostname = t.EnableHostname 57 | metricsConf.EnableRuntimeMetrics = t.EnableRuntimeMetrics 58 | metricsConf.EnableHostnameLabel = t.EnableHostnameLabel 59 | var sinks metrics.FanoutSink 60 | for _, sc := range t.Sinks.GetSinks() { 61 | sink, err := sc.Init() 62 | if err == nil { 63 | sinks = append(sinks, sink) 64 | } else if err != ErrNilConfig { 65 | return nil, err 66 | } 67 | } 68 | m, err = metrics.NewGlobal(metricsConf, sinks) 69 | if err != nil { 70 | return nil, err 71 | } 72 | } 73 | 74 | return m, nil 75 | } 76 | 77 | // Finalizes all configured sinks 78 | func (t Telemetry) Stop() error { 79 | for _, sc := range t.Sinks.GetSinks() { 80 | err := sc.Finalize() 81 | if err != nil && err != ErrNilConfig { 82 | return err 83 | } 84 | } 85 | 86 | return nil 87 | } 88 | 89 | // Sinks represent sinks configuration 90 | type Sinks struct { 91 | Inmem *InmemSink 92 | Statsd *StatsdSink 93 | Statsite *StatsiteSink 94 | Prometheus *PrometheusSink 95 | } 96 | 97 | // GetSinks returns a slice with all Sinks for easy iteration. 98 | func (c *Sinks) GetSinks() []Sink { 99 | return []Sink{ 100 | c.Inmem, 101 | c.Statsd, 102 | c.Statsite, 103 | c.Prometheus, 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /pkg/telemetry/telemetry_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2016 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | package telemetry 10 | 11 | import ( 12 | "io/ioutil" 13 | "net/http" 14 | "testing" 15 | "time" 16 | 17 | . "gopkg.in/check.v1" 18 | ) 19 | 20 | // Hook up gocheck into the "go test" runner. 21 | func Test(t *testing.T) { TestingT(t) } 22 | 23 | type TelemetryTestSuite struct { 24 | telemetry Telemetry 25 | } 26 | 27 | var _ = Suite(&TelemetryTestSuite{}) 28 | 29 | func (s *TelemetryTestSuite) SetUpSuite(t *C) { 30 | s.telemetry = Telemetry{ 31 | Enabled: true, 32 | ServiceName: "mock", 33 | HostName: "test-hostname", 34 | EnableHostname: false, 35 | EnableRuntimeMetrics: false, 36 | Sinks: Sinks{ 37 | Inmem: &InmemSink{ 38 | Interval: 10, 39 | Retain: 60, 40 | }, 41 | Statsd: &StatsdSink{ 42 | Addr: "127.0.0.1:7524", 43 | }, 44 | Statsite: &StatsiteSink{ 45 | Addr: "localhost:7523", 46 | }, 47 | Prometheus: &PrometheusSink{ 48 | Addr: "127.0.0.1:2112", 49 | Expiration: 600, 50 | }, 51 | }, 52 | } 53 | } 54 | 55 | func (s *TelemetryTestSuite) TestInit(t *C) { 56 | m, err := s.telemetry.Init() 57 | t.Assert(err, IsNil) 58 | t.Assert(m, NotNil) 59 | t.Assert(m.ServiceName, Equals, "mock") 60 | 61 | m.AddSample([]string{"test_sample"}, 42) 62 | 63 | // Wait for the metrics server to start listening 64 | time.Sleep(1 * time.Second) 65 | resp, err := http.Get("http://127.0.0.1:2112/metrics") 66 | t.Assert(err, IsNil) 67 | 68 | defer resp.Body.Close() 69 | body, err := ioutil.ReadAll(resp.Body) 70 | t.Assert(err, IsNil) 71 | 72 | t.Assert(string(body), Matches, "(?s).*mock_test_sample.*") 73 | t.Assert(string(body), Not(Matches), "(?s).*_runtime_.*") 74 | t.Assert(string(body), Not(Matches), "(?s).*test_hostname.*") 75 | s.telemetry.Stop() 76 | } 77 | 78 | func (s *TelemetryTestSuite) TestStop(t *C) { 79 | s.telemetry.Init() 80 | err := s.telemetry.Stop() 81 | t.Assert(err, IsNil) 82 | resp, err := http.Get("http://127.0.0.1:2112/metrics") 83 | t.Assert(err, NotNil) 84 | t.Assert(resp, IsNil) 85 | } 86 | 87 | func (s *TelemetryTestSuite) TestReInit(t *C) { 88 | s.telemetry.Init() 89 | err := s.telemetry.Stop() 90 | t.Assert(err, IsNil) 91 | s.telemetry.ServiceName = "mock2" 92 | m2, err := s.telemetry.Init() 93 | t.Assert(err, IsNil) 94 | t.Assert(m2, NotNil) 95 | t.Assert(m2.ServiceName, Equals, "mock2") 96 | s.telemetry.Stop() 97 | } 98 | -------------------------------------------------------------------------------- /pkg/template/backend.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2016 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | package template 10 | 11 | import ( 12 | "context" 13 | "time" 14 | 15 | "github.com/HeavyHorst/easykv" 16 | "github.com/HeavyHorst/memkv" 17 | berr "github.com/HeavyHorst/remco/pkg/backends/error" 18 | "github.com/HeavyHorst/remco/pkg/log" 19 | ) 20 | 21 | // A BackendConnector - Every backend implements this interface. 22 | // 23 | // If Connect is called a new connection to the underlaying kv-store will be established. 24 | // 25 | // Connect should also set the name and the StoreClient of the Backend. The other values of Backend will be loaded from the configuration file. 26 | type BackendConnector interface { 27 | Connect() (Backend, error) 28 | GetBackend() *Backend 29 | } 30 | 31 | // Backend is the representation of a template backend like etcd or consul 32 | type Backend struct { 33 | easykv.ReadWatcher 34 | 35 | // Name is the name of the backend for example etcd or consul. 36 | // The name is attached to the logs. 37 | Name string 38 | 39 | // Onetime - render the config file and quit. 40 | Onetime bool 41 | 42 | // Enable/Disable watch support. 43 | Watch bool 44 | 45 | // Watch only these keys 46 | WatchKeys []string 47 | 48 | // The key-path prefix. 49 | Prefix string 50 | 51 | // The backend polling interval. Can be used as a reconciliation loop for watch or standalone. 52 | Interval int 53 | 54 | // The backend keys that the template requires to be rendered correctly. 55 | Keys []string 56 | 57 | store *memkv.Store 58 | } 59 | 60 | // connectAllBackends connects to all configured backends. 61 | // This method blocks until a connection to every backend has been established or the context is canceled. 62 | func connectAllBackends(ctx context.Context, bc []BackendConnector) ([]Backend, error) { 63 | var backendList []Backend 64 | for _, config := range bc { 65 | retryloop: 66 | for { 67 | select { 68 | case <-ctx.Done(): 69 | for _, be := range backendList { 70 | be.Close() 71 | } 72 | return backendList, ctx.Err() 73 | default: 74 | b, err := config.Connect() 75 | if err == nil { 76 | backendList = append(backendList, b) 77 | } else if err != berr.ErrNilConfig { 78 | log.WithFields( 79 | "backend", b.Name, 80 | "error", err, 81 | ).Error("connect failed") 82 | 83 | //try again after 2 seconds to watch 84 | if config.GetBackend().Onetime != true { 85 | time.Sleep(2 * time.Second) 86 | continue retryloop 87 | } 88 | } 89 | break retryloop 90 | } 91 | } 92 | } 93 | 94 | return backendList, nil 95 | } 96 | 97 | func (s Backend) watch(ctx context.Context, processChan chan Backend, errChan chan berr.BackendError) { 98 | if s.Onetime { 99 | return 100 | } 101 | 102 | var lastIndex uint64 103 | keysPrefix := appendPrefix(s.Prefix, s.Keys) 104 | if len(s.WatchKeys) > 0 { 105 | keysPrefix = appendPrefix(s.Prefix, s.WatchKeys) 106 | } 107 | 108 | var backendError bool 109 | 110 | for { 111 | select { 112 | case <-ctx.Done(): 113 | return 114 | default: 115 | if backendError { 116 | processChan <- s 117 | backendError = false 118 | } 119 | 120 | index, err := s.WatchPrefix(ctx, s.Prefix, easykv.WithKeys(keysPrefix), easykv.WithWaitIndex(lastIndex)) 121 | if err != nil { 122 | if err != easykv.ErrWatchCanceled { 123 | backendError = true 124 | errChan <- berr.BackendError{Message: err.Error(), Backend: s.Name} 125 | time.Sleep(2 * time.Second) 126 | } 127 | continue 128 | } 129 | processChan <- s 130 | lastIndex = index 131 | } 132 | } 133 | } 134 | 135 | func (s Backend) interval(ctx context.Context, processChan chan Backend) { 136 | if s.Onetime { 137 | return 138 | } 139 | for { 140 | select { 141 | case <-ctx.Done(): 142 | return 143 | case <-time.After(time.Duration(s.Interval) * time.Second): 144 | processChan <- s 145 | } 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /pkg/template/executor.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2016 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | package template 10 | 11 | import ( 12 | "context" 13 | "fmt" 14 | "github.com/hashicorp/go-hclog" 15 | "os" 16 | "syscall" 17 | "time" 18 | 19 | "github.com/hashicorp/consul-template/child" 20 | "github.com/hashicorp/consul-template/signals" 21 | "github.com/mattn/go-shellwords" 22 | "github.com/pkg/errors" 23 | ) 24 | 25 | // ExecConfig represents the configuration values for the exec mode. 26 | type ExecConfig struct { 27 | // Command is the command to execute. // Backtick parsing is supported: 28 | // ./foo `echo $SHELL` 29 | Command string `json:"command"` 30 | 31 | // ReloadSignal is the Signal that is sended to the subpocess if we want to reload it. 32 | // If no signal is specified the child process will be killed (gracefully) and started again. 33 | ReloadSignal string `toml:"reload_signal" json:"reload_signal"` 34 | 35 | // KillSignal defines the signal sent to the child process when remco is gracefully shutting down. 36 | // 37 | // The application needs to exit before the kill_timeout, it will be terminated otherwise (like kill -9). 38 | // The default value is “SIGTERM”. 39 | KillSignal string `toml:"kill_signal" json:"kill_signal"` 40 | 41 | // KillTimeout - the maximum amount of time in seconds to wait for the child process to gracefully terminate. 42 | KillTimeout int `toml:"kill_timeout" json:"kill_timeout"` 43 | 44 | // A random splay to wait before killing the command. 45 | // May be useful in large clusters to prevent all child processes to reload at the same time when configuration changes occur. 46 | Splay int `json:"splay"` 47 | } 48 | 49 | type childSignal struct { 50 | signal os.Signal 51 | err chan<- error 52 | } 53 | 54 | type exitC struct { 55 | exitChan <-chan int 56 | valid bool 57 | } 58 | 59 | // An Executor controls a subprocess. 60 | // It can control the whole process lifecycle and can 61 | // reload and stop the process gracefully or send other signals to 62 | // the child process using channels. 63 | type Executor struct { 64 | execCommand string 65 | reloadSignal os.Signal 66 | killSignal os.Signal 67 | killTimeout time.Duration 68 | splay time.Duration 69 | logger hclog.Logger 70 | 71 | stopChan chan chan<- error 72 | reloadChan chan chan<- error 73 | signalChan chan childSignal 74 | exitChan chan chan exitC 75 | } 76 | 77 | // NewExecutor creates a new Executor. 78 | func NewExecutor(execCommand, reloadSignal, killSignal string, killTimeout, splay int, logger hclog.Logger) Executor { 79 | var rs, ks os.Signal 80 | var err error 81 | 82 | if logger == nil { 83 | logger = hclog.Default() 84 | } 85 | 86 | if reloadSignal != "" { 87 | rs, err = signals.Parse(reloadSignal) 88 | if err != nil { 89 | logger.Error("failed to parse reload signal", err) 90 | } 91 | } 92 | 93 | // default killSignal is SIGTERM 94 | if killSignal == "" { 95 | killSignal = "SIGTERM" 96 | } 97 | ks, err = signals.Parse(killSignal) 98 | if err != nil { 99 | logger.Error("failed to parse kill signal", err) 100 | ks = syscall.SIGTERM 101 | } 102 | 103 | // set killTimeout to 10 if its not defined 104 | if killTimeout == 0 { 105 | killTimeout = 10 106 | } 107 | 108 | return Executor{ 109 | execCommand: execCommand, 110 | reloadSignal: rs, 111 | killSignal: ks, 112 | killTimeout: time.Duration(killTimeout) * time.Second, 113 | splay: time.Duration(splay) * time.Second, 114 | logger: logger, 115 | stopChan: make(chan chan<- error), 116 | reloadChan: make(chan chan<- error), 117 | signalChan: make(chan childSignal), 118 | exitChan: make(chan chan exitC), 119 | } 120 | } 121 | 122 | // SpawnChild parses e.execCommand and starts the child process accordingly. 123 | // Backtick parsing is supported: 124 | // ./foo `echo $SHELL` 125 | // 126 | // only call this once ! 127 | func (e *Executor) SpawnChild() error { 128 | var c *child.Child 129 | if e.execCommand != "" { 130 | p := shellwords.NewParser() 131 | p.ParseBacktick = true 132 | args, err := p.Parse(e.execCommand) 133 | if err != nil { 134 | return err 135 | } 136 | 137 | c, err = child.New(&child.NewInput{ 138 | Stdin: os.Stdin, 139 | Stdout: os.Stdout, 140 | Stderr: os.Stderr, 141 | Command: args[0], 142 | Args: args[1:], 143 | ReloadSignal: e.reloadSignal, 144 | KillSignal: e.killSignal, 145 | KillTimeout: e.killTimeout, 146 | Splay: e.splay, 147 | Logger: e.logger.StandardLogger(&hclog.StandardLoggerOptions{InferLevels: true}), 148 | }) 149 | 150 | if err != nil { 151 | return fmt.Errorf("error creating child: %s", err) 152 | } 153 | //e.child = child 154 | if err := c.Start(); err != nil { 155 | return fmt.Errorf("error starting child: %s", err) 156 | } 157 | } 158 | 159 | go func() { 160 | for { 161 | select { 162 | case errchan := <-e.stopChan: 163 | if c != nil { 164 | c.Stop() 165 | } 166 | errchan <- nil 167 | return 168 | case errchan := <-e.reloadChan: 169 | var err error 170 | if c != nil { 171 | err = c.Reload() 172 | } 173 | errchan <- err 174 | case s := <-e.signalChan: 175 | var err error 176 | if c != nil { 177 | err = c.Signal(s.signal) 178 | } 179 | s.err <- err 180 | case exit := <-e.exitChan: 181 | if c != nil { 182 | ex := exitC{ 183 | valid: true, 184 | exitChan: c.ExitCh(), 185 | } 186 | exit <- ex 187 | } else { 188 | ex := exitC{ 189 | valid: false, 190 | } 191 | exit <- ex 192 | } 193 | } 194 | } 195 | }() 196 | 197 | return nil 198 | } 199 | 200 | // SignalChild forwards the os.Signal to the child process. 201 | func (e *Executor) SignalChild(s os.Signal) error { 202 | err := make(chan error) 203 | 204 | signal := childSignal{ 205 | signal: s, 206 | err: err, 207 | } 208 | 209 | e.signalChan <- signal 210 | return <-err 211 | 212 | } 213 | 214 | // StopChild stops the child process. 215 | // 216 | // It blocks until the child quits or the killTimeout is reached. 217 | // The child will be killed if it takes longer than killTimeout to stop it. 218 | func (e *Executor) StopChild() { 219 | errchan := make(chan error) 220 | e.stopChan <- errchan 221 | <-errchan 222 | } 223 | 224 | // Reload reloads the child process. 225 | // If a reloadSignal is provided it will send this signal to the child. 226 | // The child process will be killed and restarted otherwise. 227 | func (e *Executor) Reload() error { 228 | errchan := make(chan error) 229 | e.reloadChan <- errchan 230 | if err := <-errchan; err != nil { 231 | return errors.Wrap(err, "reload failed") 232 | } 233 | return nil 234 | } 235 | 236 | func (e *Executor) getExitChan() (<-chan int, bool) { 237 | ecc := make(chan exitC) 238 | e.exitChan <- ecc 239 | exit := <-ecc 240 | return exit.exitChan, exit.valid 241 | } 242 | 243 | // Wait waits for the child to stop. 244 | // Returns true if the command stops unexpectedly and false if the context is canceled. 245 | // 246 | // Wait ignores reloads. 247 | func (e *Executor) Wait(ctx context.Context) bool { 248 | exitChan, valid := e.getExitChan() 249 | if !valid { 250 | return false 251 | } 252 | 253 | for { 254 | select { 255 | case <-ctx.Done(): 256 | return false 257 | case <-exitChan: 258 | // wait a little bit to give the process time to start 259 | // in case of a reload 260 | time.Sleep(1 * time.Second) 261 | nexitChan, _ := e.getExitChan() 262 | // the exitChan has changed which means the process was reloaded 263 | // don't exit in this case 264 | if nexitChan != exitChan { 265 | exitChan = nexitChan 266 | continue 267 | } 268 | // the process exited - stop 269 | return true 270 | } 271 | } 272 | } 273 | -------------------------------------------------------------------------------- /pkg/template/executor_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2016 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | package template 10 | 11 | import ( 12 | "bytes" 13 | "context" 14 | "github.com/hashicorp/go-hclog" 15 | "io/ioutil" 16 | "os" 17 | "syscall" 18 | "testing" 19 | "time" 20 | ) 21 | 22 | func TestNew(t *testing.T) { 23 | command := "echo" 24 | reloadSignal := "SIGHUP" 25 | killSignal := "SIGKILL" 26 | killTimeout := 1 27 | splay := 0 28 | logger := hclog.Default() 29 | 30 | exec := NewExecutor(command, reloadSignal, killSignal, killTimeout, splay, logger) 31 | 32 | if exec.killSignal != os.Kill { 33 | t.Errorf("killSignal should be: %v", os.Kill) 34 | } 35 | 36 | if exec.reloadSignal != syscall.SIGHUP { 37 | t.Errorf("reloadSignal should be: %v", syscall.SIGHUP) 38 | } 39 | 40 | if exec.execCommand != command { 41 | t.Errorf("execCommand should be: %s", command) 42 | } 43 | 44 | if exec.killTimeout != time.Duration(killTimeout)*time.Second { 45 | t.Errorf("killTimeout should be: %v", time.Duration(killTimeout)*time.Second) 46 | } 47 | 48 | if exec.splay != time.Duration(splay)*time.Second { 49 | t.Errorf("splay should be: %v", time.Duration(splay)*time.Second) 50 | } 51 | } 52 | 53 | func TestNewDefaults(t *testing.T) { 54 | exec := NewExecutor("", "", "", 0, 0, hclog.Default()) 55 | 56 | if exec.killSignal != syscall.SIGTERM { 57 | t.Errorf("default killSignal should be: %v", syscall.SIGTERM) 58 | } 59 | 60 | if exec.reloadSignal != nil { 61 | t.Error("default reloadSignal should be nil") 62 | } 63 | 64 | if exec.killTimeout != 10*time.Second { 65 | t.Errorf("default killTimeout should be: %v", 10*time.Second) 66 | } 67 | } 68 | 69 | func TestNewInvalidSignals(t *testing.T) { 70 | logger := hclog.New(&hclog.LoggerOptions{Output: ioutil.Discard}) 71 | exec := NewExecutor("", "SIGBLA", "SIGBLA", 0, 0, logger) 72 | 73 | if exec.reloadSignal != nil { 74 | t.Error("reloadSignal should be nil") 75 | } 76 | 77 | if exec.killSignal != syscall.SIGTERM { 78 | t.Errorf("killSignal should be: %v", syscall.SIGTERM) 79 | } 80 | } 81 | 82 | func spawnChild(command string) (Executor, error) { 83 | reloadSignal := "SIGINT" 84 | killSignal := "SIGTERM" 85 | killTimeout := 2 86 | logger := hclog.New(&hclog.LoggerOptions{Output: ioutil.Discard}) 87 | 88 | exec := NewExecutor(command, reloadSignal, killSignal, killTimeout, 0, logger) 89 | err := exec.SpawnChild() 90 | 91 | return exec, err 92 | } 93 | 94 | func spawnTrapChild() (Executor, error) { 95 | command := `bash -c "trap ' ' SIGINT SIGTERM; while true; do sleep 10; done"` 96 | return spawnChild(command) 97 | } 98 | 99 | func spawnTimeOutChild() (Executor, error) { 100 | command := "bash -c 'sleep 5'" 101 | return spawnChild(command) 102 | } 103 | 104 | func TestStopChildTimeOut(t *testing.T) { 105 | exec, err := spawnTrapChild() 106 | if err != nil { 107 | t.Error(err) 108 | } 109 | time.Sleep(1 * time.Second) 110 | 111 | ticker := time.Tick(4 * time.Second) 112 | stopped := make(chan struct{}) 113 | 114 | go func() { 115 | exec.StopChild() 116 | stopped <- struct{}{} 117 | }() 118 | 119 | select { 120 | case <-ticker: 121 | t.Error("killTimeout is not working") 122 | case <-stopped: 123 | return 124 | } 125 | } 126 | 127 | func TestWait(t *testing.T) { 128 | exec, err := spawnTimeOutChild() 129 | if err != nil { 130 | t.Error(err) 131 | } 132 | time.Sleep(1 * time.Second) 133 | 134 | nc := make(chan bool) 135 | go func() { 136 | s := exec.Wait(context.Background()) 137 | nc <- s 138 | }() 139 | 140 | if !<-nc { 141 | t.Error("the context was not canceled, should be true") 142 | } 143 | } 144 | 145 | func TestWaitCancel(t *testing.T) { 146 | exec, err := spawnTimeOutChild() 147 | if err != nil { 148 | t.Error(err) 149 | } 150 | time.Sleep(1 * time.Second) 151 | 152 | nc := make(chan bool) 153 | go func() { 154 | ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) 155 | defer cancel() 156 | s := exec.Wait(ctx) 157 | nc <- s 158 | }() 159 | 160 | if <-nc { 161 | t.Error("the context was canceled, should be false") 162 | } 163 | 164 | exec.StopChild() 165 | } 166 | 167 | func TestReload(t *testing.T) { 168 | command := "bash -c 'sleep 5'" 169 | logger := hclog.New(&hclog.LoggerOptions{Output: ioutil.Discard}) 170 | 171 | exec := NewExecutor(command, "", "", 0, 0, logger) 172 | err := exec.SpawnChild() 173 | if err != nil { 174 | t.Error(err) 175 | } 176 | 177 | // exitChan := exec.child.ExitCh() 178 | exitChan, valid := exec.getExitChan() 179 | if !valid { 180 | t.Error("we should have a valid exitChan") 181 | } 182 | 183 | nc := make(chan bool) 184 | ctx, cancel := context.WithCancel(context.Background()) 185 | defer cancel() 186 | go func() { 187 | s := exec.Wait(ctx) 188 | nc <- s 189 | }() 190 | 191 | ticker := time.Tick(2 * time.Second) 192 | 193 | err = exec.Reload() 194 | if err != nil { 195 | t.Error(err) 196 | } 197 | 198 | select { 199 | case <-nc: 200 | t.Error("exec.Wait returned, that should never happen on a reload") 201 | case <-ticker: 202 | } 203 | 204 | // should be different after the reload 205 | nexitChan, valid := exec.getExitChan() 206 | if !valid { 207 | t.Error("we should have a valid exitChan") 208 | } 209 | if exitChan == nexitChan { 210 | t.Error("reload failed") 211 | } 212 | 213 | exec.StopChild() 214 | } 215 | 216 | func TestSignalChild(t *testing.T) { 217 | command := "bash -c 'sleep 5'" 218 | logger := hclog.New(&hclog.LoggerOptions{Output: ioutil.Discard}) 219 | 220 | exec := NewExecutor(command, "", "", 0, 0, logger) 221 | err := exec.SpawnChild() 222 | if err != nil { 223 | t.Error(err) 224 | } 225 | 226 | nc := make(chan bool) 227 | go func() { 228 | s := exec.Wait(context.Background()) 229 | nc <- s 230 | }() 231 | 232 | err = exec.SignalChild(os.Interrupt) 233 | if err != nil { 234 | t.Error(err) 235 | } 236 | 237 | // the program should exit when it receives the os.Interrupt 238 | n := <-nc 239 | if !n { 240 | t.Error("the context wasn't canceled, exec.Wait should have returned true") 241 | } 242 | 243 | exec.StopChild() 244 | } 245 | 246 | func TestExecutorLogging(t *testing.T) { 247 | command := "bash -c 'echo \"Hello\"'" 248 | var buf bytes.Buffer 249 | logger := hclog.New(&hclog.LoggerOptions{Output: &buf, DisableTime: true}).With("k1", "v1", "k2", "v2") 250 | 251 | exec := NewExecutor(command, "", "", 0, 0, logger) 252 | err := exec.SpawnChild() 253 | if err != nil { 254 | t.Error(err) 255 | } 256 | exec.StopChild() 257 | s := buf.String() 258 | if s != `[INFO] (child) spawning: bash -c echo "Hello": k1=v1 k2=v2 259 | [INFO] (child) stopping process: k1=v1 k2=v2 260 | ` { 261 | t.Errorf("Log output did not match out expectations. Was '%s'", s) 262 | } 263 | } 264 | -------------------------------------------------------------------------------- /pkg/template/fileutil/fileStat_posix.go: -------------------------------------------------------------------------------- 1 | // +build !windows 2 | 3 | /* 4 | * This file is part of remco. 5 | * Based on code from confd. 6 | * https://github.com/kelseyhightower/confd/blob/abba746a0cb7c8cb5fe135fa2d884ea3c4a5f666/resource/template/util.go 7 | * © 2013 Kelsey Hightower 8 | * © 2016 The Remco Authors 9 | * 10 | * For the full copyright and license information, please view the LICENSE 11 | * file that was distributed with this source code. 12 | */ 13 | 14 | package fileutil 15 | 16 | import ( 17 | hash "crypto/sha1" 18 | 19 | "github.com/pkg/errors" 20 | 21 | "fmt" 22 | "io" 23 | "os" 24 | "syscall" 25 | ) 26 | 27 | // stat return a fileInfo describing the named file. 28 | func stat(name string) (fi fileInfo, err error) { 29 | if IsFileExist(name) { 30 | f, err := os.Open(name) 31 | if err != nil { 32 | return fi, errors.Wrap(err, "open file failed") 33 | } 34 | defer f.Close() 35 | stats, _ := f.Stat() 36 | fi.Uid = stats.Sys().(*syscall.Stat_t).Uid 37 | fi.Gid = stats.Sys().(*syscall.Stat_t).Gid 38 | fi.Mode = stats.Mode() 39 | h := hash.New() 40 | io.Copy(h, f) 41 | fi.Hash = fmt.Sprintf("%x", h.Sum(nil)) 42 | return fi, nil 43 | } 44 | return fi, fmt.Errorf("file not found") 45 | } 46 | -------------------------------------------------------------------------------- /pkg/template/fileutil/fileStat_windows.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * Based on code from confd. 4 | * https://github.com/kelseyhightower/confd/blob/abba746a0cb7c8cb5fe135fa2d884ea3c4a5f666/resource/template/util.go 5 | * © 2013 Kelsey Hightower 6 | * © 2016 The Remco Authors 7 | * 8 | * For the full copyright and license information, please view the LICENSE 9 | * file that was distributed with this source code. 10 | */ 11 | 12 | package fileutil 13 | 14 | import ( 15 | hash "crypto/sha1" 16 | 17 | "github.com/pkg/errors" 18 | 19 | "fmt" 20 | "io" 21 | "os" 22 | ) 23 | 24 | // stat return a fileInfo describing the named file. 25 | func stat(name string) (fi fileInfo, err error) { 26 | if IsFileExist(name) { 27 | f, err := os.Open(name) 28 | defer f.Close() 29 | if err != nil { 30 | return fi, errors.Wrap(err, "open file failed") 31 | } 32 | stats, _ := f.Stat() 33 | fi.Mode = stats.Mode() 34 | h := hash.New() 35 | io.Copy(h, f) 36 | fi.Hash = fmt.Sprintf("%x", h.Sum(nil)) 37 | return fi, nil 38 | } 39 | return fi, fmt.Errorf("file not found") 40 | } 41 | -------------------------------------------------------------------------------- /pkg/template/fileutil/fileutil.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * Based on code from confd. 4 | * https://github.com/kelseyhightower/confd/blob/abba746a0cb7c8cb5fe135fa2d884ea3c4a5f666/resource/template/util.go 5 | * © 2013 Kelsey Hightower 6 | * © 2016 The Remco Authors 7 | * 8 | * For the full copyright and license information, please view the LICENSE 9 | * file that was distributed with this source code. 10 | */ 11 | 12 | package fileutil 13 | 14 | import ( 15 | "github.com/hashicorp/go-hclog" 16 | "io/ioutil" 17 | "os" 18 | "strings" 19 | 20 | "github.com/pkg/errors" 21 | ) 22 | 23 | // FileInfo describes a configuration file and is returned by filestat. 24 | type fileInfo struct { 25 | Uid uint32 26 | Gid uint32 27 | Mode os.FileMode 28 | Hash string 29 | } 30 | 31 | // IsFileExist reports whether path exits. 32 | func IsFileExist(fpath string) bool { 33 | if _, err := os.Stat(fpath); os.IsNotExist(err) { 34 | return false 35 | } 36 | return true 37 | } 38 | 39 | // ReplaceFile replaces dest with src. 40 | // 41 | // ReplaceFile just renames (move) the file if possible. 42 | // If that fails it will read the src file and write the content to the destination file. 43 | // It returns an error if any. 44 | func ReplaceFile(src, dest string, mode os.FileMode, logger hclog.Logger) error { 45 | err := os.Rename(src, dest) 46 | if err != nil { 47 | if strings.Contains(err.Error(), "device or resource busy") { 48 | logger.Debug("Rename failed - target is likely a mount. Trying to write instead") 49 | // try to open the file and write to it 50 | var contents []byte 51 | var rerr error 52 | contents, rerr = ioutil.ReadFile(src) 53 | if rerr != nil { 54 | return errors.Wrap(rerr, "couldn't read source file") 55 | } 56 | err := ioutil.WriteFile(dest, contents, mode) 57 | if err != nil { 58 | return errors.Wrap(rerr, "couldn't write destination file") 59 | } 60 | } else { 61 | return errors.Wrap(err, "couldn't rename src -> dst") 62 | } 63 | } 64 | return nil 65 | } 66 | 67 | // SameFile reports whether src and dest config files are equal. 68 | // Two config files are equal when they have the same file contents and 69 | // Unix permissions. The owner, group, and mode must match. 70 | // It return false in other cases. 71 | func SameFile(src, dest string, logger hclog.Logger) (bool, error) { 72 | if !IsFileExist(dest) { 73 | return false, nil 74 | } 75 | d, err := stat(dest) 76 | if err != nil { 77 | return false, err 78 | } 79 | s, err := stat(src) 80 | if err != nil { 81 | return false, err 82 | } 83 | if d.Uid != s.Uid { 84 | logger.With( 85 | "config", dest, 86 | "current", d.Uid, 87 | "new", s.Uid, 88 | ).Info("wrong UID") 89 | } 90 | if d.Gid != s.Gid { 91 | logger.With( 92 | "config", dest, 93 | "current", d.Gid, 94 | "new", s.Gid, 95 | ).Info("wrong GID") 96 | } 97 | if d.Mode != s.Mode { 98 | logger.With( 99 | "config", dest, 100 | "current", d.Mode, 101 | "new", s.Mode, 102 | ).Info("wrong filemode") 103 | } 104 | if d.Hash != s.Hash { 105 | logger.With( 106 | "config", dest, 107 | "current", d.Hash, 108 | "new", s.Hash, 109 | ).Info("wrong hashsum") 110 | } 111 | if d.Uid != s.Uid || d.Gid != s.Gid || d.Mode != s.Mode || d.Hash != s.Hash { 112 | return false, nil 113 | } 114 | return true, nil 115 | } 116 | -------------------------------------------------------------------------------- /pkg/template/fileutil/fileutil_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2016 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | package fileutil 10 | 11 | import ( 12 | "github.com/hashicorp/go-hclog" 13 | "io/ioutil" 14 | "os" 15 | "testing" 16 | 17 | . "gopkg.in/check.v1" 18 | ) 19 | 20 | // Hook up gocheck into the "go test" runner. 21 | func Test(t *testing.T) { TestingT(t) } 22 | 23 | type TestSuite struct { 24 | file *os.File 25 | sameFile *os.File 26 | differentHash *os.File 27 | 28 | replaceFile *os.File 29 | replaceFile1 *os.File 30 | } 31 | 32 | var _ = Suite(&TestSuite{}) 33 | 34 | func writeFile(t *C, file *os.File, value string) { 35 | _, err := file.WriteString(value) 36 | if err != nil { 37 | t.Error(err.Error()) 38 | } 39 | } 40 | 41 | func createTempFile(t *C, name string) *os.File { 42 | tempFile, err := ioutil.TempFile("", name) 43 | if err != nil { 44 | t.Error(err.Error()) 45 | } 46 | return tempFile 47 | } 48 | 49 | func (s *TestSuite) SetUpSuite(t *C) { 50 | src := createTempFile(t, "src") 51 | defer src.Close() 52 | 53 | dst := createTempFile(t, "dest") 54 | defer dst.Close() 55 | 56 | differentHash := createTempFile(t, "hash") 57 | defer differentHash.Close() 58 | 59 | replaceFile := createTempFile(t, "replace") 60 | defer replaceFile.Close() 61 | replaceFile1 := createTempFile(t, "replace1") 62 | defer replaceFile1.Close() 63 | 64 | writeFile(t, src, "bla") 65 | writeFile(t, dst, "bla") 66 | writeFile(t, replaceFile, "bla") 67 | writeFile(t, differentHash, "mmmh lecker Gurkensalat!") 68 | 69 | s.file = src 70 | s.sameFile = dst 71 | s.differentHash = differentHash 72 | s.replaceFile = replaceFile 73 | s.replaceFile1 = replaceFile1 74 | } 75 | 76 | func (s *TestSuite) TearDownSuite(t *C) { 77 | os.Remove(s.file.Name()) 78 | os.Remove(s.sameFile.Name()) 79 | os.Remove(s.differentHash.Name()) 80 | os.Remove(s.replaceFile.Name()) 81 | os.Remove(s.replaceFile1.Name()) 82 | } 83 | 84 | func (s *TestSuite) TestSameFileTrue(t *C) { 85 | status, err := SameFile(s.file.Name(), s.sameFile.Name(), hclog.Default()) 86 | if err != nil { 87 | t.Error(err.Error()) 88 | } 89 | t.Check(status, Equals, true) 90 | } 91 | 92 | func (s *TestSuite) TestSameFileFalse(t *C) { 93 | status, err := SameFile(s.file.Name(), s.differentHash.Name(), hclog.Default()) 94 | if err != nil { 95 | t.Error(err.Error()) 96 | } 97 | t.Check(status, Equals, false) 98 | } 99 | 100 | func (s *TestSuite) TestReplaceFile(t *C) { 101 | fileStat, err := stat(s.file.Name()) 102 | if err != nil { 103 | t.Error(err.Error()) 104 | } 105 | 106 | err = ReplaceFile(s.replaceFile.Name(), s.replaceFile1.Name(), fileStat.Mode, hclog.Default()) 107 | if err != nil { 108 | t.Error(err.Error()) 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /pkg/template/renderer.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * Based on code from confd. https://github.com/kelseyhightower/confd 4 | * © 2013 Kelsey Hightower 5 | * © 2016 The Remco Authors 6 | * 7 | * For the full copyright and license information, please view the LICENSE 8 | * file that was distributed with this source code. 9 | */ 10 | 11 | package template 12 | 13 | import ( 14 | "bytes" 15 | "fmt" 16 | "github.com/hashicorp/go-hclog" 17 | "io/ioutil" 18 | "os" 19 | "os/exec" 20 | "path" 21 | "path/filepath" 22 | "strconv" 23 | "sync" 24 | "text/template" 25 | "time" 26 | 27 | "github.com/HeavyHorst/pongo2" 28 | "github.com/HeavyHorst/remco/pkg/template/fileutil" 29 | "github.com/armon/go-metrics" 30 | "github.com/pkg/errors" 31 | ) 32 | 33 | func init() { 34 | pongo2.SetAutoescape(false) 35 | } 36 | 37 | // Renderer contains all data needed for the template processing 38 | type Renderer struct { 39 | Src string `json:"src"` 40 | Dst string `json:"dst"` 41 | MkDirs bool `toml:"make_directories"` 42 | Mode string `json:"mode"` 43 | UID int `json:"uid"` 44 | GID int `json:"gid"` 45 | ReloadCmd string `toml:"reload_cmd" json:"reload_cmd"` 46 | CheckCmd string `toml:"check_cmd" json:"check_cmd"` 47 | stageFile *os.File 48 | logger hclog.Logger 49 | ReapLock *sync.RWMutex 50 | } 51 | 52 | // createStageFile stages the src configuration file by processing the src 53 | // template and setting the desired owner, group, and mode. It also sets the 54 | // StageFile for the template resource. 55 | // It returns an error if any. 56 | func (s *Renderer) createStageFile(funcMap map[string]interface{}) error { 57 | if !fileutil.IsFileExist(s.Src) { 58 | return fmt.Errorf("missing template: %s", s.Src) 59 | } 60 | 61 | s.logger.With( 62 | "template", s.Src, 63 | ).Debug("compiling source template") 64 | 65 | set := pongo2.NewSet("local", &pongo2.LocalFilesystemLoader{}) 66 | set.Options = &pongo2.Options{ 67 | TrimBlocks: true, 68 | LStripBlocks: true, 69 | } 70 | tmpl, err := set.FromFile(s.Src) 71 | if err != nil { 72 | return errors.Wrapf(err, "set.FromFile(%s) failed", s.Src) 73 | } 74 | 75 | // create TempFile in Dest directory to avoid cross-filesystem issues 76 | if s.MkDirs { 77 | if err := os.MkdirAll(filepath.Dir(s.Dst), 0755); err != nil { 78 | return errors.Wrap(err, "MkdirAll failed") 79 | } 80 | } 81 | temp, err := ioutil.TempFile(filepath.Dir(s.Dst), "."+filepath.Base(s.Dst)) 82 | if err != nil { 83 | return errors.Wrap(err, "couldn't create tempfile") 84 | } 85 | 86 | executionStartTime := time.Now() 87 | if err = tmpl.ExecuteWriter(funcMap, temp); err != nil { 88 | temp.Close() 89 | os.Remove(temp.Name()) 90 | return errors.Wrap(err, "template execution failed") 91 | } 92 | metrics.MeasureSince([]string{"files", "template_execution_duration"}, executionStartTime) 93 | 94 | temp.Close() 95 | 96 | fileMode, err := s.getFileMode() 97 | if err != nil { 98 | return errors.Wrap(err, "getFileMode failed") 99 | } 100 | 101 | // Set the owner, group, and mode on the stage file now to make it easier to 102 | // compare against the destination configuration file later. 103 | os.Chmod(temp.Name(), fileMode) 104 | os.Chown(temp.Name(), s.UID, s.GID) 105 | s.stageFile = temp 106 | 107 | return nil 108 | } 109 | 110 | // syncFiles compares the staged and dest config files and attempts to sync them 111 | // if they differ. syncFiles will run a config check command if set before 112 | // overwriting the target config file. Finally, syncFile will run a reload command 113 | // if set to have the application or service pick up the changes. 114 | // It returns a boolean indicating if the file has changed and an error if any. 115 | func (s *Renderer) syncFiles(runCommands bool) (bool, error) { 116 | var changed bool 117 | staged := s.stageFile.Name() 118 | defer os.Remove(staged) 119 | 120 | s.logger.With( 121 | "staged", path.Base(staged), 122 | "dest", s.Dst, 123 | ).Debug("comparing staged and dest config files") 124 | 125 | ok, err := fileutil.SameFile(staged, s.Dst, s.logger) 126 | if err != nil { 127 | s.logger.Error(err.Error()) 128 | } 129 | 130 | if !ok { 131 | s.logger.With( 132 | "config", s.Dst, 133 | ).Info("target config out of sync") 134 | 135 | if runCommands { 136 | if err := s.check(staged); err != nil { 137 | return changed, errors.Wrap(err, "config check failed") 138 | } 139 | } 140 | 141 | s.logger.With( 142 | "config", s.Dst, 143 | ).Debug("overwriting target config") 144 | 145 | fileMode, err := s.getFileMode() 146 | if err != nil { 147 | return changed, errors.Wrap(err, "getFileMode failed") 148 | } 149 | if err := fileutil.ReplaceFile(staged, s.Dst, fileMode, s.logger); err != nil { 150 | return changed, errors.Wrap(err, "replace file failed") 151 | } 152 | 153 | // make sure owner and group match the temp file, in case the file was created with WriteFile 154 | os.Chown(s.Dst, s.UID, s.GID) 155 | changed = true 156 | 157 | if runCommands { 158 | if err := s.reload(s.Dst); err != nil { 159 | return changed, errors.Wrap(err, "reload command failed") 160 | } 161 | } 162 | 163 | s.logger.With( 164 | "config", s.Dst, 165 | ).Info("target config has been updated") 166 | 167 | } else { 168 | s.logger.With( 169 | "config", s.Dst, 170 | ).Debug("target config in sync") 171 | 172 | } 173 | return changed, nil 174 | } 175 | 176 | func (s *Renderer) getFileMode() (os.FileMode, error) { 177 | if s.Mode == "" { 178 | if !fileutil.IsFileExist(s.Dst) { 179 | return 0644, nil 180 | } 181 | fi, err := os.Stat(s.Dst) 182 | if err != nil { 183 | return 0, errors.Wrap(err, "os.Stat failed") 184 | } 185 | return fi.Mode(), nil 186 | } 187 | mode, err := strconv.ParseUint(s.Mode, 0, 32) 188 | if err != nil { 189 | return 0, errors.Wrapf(err, "parsing filemode failed: %s", s.Mode) 190 | } 191 | return os.FileMode(mode), nil 192 | 193 | } 194 | 195 | // check executes the check command to validate the staged config file. The 196 | // command is modified so that any references to src template are substituted 197 | // with a string representing the full path of the staged file. This allows the 198 | // check to be run on the staged file before overwriting the destination config file. 199 | // It returns nil if the check command returns 0 and there are no other errors. 200 | func (s *Renderer) check(stageFile string) error { 201 | if s.CheckCmd == "" { 202 | return nil 203 | } 204 | defer metrics.MeasureSince([]string{"files", "check_command_duration"}, time.Now()) 205 | cmd, err := renderTemplate(s.CheckCmd, map[string]string{"src": stageFile}) 206 | if err != nil { 207 | return errors.Wrap(err, "rendering check command failed") 208 | } 209 | output, err := execCommand(cmd, s.logger, s.ReapLock) 210 | if err != nil { 211 | s.logger.Error(fmt.Sprintf("%q", string(output))) 212 | return errors.Wrap(err, "the check command failed") 213 | } 214 | s.logger.Debug(fmt.Sprintf("%q", string(output))) 215 | return nil 216 | } 217 | 218 | // reload executes the reload command. 219 | // It returns nil if the reload command returns 0 and an error otherwise. 220 | func (s *Renderer) reload(renderedFile string) error { 221 | if s.ReloadCmd == "" { 222 | return nil 223 | } 224 | defer metrics.MeasureSince([]string{"files", "reload_command_duration"}, time.Now()) 225 | cmd, err := renderTemplate(s.ReloadCmd, map[string]string{"dst": renderedFile}) 226 | if err != nil { 227 | return errors.Wrap(err, "rendering reload command failed") 228 | } 229 | output, err := execCommand(cmd, s.logger, s.ReapLock) 230 | if err != nil { 231 | s.logger.Error(fmt.Sprintf("%q", string(output))) 232 | return errors.Wrap(err, "the reload command failed") 233 | } 234 | s.logger.Debug(fmt.Sprintf("%q", string(output))) 235 | return nil 236 | } 237 | 238 | func renderTemplate(unparsed string, data interface{}) (string, error) { 239 | var rendered bytes.Buffer 240 | tmpl, err := template.New("").Parse(unparsed) 241 | if err != nil { 242 | return "", errors.Wrap(err, "parsing template failed") 243 | } 244 | if err := tmpl.Execute(&rendered, data); err != nil { 245 | return "", errors.Wrap(err, "template execution failed") 246 | } 247 | return rendered.String(), nil 248 | } 249 | 250 | func execCommand(cmd string, logger hclog.Logger, rl *sync.RWMutex) ([]byte, error) { 251 | logger.Debug("Running cmd", "command", cmd) 252 | c := exec.Command("/bin/sh", "-c", cmd) 253 | 254 | if rl != nil { 255 | rl.RLock() 256 | defer rl.RUnlock() 257 | } 258 | 259 | return c.CombinedOutput() 260 | } 261 | -------------------------------------------------------------------------------- /pkg/template/resource.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * Based on code from confd. 4 | * https://github.com/kelseyhightower/confd/blob/30663b9822fe8e800d1f2ea78447fba0ebce8f6c/resource/template/resource.go 5 | * Users who have contributed to this file 6 | * © 2013 Kelsey Hightower 7 | * © 2014 Armon Dadgar 8 | * © 2014 Ernesto Jimenez 9 | * © 2014 Nathan Fritz 10 | * © 2014 John Engelman 11 | * © 2014 Joanna Solmon 12 | * © 2014 Chris Armstrong 13 | * © 2014 Chris McNabb 14 | * © 2015 Phil Kates 15 | * © 2015 Matthew Fisher 16 | * 17 | * © 2016 The Remco Authors 18 | * 19 | * For the full copyright and license information, please view the LICENSE 20 | * file that was distributed with this source code. 21 | */ 22 | 23 | package template 24 | 25 | import ( 26 | "context" 27 | "fmt" 28 | "github.com/hashicorp/go-hclog" 29 | "math/rand" 30 | "os" 31 | "path" 32 | "strings" 33 | "sync" 34 | "time" 35 | 36 | "github.com/HeavyHorst/memkv" 37 | berr "github.com/HeavyHorst/remco/pkg/backends/error" 38 | "github.com/HeavyHorst/remco/pkg/log" 39 | "github.com/armon/go-metrics" 40 | "github.com/pkg/errors" 41 | ) 42 | 43 | // Resource is the representation of a parsed template resource. 44 | type Resource struct { 45 | backends []Backend 46 | funcMap map[string]interface{} 47 | store *memkv.Store 48 | sources []*Renderer 49 | logger hclog.Logger 50 | 51 | exec Executor 52 | startCmd string 53 | reloadCmd string 54 | // SignalChan is a channel to send os.Signal's to all child processes. 55 | SignalChan chan os.Signal 56 | 57 | // Failed is true if we run Monitor() in exec mode and the child process exits unexpectedly. 58 | // If the monitor context is canceled as usual Failed is false. 59 | // Failed is used to restart the Resource on failure. 60 | Failed bool 61 | 62 | // Set to true if this resource has backends only using "Onetime=true" flag to 63 | // exit on failure if the resource has some templating error 64 | OnetimeOnly bool 65 | } 66 | 67 | // ResourceConfig is a configuration struct to create a new resource. 68 | type ResourceConfig struct { 69 | Exec ExecConfig 70 | StartCmd string 71 | ReloadCmd string 72 | 73 | // Template is the configuration for all template options. 74 | // You can configure as much template-destination pairs as you like. 75 | Template []*Renderer 76 | 77 | // Name gives the Resource a name. 78 | // This name is added to the logs to distinguish between different resources. 79 | Name string 80 | 81 | // Connectors is a list of BackendConnectors. 82 | // The Resource will establish a connection to all of these. 83 | Connectors []BackendConnector 84 | } 85 | 86 | // ErrEmptySrc is returned if an emty src template is passed to NewResource 87 | var ErrEmptySrc = fmt.Errorf("empty src template") 88 | 89 | // NewResourceFromResourceConfig creates a new resource from the given ResourceConfig. 90 | func NewResourceFromResourceConfig(ctx context.Context, reapLock *sync.RWMutex, r ResourceConfig) (*Resource, error) { 91 | backendList, err := connectAllBackends(ctx, r.Connectors) 92 | if err != nil { 93 | return nil, errors.Wrap(err, "connectAllBackends failed") 94 | } 95 | 96 | for _, p := range r.Template { 97 | p.ReapLock = reapLock 98 | } 99 | 100 | logger := log.WithFields("resource", r.Name) 101 | exec := NewExecutor(r.Exec.Command, r.Exec.ReloadSignal, r.Exec.KillSignal, r.Exec.KillTimeout, r.Exec.Splay, logger) 102 | res, err := NewResource(backendList, r.Template, r.Name, exec, r.StartCmd, r.ReloadCmd) 103 | if err != nil { 104 | for _, v := range backendList { 105 | v.Close() 106 | } 107 | } 108 | return res, err 109 | } 110 | 111 | // NewResource creates a Resource. 112 | func NewResource(backends []Backend, sources []*Renderer, name string, exec Executor, startCmd, reloadCmd string) (*Resource, error) { 113 | if len(backends) == 0 { 114 | return nil, fmt.Errorf("a valid StoreClient is required") 115 | } 116 | 117 | logger := log.WithFields("resource", name) 118 | 119 | for _, v := range sources { 120 | if v.Src == "" { 121 | return nil, ErrEmptySrc 122 | } 123 | v.logger = logger 124 | } 125 | 126 | tr := &Resource{ 127 | backends: backends, 128 | store: memkv.New(), 129 | funcMap: newFuncMap(), 130 | sources: sources, 131 | logger: logger, 132 | SignalChan: make(chan os.Signal, 1), 133 | exec: exec, 134 | startCmd: startCmd, 135 | reloadCmd: reloadCmd, 136 | } 137 | 138 | // initialize the individual backend memkv Stores 139 | for i := range tr.backends { 140 | store := memkv.New() 141 | tr.backends[i].store = store 142 | 143 | if tr.backends[i].Interval <= 0 && !tr.backends[i].Onetime && !tr.backends[i].Watch { 144 | logger.Warn("interval needs to be > 0: setting interval to 60") 145 | tr.backends[i].Interval = 60 146 | } 147 | } 148 | 149 | addFuncs(tr.funcMap, tr.store.FuncMap) 150 | 151 | // check all backends for onetime or interval/watch, used for global error handling 152 | tr.OnetimeOnly = true 153 | for _, b := range tr.backends { 154 | tr.OnetimeOnly = tr.OnetimeOnly && b.Onetime 155 | } 156 | 157 | return tr, nil 158 | } 159 | 160 | // Close closes the connection to all underlying backends. 161 | func (t *Resource) Close() { 162 | for _, v := range t.backends { 163 | t.logger.With( 164 | "backend", v.Name, 165 | ).Debug("closing client connection") 166 | v.Close() 167 | } 168 | } 169 | 170 | // setVars reads all KV-Pairs for the backend 171 | // and writes these pairs to the individual (per backend) memkv store. 172 | // After that, the instance wide memkv store gets purged and is recreated with all individual 173 | // memkv KV-Pairs. 174 | // Key collisions are logged. 175 | // It returns an error if any. 176 | func (t *Resource) setVars(storeClient Backend) error { 177 | var err error 178 | 179 | t.logger.With( 180 | "backend", storeClient.Name, 181 | "key_prefix", storeClient.Prefix, 182 | ).Debug("retrieving keys") 183 | 184 | result, err := storeClient.GetValues(appendPrefix(storeClient.Prefix, storeClient.Keys)) 185 | if err != nil { 186 | return errors.Wrap(err, "getValues failed") 187 | } 188 | 189 | storeClient.store.Purge() 190 | 191 | for key, value := range result { 192 | storeClient.store.Set(path.Join("/", strings.TrimPrefix(key, storeClient.Prefix)), value) 193 | } 194 | 195 | //merge all stores 196 | t.store.Purge() 197 | for _, v := range t.backends { 198 | for _, kv := range v.store.GetAllKVs() { 199 | if t.store.Exists(kv.Key) { 200 | t.logger.Warn("key collision", "key", kv.Key) 201 | } 202 | t.store.Set(kv.Key, kv.Value) 203 | } 204 | } 205 | 206 | return nil 207 | } 208 | 209 | func (t *Resource) createStageFileAndSync(runCommands bool) (bool, error) { 210 | var changed bool 211 | for _, s := range t.sources { 212 | err := s.createStageFile(t.funcMap) 213 | if err != nil { 214 | metrics.IncrCounter([]string{"files", "stage_errors_total"}, 1) 215 | return changed, errors.Wrap(err, "create stage file failed") 216 | } 217 | metrics.IncrCounter([]string{"files", "staged_total"}, 1) 218 | c, err := s.syncFiles(runCommands) 219 | changed = changed || c 220 | if err != nil { 221 | metrics.IncrCounter([]string{"files", "sync_errors_total"}, 1) 222 | return changed, errors.Wrap(err, "sync files failed") 223 | } 224 | metrics.IncrCounter([]string{"files", "synced_total"}, 1) 225 | } 226 | return changed, nil 227 | } 228 | 229 | // Process is a convenience function that wraps calls to the three main tasks 230 | // required to keep local configuration files in sync. First we gather vars 231 | // from the store, then we stage a candidate configuration file, and finally sync 232 | // things up. 233 | // It returns an error if any. 234 | func (t *Resource) process(storeClients []Backend, runCommands bool) (bool, error) { 235 | var changed bool 236 | var err error 237 | for _, storeClient := range storeClients { 238 | labels := []metrics.Label{{Name: "name", Value: storeClient.Name}} 239 | if err = t.setVars(storeClient); err != nil { 240 | metrics.IncrCounterWithLabels([]string{"backends", "sync_errors_total"}, 1, labels) 241 | return changed, berr.BackendError{ 242 | Message: errors.Wrap(err, "setVars failed").Error(), 243 | Backend: storeClient.Name, 244 | } 245 | } 246 | metrics.IncrCounterWithLabels([]string{"backends", "synced_total"}, 1, labels) 247 | } 248 | if changed, err = t.createStageFileAndSync(runCommands); err != nil { 249 | return changed, errors.Wrap(err, "createStageFileAndSync failed") 250 | } 251 | return changed, nil 252 | } 253 | 254 | // Monitor will start to monitor all given Backends for changes. 255 | // It accepts a ctx.Context for cancelation. 256 | // It will process all given templates on changes. 257 | func (t *Resource) Monitor(ctx context.Context) { 258 | t.Failed = false 259 | wg := &sync.WaitGroup{} 260 | 261 | ctx, cancel := context.WithCancel(ctx) 262 | defer cancel() 263 | 264 | processChan := make(chan Backend) 265 | defer close(processChan) 266 | errChan := make(chan berr.BackendError, 10) 267 | 268 | // try to process the template resource with all given backends 269 | // we wait a random amount of time (between 0 - 30 seconds) 270 | // to prevent ddossing our backends and try again (with all backends - no stale data) 271 | retryChan := make(chan struct{}, 1) 272 | retryChan <- struct{}{} 273 | retryloop: 274 | for { 275 | select { 276 | case <-ctx.Done(): 277 | return 278 | case <-retryChan: 279 | if _, err := t.process(t.backends, t.startCmd == ""); err != nil { 280 | switch err := err.(type) { 281 | case berr.BackendError: 282 | t.logger.With( 283 | "backend", err.Backend, 284 | "error", err, 285 | ).Error("backend error") 286 | default: 287 | t.logger.Error("failed to process", "error", err) 288 | } 289 | 290 | if t.OnetimeOnly { 291 | t.Failed = true 292 | cancel() 293 | return 294 | } else { 295 | go func() { 296 | rn := rand.Int63n(30) 297 | t.logger.Error(fmt.Sprintf("not all templates could be rendered, trying again after %d seconds", rn)) 298 | time.Sleep(time.Duration(rn) * time.Second) 299 | select { 300 | case <-ctx.Done(): 301 | return 302 | default: 303 | retryChan <- struct{}{} 304 | } 305 | }() 306 | continue retryloop 307 | } 308 | } 309 | break retryloop 310 | } 311 | } 312 | 313 | if t.startCmd != "" { 314 | output, err := execCommand(t.startCmd, t.logger, nil) 315 | if err != nil { 316 | t.logger.Error(fmt.Sprintf("failed to execute the start cmd - %q", string(output))) 317 | t.Failed = true 318 | cancel() 319 | } else { 320 | t.logger.Debug(fmt.Sprintf("%q", string(output))) 321 | } 322 | } 323 | 324 | err := t.exec.SpawnChild() 325 | if err != nil { 326 | t.logger.Error("failed to spawn child", "error", err) 327 | t.Failed = true 328 | cancel() 329 | } else { 330 | defer t.exec.StopChild() 331 | } 332 | 333 | done := make(chan struct{}) 334 | wg.Add(1) 335 | go func() { 336 | // Wait for the child process to quit. 337 | // If the process terminates unexpectedly (the context was NOT canceled), we set t.Failed to true 338 | // and cancel the resource context. Remco will try to restart the resource if t.Failed is true. 339 | defer wg.Done() 340 | failed := t.exec.Wait(ctx) 341 | if failed { 342 | t.Failed = true 343 | cancel() 344 | } 345 | }() 346 | 347 | // start the watch and interval processors so that we get notfied on changes 348 | for _, sc := range t.backends { 349 | if sc.Watch { 350 | wg.Add(1) 351 | go func(s Backend) { 352 | defer wg.Done() 353 | s.watch(ctx, processChan, errChan) 354 | }(sc) 355 | } 356 | 357 | if sc.Interval > 0 { 358 | wg.Add(1) 359 | go func(s Backend) { 360 | defer wg.Done() 361 | s.interval(ctx, processChan) 362 | }(sc) 363 | } 364 | } 365 | 366 | go func() { 367 | // If there is no goroutine left - quit 368 | wg.Wait() 369 | close(done) 370 | }() 371 | 372 | for { 373 | select { 374 | case storeClient := <-processChan: 375 | changed, err := t.process([]Backend{storeClient}, true) 376 | if err != nil { 377 | switch err.(type) { 378 | case berr.BackendError: 379 | t.logger.With("backend", storeClient.Name, "error", err).Error("backend error") 380 | default: 381 | t.logger.Error("default handler", "error", err) 382 | } 383 | } else if changed { 384 | if err := t.exec.Reload(); err != nil { 385 | t.logger.Error("failed to reload", "error", err) 386 | } 387 | 388 | if t.reloadCmd != "" { 389 | output, err := execCommand(t.reloadCmd, t.logger, nil) 390 | if err != nil { 391 | t.logger.Error("failed to execute the resource reload cmd", "output", string(output), "error", err) 392 | } 393 | } 394 | } 395 | case s := <-t.SignalChan: 396 | err := t.exec.SignalChild(s) 397 | if err != nil { 398 | t.logger.Error("failed to signal child", "error", err) 399 | } 400 | case err := <-errChan: 401 | t.logger.With("backend", err.Backend).Error("error", "message", err.Message) 402 | case <-ctx.Done(): 403 | go func() { 404 | for range processChan { 405 | } 406 | }() 407 | wg.Wait() 408 | return 409 | case <-done: 410 | return 411 | } 412 | } 413 | } 414 | -------------------------------------------------------------------------------- /pkg/template/resource_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2016 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | package template 10 | 11 | import ( 12 | "context" 13 | "fmt" 14 | "io/ioutil" 15 | "os" 16 | "time" 17 | 18 | "github.com/HeavyHorst/easykv/mock" 19 | 20 | . "gopkg.in/check.v1" 21 | ) 22 | 23 | const ( 24 | tmplString string = "{{ getallkvs() | toPrettyJSON }}" 25 | tmplFile string = `[ 26 | { 27 | "Key": "/some/path/data", 28 | "Value": "someData" 29 | } 30 | ]` 31 | ) 32 | 33 | type ResourceSuite struct { 34 | templateFile string 35 | backend Backend 36 | renderer *Renderer 37 | resource *Resource 38 | } 39 | 40 | var _ = Suite(&ResourceSuite{}) 41 | 42 | func (s *ResourceSuite) SetUpSuite(t *C) { 43 | // create simple template file 44 | f, err := ioutil.TempFile("", "template") 45 | t.Assert(err, IsNil) 46 | defer f.Close() 47 | _, err = f.WriteString(tmplString) 48 | t.Assert(err, IsNil) 49 | s.templateFile = f.Name() 50 | 51 | // create a backend 52 | s.backend = Backend{ 53 | Name: "mock", 54 | Onetime: false, 55 | Watch: true, 56 | Prefix: "/", 57 | Interval: 1, 58 | Keys: []string{"/"}, 59 | } 60 | s.backend.ReadWatcher, _ = mock.New(nil, map[string]string{"/some/path/data": "someData"}) 61 | 62 | // create a renderer 63 | s.renderer = &Renderer{ 64 | Src: s.templateFile, 65 | Dst: "/tmp/remco-basic-test.conf", 66 | CheckCmd: "exit 0", 67 | ReloadCmd: "exit 0", 68 | } 69 | 70 | exec := NewExecutor("", "", "", 0, 0, nil) 71 | res, err := NewResource([]Backend{s.backend}, []*Renderer{s.renderer}, "test", exec, "", "") 72 | t.Assert(err, IsNil) 73 | s.resource = res 74 | } 75 | 76 | func (s *ResourceSuite) TearDownSuite(t *C) { 77 | err := os.Remove(s.templateFile) 78 | t.Check(err, IsNil) 79 | } 80 | 81 | func (s *ResourceSuite) TestNewResource(t *C) { 82 | t.Check(s.resource.backends, HasLen, 1) 83 | t.Check(s.resource.backends[0].store, NotNil) 84 | t.Check(s.resource.store, NotNil) 85 | t.Check(s.resource.logger, NotNil) 86 | 87 | fm := newFuncMap() 88 | addFuncs(fm, s.resource.store.FuncMap) 89 | t.Check(s.resource.funcMap, HasLen, len(fm)) 90 | t.Check(s.resource.sources, DeepEquals, []*Renderer{s.renderer}) 91 | t.Check(s.resource.SignalChan, NotNil) 92 | } 93 | 94 | func (s *ResourceSuite) TestClose(t *C) { 95 | s.resource.Close() 96 | } 97 | 98 | func (s *ResourceSuite) TestSetVars(t *C) { 99 | err := s.resource.setVars(s.resource.backends[0]) 100 | t.Check(err, IsNil) 101 | // the backend trie and the global tree should hold the same values 102 | t.Check(s.resource.store.GetAllKVs(), DeepEquals, s.resource.backends[0].store.GetAllKVs()) 103 | } 104 | 105 | func (s *ResourceSuite) TestCreateStageFileAndSync(t *C) { 106 | _, err := s.resource.createStageFileAndSync(true) 107 | t.Check(err, IsNil) 108 | } 109 | 110 | func (s *ResourceSuite) TestProcess(t *C) { 111 | _, err := s.resource.process(s.resource.backends, true) 112 | t.Check(err, IsNil) 113 | 114 | data, err := ioutil.ReadFile("/tmp/remco-basic-test.conf") 115 | t.Assert(err, IsNil) 116 | t.Check(string(data), Equals, tmplFile) 117 | } 118 | 119 | func (s *ResourceSuite) TestMonitor(t *C) { 120 | ctx, cancel := context.WithCancel(context.Background()) 121 | defer cancel() 122 | 123 | go func() { 124 | time.Sleep(5 * time.Second) 125 | cancel() 126 | }() 127 | 128 | s.resource.Monitor(ctx) 129 | t.Check(s.resource.Failed, Equals, false) 130 | } 131 | 132 | func (s *ResourceSuite) TestMonitorWithBackendError(t *C) { 133 | s.resource.backends[0].ReadWatcher.(*mock.Client).Err = fmt.Errorf("some error") 134 | ctx, cancel := context.WithCancel(context.Background()) 135 | defer cancel() 136 | 137 | go func() { 138 | time.Sleep(5 * time.Second) 139 | cancel() 140 | }() 141 | 142 | s.resource.Monitor(ctx) 143 | t.Check(s.resource.Failed, Equals, false) 144 | s.resource.backends[0].ReadWatcher.(*mock.Client).Err = nil 145 | } 146 | -------------------------------------------------------------------------------- /pkg/template/template_filters.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2016 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | package template 10 | 11 | import ( 12 | "bytes" 13 | "encoding/base64" 14 | "encoding/json" 15 | "fmt" 16 | "io/ioutil" 17 | "net/url" 18 | "path" 19 | "path/filepath" 20 | "reflect" 21 | "sort" 22 | "strconv" 23 | "strings" 24 | 25 | "github.com/HeavyHorst/memkv" 26 | "github.com/HeavyHorst/pongo2" 27 | "github.com/dop251/goja" 28 | "github.com/pkg/errors" 29 | "gopkg.in/yaml.v3" 30 | gyml "sigs.k8s.io/yaml" 31 | ) 32 | 33 | func init() { 34 | pongo2.RegisterFilter("sortByLength", filterSortByLength) 35 | pongo2.RegisterFilter("parseInt", filterParseInt) 36 | pongo2.RegisterFilter("parseFloat", filterParseFloat) 37 | pongo2.RegisterFilter("parseYAML", filterUnmarshalYAML) 38 | pongo2.RegisterFilter("parseJSON", filterUnmarshalYAML) // just an alias 39 | pongo2.RegisterFilter("parseYAMLArray", filterUnmarshalYAML) // deprecated 40 | pongo2.RegisterFilter("toJSON", filterToJSON) 41 | pongo2.RegisterFilter("toPrettyJSON", filterToPrettyJSON) 42 | pongo2.RegisterFilter("toYAML", filterToYAML) 43 | pongo2.RegisterFilter("dir", filterDir) 44 | pongo2.RegisterFilter("base", filterBase) 45 | pongo2.RegisterFilter("base64", filterBase64) 46 | pongo2.RegisterFilter("index", filterIndex) 47 | pongo2.RegisterFilter("mapValue", filterMapValue) 48 | } 49 | 50 | // RegisterCustomJsFilters loads all filters from the given directory. 51 | // It returns an error if any. 52 | func RegisterCustomJsFilters(folder string) error { 53 | files, _ := ioutil.ReadDir(folder) 54 | for _, file := range files { 55 | if strings.HasSuffix(file.Name(), ".js") { 56 | fp := filepath.Join(folder, file.Name()) 57 | buf, err := ioutil.ReadFile(fp) 58 | if err != nil { 59 | return errors.Errorf("couldn't load custom filter %s", fp) 60 | } 61 | name := file.Name() 62 | name = name[0 : len(name)-3] 63 | 64 | filterFunc := pongoJSFilter(name, string(buf)) 65 | 66 | if err := pongo2.RegisterFilter(name, filterFunc); err != nil { 67 | if err := pongo2.ReplaceFilter(name, filterFunc); err != nil { 68 | return errors.Errorf("couldn't replace existing filter %s", name) 69 | } 70 | } 71 | } 72 | } 73 | return nil 74 | } 75 | 76 | func pongoJSFilter(name string, js string) func(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *pongo2.Error) { 77 | return func(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *pongo2.Error) { 78 | vm := goja.New() 79 | 80 | vm.Set("In", in.Interface()) 81 | vm.Set("Param", param.Interface()) 82 | 83 | v, err := vm.RunString(js) 84 | if err != nil { 85 | return nil, &pongo2.Error{ 86 | Sender: "javascript-filter:" + name, 87 | OrigError: err, 88 | } 89 | } 90 | 91 | return pongo2.AsValue(v.Export()), nil 92 | } 93 | } 94 | 95 | func parseParamMap(in string) (url.Values, error) { 96 | in = strings.ReplaceAll(in, ", ", ",") 97 | in = strings.ReplaceAll(in, ",", "&") 98 | return url.ParseQuery(in) 99 | } 100 | 101 | func filterBase64(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *pongo2.Error) { 102 | if !in.IsString() { 103 | return in, nil 104 | } 105 | sEnc := base64.StdEncoding.EncodeToString([]byte(in.String())) 106 | return pongo2.AsValue(sEnc), nil 107 | } 108 | 109 | func filterBase(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *pongo2.Error) { 110 | if !in.IsString() { 111 | return in, nil 112 | } 113 | return pongo2.AsValue(path.Base(in.String())), nil 114 | } 115 | 116 | func filterDir(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *pongo2.Error) { 117 | if !in.IsString() { 118 | return in, nil 119 | } 120 | return pongo2.AsValue(path.Dir(in.String())), nil 121 | } 122 | 123 | func filterToPrettyJSON(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *pongo2.Error) { 124 | b, err := json.MarshalIndent(in.Interface(), "", " ") 125 | if err != nil { 126 | return nil, &pongo2.Error{ 127 | Sender: "filter:filterToPrettyJSON", 128 | OrigError: err, 129 | } 130 | } 131 | return pongo2.AsValue(string(b)), nil 132 | } 133 | 134 | func filterToJSON(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *pongo2.Error) { 135 | b, err := json.Marshal(in.Interface()) 136 | if err != nil { 137 | return nil, &pongo2.Error{ 138 | Sender: "filter:filterToJSON", 139 | OrigError: err, 140 | } 141 | } 142 | return pongo2.AsValue(string(b)), nil 143 | } 144 | 145 | func filterToYAML(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *pongo2.Error) { 146 | b := bytes.Buffer{} 147 | yamlEncoder := yaml.NewEncoder(&b) 148 | 149 | if param != nil && param.String() != "" { 150 | pm, err := parseParamMap(param.String()) 151 | if err != nil { 152 | return nil, &pongo2.Error{ 153 | Sender: "filter:filterToYAML", 154 | OrigError: fmt.Errorf("could't parese parameter list: %w", err), 155 | } 156 | } 157 | 158 | indent, err := strconv.Atoi(pm.Get("indent")) 159 | if err != nil { 160 | return nil, &pongo2.Error{ 161 | Sender: "filter:filterToYAML", 162 | OrigError: fmt.Errorf("couldn't parse integer: %w", err), 163 | } 164 | } 165 | 166 | yamlEncoder.SetIndent(indent) 167 | } 168 | 169 | err := yamlEncoder.Encode(in.Interface()) 170 | if err != nil { 171 | return nil, &pongo2.Error{ 172 | Sender: "filter:filterToYAML", 173 | OrigError: err, 174 | } 175 | } 176 | return pongo2.AsValue(string(b.Bytes())), nil 177 | } 178 | 179 | func filterParseInt(in, param *pongo2.Value) (*pongo2.Value, *pongo2.Error) { 180 | if !in.IsString() { 181 | return in, nil 182 | } 183 | 184 | ins := in.String() 185 | if ins == "" { 186 | return pongo2.AsValue(0), nil 187 | } 188 | 189 | result, err := strconv.ParseInt(ins, 10, 64) 190 | if err != nil { 191 | return nil, &pongo2.Error{ 192 | Sender: "filter:filterParseInt", 193 | OrigError: err, 194 | } 195 | } 196 | 197 | return pongo2.AsValue(result), nil 198 | } 199 | 200 | func filterParseFloat(in, param *pongo2.Value) (*pongo2.Value, *pongo2.Error) { 201 | if !in.IsString() { 202 | return in, nil 203 | } 204 | 205 | ins := in.String() 206 | if ins == "" { 207 | return pongo2.AsValue(0.0), nil 208 | } 209 | 210 | result, err := strconv.ParseFloat(ins, 10) 211 | if err != nil { 212 | return nil, &pongo2.Error{ 213 | Sender: "filter:filterParseFloat", 214 | OrigError: err, 215 | } 216 | } 217 | 218 | return pongo2.AsValue(result), nil 219 | } 220 | 221 | func filterUnmarshalYAML(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *pongo2.Error) { 222 | if !in.IsString() { 223 | return in, nil 224 | } 225 | 226 | var ret interface{} 227 | if err := gyml.Unmarshal([]byte(in.String()), &ret); err != nil { 228 | return nil, &pongo2.Error{ 229 | Sender: "filter:filterUnmarshalYAML", 230 | OrigError: err, 231 | } 232 | } 233 | 234 | return pongo2.AsValue(ret), nil 235 | } 236 | 237 | func filterIndex(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *pongo2.Error) { 238 | if !in.CanSlice() { 239 | return in, nil 240 | } 241 | 242 | index := param.Integer() 243 | if index < 0 { 244 | index = in.Len() + index 245 | } 246 | 247 | return pongo2.AsValue(in.Index(index).Interface()), nil 248 | } 249 | 250 | func filterMapValue(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *pongo2.Error) { 251 | if in == nil || in.IsNil() { 252 | return pongo2.AsValue(nil), nil 253 | } 254 | 255 | val := reflect.ValueOf(in.Interface()) 256 | if val.Kind() == reflect.Map { 257 | valueType := val.Type().Key().Kind() 258 | paramValue := reflect.ValueOf(param.Interface()) 259 | 260 | if paramValue.Kind() != valueType { 261 | return pongo2.AsValue(nil), nil 262 | } 263 | 264 | mv := val.MapIndex(paramValue) 265 | if !mv.IsValid() { 266 | return pongo2.AsValue(nil), nil 267 | } 268 | 269 | return pongo2.AsValue(mv.Interface()), nil 270 | } 271 | return pongo2.AsValue(nil), nil 272 | } 273 | 274 | func filterSortByLength(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *pongo2.Error) { 275 | if !in.CanSlice() { 276 | return in, nil 277 | } 278 | 279 | values := in.Interface() 280 | switch v := values.(type) { 281 | case []string: 282 | sort.Slice(v, func(i, j int) bool { 283 | return len(v[i]) < len(v[j]) 284 | }) 285 | return pongo2.AsValue(v), nil 286 | case memkv.KVPairs: 287 | sort.Slice(v, func(i, j int) bool { 288 | return len(v[i].Key) < len(v[j].Key) 289 | }) 290 | return pongo2.AsValue(v), nil 291 | } 292 | 293 | return in, nil 294 | } 295 | -------------------------------------------------------------------------------- /pkg/template/template_filters_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2016 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | package template 10 | 11 | import ( 12 | "testing" 13 | 14 | "github.com/HeavyHorst/memkv" 15 | "github.com/HeavyHorst/pongo2" 16 | . "gopkg.in/check.v1" 17 | ) 18 | 19 | // Hook up gocheck into the "go test" runner. 20 | func Test(t *testing.T) { TestingT(t) } 21 | 22 | type FilterSuite struct{} 23 | 24 | var _ = Suite(&FilterSuite{}) 25 | 26 | func (s *FilterSuite) TestFilterBase64(t *C) { 27 | in := pongo2.AsValue("foo") 28 | res, err := filterBase64(in, nil) 29 | if err != nil { 30 | t.Error(err.OrigError) 31 | } 32 | 33 | t.Check(res.String(), Equals, "Zm9v") 34 | } 35 | 36 | func (s *FilterSuite) TestFilterBase(t *C) { 37 | in := pongo2.AsValue("/etc/foo/bar") 38 | res, err := filterBase(in, nil) 39 | if err != nil { 40 | t.Error(err.OrigError) 41 | } 42 | 43 | t.Check(res.String(), Equals, "bar") 44 | } 45 | 46 | func (s *FilterSuite) TestFilterParseInt(t *C) { 47 | in := pongo2.AsValue("100") 48 | res, err := filterParseInt(in, nil) 49 | if err != nil { 50 | t.Error(err.OrigError) 51 | } 52 | 53 | t.Check(res.Integer(), Equals, 100) 54 | } 55 | 56 | func (s *FilterSuite) TestFilterParseFloat(t *C) { 57 | in := pongo2.AsValue("22.45") 58 | res, err := filterParseFloat(in, nil) 59 | if err != nil { 60 | t.Error(err.OrigError) 61 | } 62 | 63 | t.Check(res.Float(), Equals, 22.45) 64 | } 65 | 66 | func (s *FilterSuite) TestFilterDir(t *C) { 67 | in := pongo2.AsValue("/etc/foo/bar") 68 | res, err := filterDir(in, nil) 69 | if err != nil { 70 | t.Error(err.OrigError) 71 | } 72 | 73 | t.Check(res.String(), Equals, "/etc/foo") 74 | } 75 | 76 | func (s *FilterSuite) TestFilterToPrettyJSON(t *C) { 77 | expected := `{ 78 | "test": "bla", 79 | "test2": 1, 80 | "test3": 2.5 81 | }` 82 | in := pongo2.AsValue(map[string]interface{}{ 83 | "test": "bla", 84 | "test2": 1, 85 | "test3": 2.5, 86 | }) 87 | res, err := filterToPrettyJSON(in, nil) 88 | if err != nil { 89 | t.Error(err.OrigError) 90 | } 91 | 92 | t.Check(res.String(), Equals, expected) 93 | } 94 | 95 | func (s *FilterSuite) TestFilterToJSON(t *C) { 96 | expected := `{"test":"bla","test2":1,"test3":2.5}` 97 | in := pongo2.AsValue(map[string]interface{}{ 98 | "test": "bla", 99 | "test2": 1, 100 | "test3": 2.5, 101 | }) 102 | res, err := filterToJSON(in, nil) 103 | if err != nil { 104 | t.Error(err.OrigError) 105 | } 106 | 107 | t.Check(res.String(), Equals, expected) 108 | } 109 | 110 | func (s *FilterSuite) TestFilterToYAML(t *C) { 111 | expected := `test: bla 112 | test2: 1 113 | test3: 2.5 114 | ` 115 | in := pongo2.AsValue(map[string]interface{}{ 116 | "test": "bla", 117 | "test2": 1, 118 | "test3": 2.5, 119 | }) 120 | res, err := filterToYAML(in, nil) 121 | if err != nil { 122 | t.Error(err.OrigError) 123 | } 124 | 125 | t.Check(res.String(), Equals, expected) 126 | } 127 | 128 | func (s *FilterSuite) TestFilterUnmarshalYAMLObject(t *C) { 129 | in := pongo2.AsValue(`{"test":"bla","test2":"1","test3":"2.5"}`) 130 | expected := map[string]interface{}{ 131 | "test": "bla", 132 | "test2": "1", 133 | "test3": "2.5", 134 | } 135 | res, err := filterUnmarshalYAML(in, nil) 136 | if err != nil { 137 | t.Error(err.OrigError) 138 | } 139 | m1 := res.Interface().(map[string]interface{}) 140 | t.Check(m1, DeepEquals, expected) 141 | } 142 | 143 | func (s *FilterSuite) TestFilterUnmarshalYAMLArray(t *C) { 144 | in := pongo2.AsValue(`["a", "b", "c"]`) 145 | expected := []interface{}{"a", "b", "c"} 146 | res, err := filterUnmarshalYAML(in, nil) 147 | if err != nil { 148 | t.Error(err.OrigError) 149 | } 150 | m1 := res.Interface().([]interface{}) 151 | t.Check(m1, DeepEquals, expected) 152 | } 153 | 154 | func (s *FilterSuite) TestFilterSortByLengthString(t *C) { 155 | in := pongo2.AsValue([]string{"123", "foobar", "1234"}) 156 | expected := []string{"123", "1234", "foobar"} 157 | res, err := filterSortByLength(in, nil) 158 | if err != nil { 159 | t.Error(err.OrigError) 160 | } 161 | m1 := res.Interface().([]string) 162 | t.Check(m1, DeepEquals, expected) 163 | } 164 | 165 | func (s *FilterSuite) TestFilterSortByLengthKVPair(t *C) { 166 | a := memkv.KVPair{Key: "123", Value: "Test"} 167 | b := memkv.KVPair{Key: "1234", Value: "Test"} 168 | c := memkv.KVPair{Key: "foobar", Value: "Test"} 169 | in := pongo2.AsValue(memkv.KVPairs{a, c, b}) 170 | expected := memkv.KVPairs{a, b, c} 171 | res, err := filterSortByLength(in, nil) 172 | if err != nil { 173 | t.Error(err.OrigError) 174 | } 175 | m1 := res.Interface().(memkv.KVPairs) 176 | t.Check(m1, DeepEquals, expected) 177 | } 178 | 179 | func (s *FilterSuite) TestFilterMapValue(t *C) { 180 | a := map[string]int{ 181 | "hallo": 1, 182 | "moin": 2, 183 | } 184 | fm, err := filterMapValue(pongo2.AsValue(a), pongo2.AsValue("moin")) 185 | if err != nil { 186 | t.Error(err.OrigError) 187 | } 188 | t.Check(fm.Interface(), DeepEquals, 2) 189 | 190 | b := map[string]string{ 191 | "servus": "one", 192 | "hi": "oneone", 193 | } 194 | fm, err = filterMapValue(pongo2.AsValue(b), pongo2.AsValue("servus")) 195 | if err != nil { 196 | t.Error(err.OrigError) 197 | } 198 | t.Check(fm.Interface(), DeepEquals, "one") 199 | 200 | c := map[string]interface{}{ 201 | "servus": "one", 202 | "hi": 100, 203 | } 204 | fm, err = filterMapValue(pongo2.AsValue(c), pongo2.AsValue("hi")) 205 | if err != nil { 206 | t.Error(err.OrigError) 207 | } 208 | t.Check(fm.Interface(), DeepEquals, 100) 209 | 210 | d := map[int]string{ 211 | 1: "one", 212 | 2: "oneone", 213 | } 214 | fm, err = filterMapValue(pongo2.AsValue(d), pongo2.AsValue(2)) 215 | if err != nil { 216 | t.Error(err.OrigError) 217 | } 218 | t.Check(fm.Interface(), DeepEquals, "oneone") 219 | } 220 | 221 | func (s *FilterSuite) TestFilterIndex(t *C) { 222 | in := pongo2.AsValue([]string{"Hallo", "Test", "123", "Moin"}) 223 | expected := "123" 224 | 225 | res, err := filterIndex(in, pongo2.AsValue(2)) 226 | if err != nil { 227 | t.Error(err.OrigError) 228 | } 229 | 230 | // test negative index 231 | res2, err := filterIndex(in, pongo2.AsValue(-2)) 232 | if err != nil { 233 | t.Error(err.OrigError) 234 | } 235 | 236 | m1 := res.String() 237 | m2 := res2.String() 238 | t.Check(m1, DeepEquals, expected) 239 | t.Check(m2, DeepEquals, expected) 240 | } 241 | -------------------------------------------------------------------------------- /pkg/template/template_funcs.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * Based on code from confd. 4 | * https://github.com/kelseyhightower/confd/blob/6bb3c21a63459c3be340d53c4d3463397c8324c6/resource/template/template_funcs.go 5 | * © 2013 Kelsey Hightower 6 | * © 2015 Justin Burnham 7 | * © 2016 odedlaz 8 | * 9 | * © 2016 The Remco Authors 10 | * 11 | * For the full copyright and license information, please view the LICENSE 12 | * file that was distributed with this source code. 13 | */ 14 | 15 | package template 16 | 17 | import ( 18 | "encoding/json" 19 | "fmt" 20 | "net" 21 | "os" 22 | "sort" 23 | "strconv" 24 | "strings" 25 | "time" 26 | 27 | "github.com/HeavyHorst/remco/pkg/template/fileutil" 28 | ) 29 | 30 | type interfaceSet map[string]struct{} 31 | 32 | func (s interfaceSet) Append(value interface{}) string { 33 | v := fmt.Sprintf("%v", value) 34 | s[v] = struct{}{} 35 | return "" 36 | } 37 | 38 | func (s interfaceSet) Remove(value string) string { 39 | v := fmt.Sprintf("%v", value) 40 | delete(s, v) 41 | return "" 42 | } 43 | 44 | func (s interfaceSet) Contains(value interface{}) bool { 45 | v := fmt.Sprintf("%v", value) 46 | _, c := s[v] 47 | return c 48 | } 49 | 50 | func (s interfaceSet) SortedSet() []string { 51 | var i []string 52 | for k := range s { 53 | i = append(i, k) 54 | } 55 | sort.Strings(i) 56 | return i 57 | } 58 | 59 | func (s interfaceSet) MarshalYAML() (interface{}, error) { 60 | return s.SortedSet(), nil 61 | } 62 | 63 | func (s interfaceSet) MarshalJSON() ([]byte, error) { 64 | return json.Marshal(s.SortedSet()) 65 | } 66 | 67 | type templateMap map[string]interface{} 68 | 69 | func (t templateMap) Set(key string, value interface{}) string { 70 | t[key] = value 71 | return "" 72 | } 73 | 74 | func (t templateMap) Remove(key string) string { 75 | delete(t, key) 76 | return "" 77 | } 78 | 79 | func (t templateMap) Get(key string) interface{} { 80 | return t[key] 81 | } 82 | 83 | func newFuncMap() map[string]interface{} { 84 | m := map[string]interface{}{ 85 | "getenv": getenv, 86 | "contains": strings.Contains, 87 | "replace": strings.Replace, 88 | "lookupIP": lookupIP, 89 | "lookupSRV": lookupSRV, 90 | "fileExists": fileutil.IsFileExist, 91 | "printf": fmt.Sprintf, 92 | "unixTS": unixTimestampNow, 93 | "dateRFC3339": dateRFC3339Now, 94 | "createMap": createMap, 95 | "createSet": createSet, 96 | } 97 | 98 | return m 99 | } 100 | 101 | func addFuncs(out, in map[string]interface{}) { 102 | for name, fn := range in { 103 | out[name] = fn 104 | } 105 | } 106 | 107 | // Getenv retrieves the value of the environment variable named by the key. 108 | // It returns the value, which will the default value if the variable is not present. 109 | // If no default value was given - returns "". 110 | func getenv(key string, v ...string) string { 111 | defaultValue := "" 112 | if len(v) > 0 { 113 | defaultValue = v[0] 114 | } 115 | 116 | value := os.Getenv(key) 117 | if value == "" { 118 | return defaultValue 119 | } 120 | return value 121 | } 122 | 123 | func lookupIP(data string) ([]string, error) { 124 | ips, err := net.LookupIP(data) 125 | if err != nil { 126 | return nil, err 127 | } 128 | // "Cast" IPs into strings and sort the array 129 | ipStrings := make([]string, len(ips)) 130 | 131 | for i, ip := range ips { 132 | ipStrings[i] = ip.String() 133 | } 134 | sort.Strings(ipStrings) 135 | return ipStrings, nil 136 | } 137 | 138 | func createMap() templateMap { 139 | tm := make(map[string]interface{}) 140 | return tm 141 | } 142 | 143 | func createSet() interfaceSet { 144 | return make(map[string]struct{}) 145 | } 146 | 147 | func lookupSRV(service, proto, name string) ([]*net.SRV, error) { 148 | _, addrs, err := net.LookupSRV(service, proto, name) 149 | if err != nil { 150 | return nil, err 151 | } 152 | sort.Slice(addrs, func(i, j int) bool { 153 | str1 := fmt.Sprintf("%s%d%d%d", addrs[i].Target, addrs[i].Port, addrs[i].Priority, addrs[i].Weight) 154 | str2 := fmt.Sprintf("%s%d%d%d", addrs[j].Target, addrs[j].Port, addrs[j].Priority, addrs[j].Weight) 155 | return str1 < str2 156 | }) 157 | return addrs, nil 158 | } 159 | 160 | func unixTimestampNow() string { 161 | return strconv.FormatInt(time.Now().Unix(), 10) 162 | } 163 | 164 | func dateRFC3339Now() string { 165 | return time.Now().Format(time.RFC3339) 166 | } 167 | -------------------------------------------------------------------------------- /pkg/template/template_funcs_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * © 2016 The Remco Authors 4 | * 5 | * For the full copyright and license information, please view the LICENSE 6 | * file that was distributed with this source code. 7 | */ 8 | 9 | package template 10 | 11 | import ( 12 | "net" 13 | "os" 14 | 15 | . "gopkg.in/check.v1" 16 | ) 17 | 18 | type FunctionTestSuite struct{} 19 | 20 | var _ = Suite(&FunctionTestSuite{}) 21 | 22 | func (s *FunctionTestSuite) TestAddFuncs(t *C) { 23 | in := map[string]interface{}{ 24 | "a": "hallo", 25 | "b": "hello", 26 | } 27 | out := make(map[string]interface{}) 28 | 29 | addFuncs(out, in) 30 | 31 | t.Check(len(out), Equals, len(in)) 32 | } 33 | 34 | func (s *FunctionTestSuite) TestLookupIP(t *C) { 35 | ips, err := lookupIP("localhost") 36 | if err != nil { 37 | t.Error(err) 38 | } 39 | if len(ips) > 0 { 40 | t.Check(ips[0], Equals, "127.0.0.1") 41 | } else { 42 | t.Error("lookupIP failed") 43 | } 44 | } 45 | 46 | func (s *FunctionTestSuite) TestLookupSRV(t *C) { 47 | expected := []*net.SRV{ 48 | { 49 | Target: "alt1.xmpp-server.l.google.com.", 50 | Port: 5269, 51 | Priority: 20, 52 | Weight: 0, 53 | }, 54 | { 55 | Target: "alt2.xmpp-server.l.google.com.", 56 | Port: 5269, 57 | Priority: 20, 58 | Weight: 0, 59 | }, 60 | { 61 | Target: "alt3.xmpp-server.l.google.com.", 62 | Port: 5269, 63 | Priority: 20, 64 | Weight: 0, 65 | }, 66 | { 67 | Target: "alt4.xmpp-server.l.google.com.", 68 | Port: 5269, 69 | Priority: 20, 70 | Weight: 0, 71 | }, 72 | { 73 | Target: "xmpp-server.l.google.com.", 74 | Port: 5269, 75 | Priority: 5, 76 | Weight: 0, 77 | }, 78 | } 79 | 80 | srv, err := lookupSRV("xmpp-server", "tcp", "google.com") 81 | if err != nil { 82 | t.Error(err) 83 | } 84 | t.Check(srv, DeepEquals, expected) 85 | } 86 | 87 | func (s *FunctionTestSuite) TestGetEnv(t *C) { 88 | key := "coolEnvVar" 89 | expected := "mmmh lecker saure Gurken!" 90 | err := os.Setenv(key, expected) 91 | if err != nil { 92 | t.Error(err) 93 | } 94 | 95 | t.Check(getenv(key), Equals, expected) 96 | } 97 | 98 | func (s *FunctionTestSuite) TestGetEnvDefault(t *C) { 99 | key := "ihopethisenvvardontexists" 100 | expected := "default" 101 | 102 | t.Check(getenv(key, "default"), Equals, expected) 103 | } 104 | 105 | func (s *FunctionTestSuite) TestInterfaceSet(t *C) { 106 | set := createSet() 107 | set.Append("Hallo") 108 | set.Append("Hallo") 109 | set.Append(1) 110 | set.Append(true) 111 | set.Append(false) 112 | 113 | t.Check(len(set), Equals, 4) 114 | t.Check(set.Contains("Hallo"), Equals, true) 115 | set.Remove("Hallo") 116 | t.Check(len(set), Equals, 3) 117 | t.Check(set.Contains("Hallo"), Equals, false) 118 | t.Check(set.Contains(false), Equals, true) 119 | 120 | t.Check(len(set.SortedSet()), Equals, 3) 121 | t.Check(set.SortedSet()[0], Equals, "1") 122 | t.Check(set.SortedSet()[1], Equals, "false") 123 | t.Check(set.SortedSet()[2], Equals, "true") 124 | } 125 | 126 | func (s *FunctionTestSuite) TestTemplateMap(t *C) { 127 | m := createMap() 128 | m.Set("Hallo", "OneOneOne") 129 | m.Set("Test", "Snickers") 130 | m.Set("One", 1) 131 | 132 | t.Check(m.Get("Hallo"), DeepEquals, "OneOneOne") 133 | t.Check(m.Get("One"), DeepEquals, 1) 134 | 135 | m.Remove("One") 136 | t.Check(m.Get("One"), DeepEquals, nil) 137 | } 138 | -------------------------------------------------------------------------------- /pkg/template/util.go: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of remco. 3 | * Based on confd. 4 | * https://github.com/kelseyhightower/confd/blob/abba746a0cb7c8cb5fe135fa2d884ea3c4a5f666/resource/template/util.go 5 | * © 2013 Kelsey Hightower 6 | * © 2016 The Remco Authors 7 | * 8 | * For the full copyright and license information, please view the LICENSE 9 | * file that was distributed with this source code. 10 | */ 11 | 12 | package template 13 | 14 | import ( 15 | "path" 16 | ) 17 | 18 | func appendPrefix(prefix string, keys []string) []string { 19 | s := make([]string, len(keys)) 20 | for i, k := range keys { 21 | s[i] = path.Join(prefix, k) 22 | } 23 | return s 24 | } 25 | -------------------------------------------------------------------------------- /scripts/deploy-ghpages.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e # Exit with nonzero exit code if anything fails 3 | 4 | SOURCE_BRANCH="master" 5 | TARGET_BRANCH="gh-pages" 6 | 7 | # Save some useful information 8 | REPO=`git config remote.origin.url` 9 | SSH_REPO=${REPO/https:\/\/github.com\//git@github.com:} 10 | SHA=`git rev-parse --verify HEAD` 11 | 12 | # Clone the existing gh-pages for this repo into out/ 13 | # Create a new empty branch if gh-pages doesn't exist yet (should only happen on first deply) 14 | git clone $REPO out 15 | cd out 16 | git checkout $TARGET_BRANCH || git checkout --orphan $TARGET_BRANCH 17 | cd .. 18 | 19 | # Clean out existing contents 20 | rm -rf out/**/* || exit 0 21 | 22 | # Install hugo 23 | wget https://github.com/spf13/hugo/releases/download/v0.17/hugo_0.17_Linux-64bit.tar.gz 24 | tar -xf hugo_0.17_Linux-64bit.tar.gz 25 | mv hugo_0.17_linux_amd64/hugo_0.17_linux_amd64 hugo 26 | chmod +x hugo 27 | 28 | # Build the page 29 | cd docs 30 | ../hugo 31 | cd .. 32 | mv docs/public/* out/ 33 | 34 | cd out 35 | 36 | # If there are no changes (e.g. this is a README update) then just bail. 37 | if [ -z `git diff --exit-code` ]; then 38 | echo "No changes to the spec on this push; exiting." 39 | exit 0 40 | fi 41 | 42 | # Commit the "changes", i.e. the new version. 43 | # The delta will show diffs between new and old versions. 44 | git add --all 45 | git commit -m "Deploy to GitHub Pages: ${SHA}" 46 | 47 | # Now that we're all set up, we can push. 48 | git push $SSH_REPO $TARGET_BRANCH 49 | -------------------------------------------------------------------------------- /test: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | echo "" > coverage.txt 5 | 6 | for d in $(go list ./... | grep -v vendor); do 7 | go test -race -coverprofile=profile.out $d 8 | if [ -f profile.out ]; then 9 | cat profile.out >> coverage.txt 10 | rm profile.out 11 | fi 12 | done 13 | --------------------------------------------------------------------------------