├── .github ├── CODEOWNERS └── workflows │ └── ci.yaml ├── .gitignore ├── Dockerfile ├── Gopkg.lock ├── Gopkg.toml ├── LICENSE ├── Makefile ├── README.md ├── cmd ├── command.go ├── config.go ├── handler.go ├── handler_test.go └── version.go ├── golang.mk ├── main.go └── merger.yaml /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @rebuy-de/prp-exporter-merger 2 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: Golang 2 | 3 | on: 4 | push: 5 | branches: [master] 6 | pull_request: 7 | types: [opened, reopened, synchronize] 8 | release: 9 | types: [published] 10 | schedule: 11 | - cron: '15 3 * * 0' 12 | 13 | jobs: 14 | build: 15 | runs-on: ubuntu-22.04 16 | name: CI Build 17 | 18 | steps: 19 | - name: Setup Go 20 | uses: actions/setup-go@v2 21 | with: 22 | go-version: '1.19' 23 | - name: Checkout code 24 | uses: actions/checkout@v3 25 | with: 26 | fetch-depth: 0 27 | path: go/src/github.com/rebuy-de/exporter-merger 28 | - name: Setup tools 29 | env: 30 | GO111MODULE: off 31 | GOPATH: /home/runner/work/exporter-merger/exporter-merger/go 32 | run: | 33 | cd go/src/github.com/rebuy-de/exporter-merger 34 | go get -u golang.org/x/lint/golint 35 | go get -u github.com/golang/dep/cmd/dep 36 | echo "/home/runner/work/exporter-merger/exporter-merger/go/bin" >> $GITHUB_PATH 37 | - name: Build Project 38 | env: 39 | GO111MODULE: off 40 | GOPATH: /home/runner/work/exporter-merger/exporter-merger/go 41 | run: | 42 | cd go/src/github.com/rebuy-de/exporter-merger 43 | make vendor 44 | make 45 | 46 | container_build: 47 | runs-on: ubuntu-22.04 48 | name: Container Build 49 | 50 | steps: 51 | - uses: actions/checkout@v3 52 | with: 53 | fetch-depth: 0 54 | 55 | - name: Generate image tags for releaes 56 | if: ${{ github.event_name == 'release' }} 57 | shell: bash 58 | run: echo "##[set-output name=tags;]quay.io/rebuy/exporter-merger:${GITHUB_REF#refs/tags/},074509403805.dkr.ecr.eu-west-1.amazonaws.com/exporter-merger:${GITHUB_REF#refs/tags/}" 59 | id: generate_tags_release 60 | 61 | - name: Generate image tags for PRs 62 | if: ${{ github.event_name != 'release' }} 63 | shell: bash 64 | run: | 65 | if [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then 66 | echo "##[set-output name=tags;]quay.io/rebuy/exporter-merger:${GITHUB_HEAD_REF},074509403805.dkr.ecr.eu-west-1.amazonaws.com/exporter-merger:${GITHUB_HEAD_REF}" 67 | else 68 | echo "##[set-output name=tags;]quay.io/rebuy/exporter-merger:master,074509403805.dkr.ecr.eu-west-1.amazonaws.com/exporter-merger:master,\ 69 | quay.io/rebuy/exporter-merger:latest,074509403805.dkr.ecr.eu-west-1.amazonaws.com/exporter-merger:latest" 70 | fi 71 | id: generate_tags_pr 72 | 73 | - name: Set up QEMU 74 | if: ${{ github.event_name == 'release' }} 75 | id: qemu 76 | uses: docker/setup-qemu-action@v1 77 | with: 78 | platforms: arm64 79 | 80 | - name: Set up Docker Buildx 81 | uses: docker/setup-buildx-action@v1 82 | with: 83 | install: true 84 | 85 | # Only used to prevent rate limits 86 | - name: Login to Docker Hub 87 | uses: docker/login-action@v1 88 | with: 89 | username: ${{ secrets.DOCKER_USERNAME }} 90 | password: ${{ secrets.DOCKER_PASSWORD }} 91 | 92 | - name: Login to ECR 93 | uses: docker/login-action@v1 94 | with: 95 | registry: 074509403805.dkr.ecr.eu-west-1.amazonaws.com 96 | username: ${{ secrets.AWS_ECR_ACCESS_KEY_ID }} 97 | password: ${{ secrets.AWS_ECR_SECRET_ACCESS_KEY }} 98 | 99 | - name: Login to Quay.io 100 | uses: docker/login-action@v1 101 | with: 102 | registry: quay.io 103 | username: ${{ secrets.QUAY_USERNAME }} 104 | password: ${{ secrets.QUAY_PASSWORD }} 105 | 106 | - name: Build and push 107 | if: ${{ github.event_name == 'release' }} 108 | uses: docker/build-push-action@v2 109 | with: 110 | context: . 111 | push: true 112 | tags: ${{ steps.generate_tags_release.outputs.tags }} 113 | platforms: linux/amd64,linux/arm64 114 | 115 | - name: Build and push 116 | if: ${{ github.event_name != 'release' }} 117 | uses: docker/build-push-action@v2 118 | with: 119 | context: . 120 | push: true 121 | tags: ${{ steps.generate_tags_pr.outputs.tags }} 122 | platforms: linux/amd64 123 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /exporter-merger* 2 | /vendor/ 3 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.19-alpine AS build-env 2 | 3 | RUN apk add --no-cache git make 4 | 5 | # Configure Go 6 | ENV GOPATH /go 7 | ENV PATH /go/bin:$PATH 8 | ENV GO111MODULE=off 9 | RUN mkdir -p ${GOPATH}/src ${GOPATH}/bin 10 | 11 | # Install Go Tools 12 | RUN go get -u golang.org/x/lint/golint 13 | RUN go get -u github.com/golang/dep/cmd/dep 14 | 15 | ADD . /go/src/github.com/rebuy-de/exporter-merger/ 16 | RUN cd /go/src/github.com/rebuy-de/exporter-merger/ && make vendor && CGO_ENABLED=0 make install 17 | 18 | # final stage 19 | FROM alpine 20 | WORKDIR /app 21 | COPY --from=build-env /go/src/github.com/rebuy-de/exporter-merger/merger.yaml /app/ 22 | COPY --from=build-env /go/bin/exporter-merger /app/ 23 | ENTRYPOINT ./exporter-merger 24 | -------------------------------------------------------------------------------- /Gopkg.lock: -------------------------------------------------------------------------------- 1 | # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. 2 | 3 | 4 | [[projects]] 5 | name = "github.com/fsnotify/fsnotify" 6 | packages = ["."] 7 | revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" 8 | version = "v1.4.7" 9 | 10 | [[projects]] 11 | name = "github.com/golang/protobuf" 12 | packages = ["proto"] 13 | revision = "925541529c1fa6821df4e44ce2723319eb2be768" 14 | version = "v1.0.0" 15 | 16 | [[projects]] 17 | name = "github.com/hashicorp/hcl" 18 | packages = [ 19 | ".", 20 | "hcl/ast", 21 | "hcl/parser", 22 | "hcl/printer", 23 | "hcl/scanner", 24 | "hcl/strconv", 25 | "hcl/token", 26 | "json/parser", 27 | "json/scanner", 28 | "json/token" 29 | ] 30 | revision = "8cb6e5b959231cc1119e43259c4a608f9c51a241" 31 | version = "v1.0.0" 32 | 33 | [[projects]] 34 | name = "github.com/inconshreveable/mousetrap" 35 | packages = ["."] 36 | revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75" 37 | version = "v1.0" 38 | 39 | [[projects]] 40 | name = "github.com/magiconair/properties" 41 | packages = ["."] 42 | revision = "c2353362d570a7bfa228149c62842019201cfb71" 43 | version = "v1.8.0" 44 | 45 | [[projects]] 46 | name = "github.com/matttproud/golang_protobuf_extensions" 47 | packages = ["pbutil"] 48 | revision = "3247c84500bff8d9fb6d579d800f20b3e091582c" 49 | version = "v1.0.0" 50 | 51 | [[projects]] 52 | name = "github.com/mitchellh/mapstructure" 53 | packages = ["."] 54 | revision = "3536a929edddb9a5b34bd6861dc4a9647cb459fe" 55 | version = "v1.1.2" 56 | 57 | [[projects]] 58 | name = "github.com/pelletier/go-toml" 59 | packages = ["."] 60 | revision = "c01d1270ff3e442a8a57cddc1c92dc1138598194" 61 | version = "v1.2.0" 62 | 63 | [[projects]] 64 | name = "github.com/pkg/errors" 65 | packages = ["."] 66 | revision = "645ef00459ed84a119197bfb8d8205042c6df63d" 67 | version = "v0.8.0" 68 | 69 | [[projects]] 70 | branch = "master" 71 | name = "github.com/prometheus/client_model" 72 | packages = ["go"] 73 | revision = "99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c" 74 | 75 | [[projects]] 76 | branch = "master" 77 | name = "github.com/prometheus/common" 78 | packages = [ 79 | "expfmt", 80 | "internal/bitbucket.org/ww/goautoneg", 81 | "model" 82 | ] 83 | revision = "89604d197083d4781071d3c65855d24ecfb0a563" 84 | 85 | [[projects]] 86 | name = "github.com/sirupsen/logrus" 87 | packages = ["."] 88 | revision = "d682213848ed68c0a260ca37d6dd5ace8423f5ba" 89 | version = "v1.0.4" 90 | 91 | [[projects]] 92 | name = "github.com/spf13/afero" 93 | packages = [ 94 | ".", 95 | "mem" 96 | ] 97 | revision = "a5d6946387efe7d64d09dcba68cdd523dc1273a3" 98 | version = "v1.2.0" 99 | 100 | [[projects]] 101 | name = "github.com/spf13/cast" 102 | packages = ["."] 103 | revision = "8c9545af88b134710ab1cd196795e7f2388358d7" 104 | version = "v1.3.0" 105 | 106 | [[projects]] 107 | branch = "master" 108 | name = "github.com/spf13/cobra" 109 | packages = ["."] 110 | revision = "be77323fc05148ef091e83b3866c0d47c8e74a8b" 111 | 112 | [[projects]] 113 | name = "github.com/spf13/jwalterweatherman" 114 | packages = ["."] 115 | revision = "4a4406e478ca629068e7768fc33f3f044173c0a6" 116 | version = "v1.0.0" 117 | 118 | [[projects]] 119 | name = "github.com/spf13/pflag" 120 | packages = ["."] 121 | revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66" 122 | version = "v1.0.0" 123 | 124 | [[projects]] 125 | name = "github.com/spf13/viper" 126 | packages = ["."] 127 | revision = "6d33b5a963d922d182c91e8a1c88d81fd150cfd4" 128 | version = "v1.3.1" 129 | 130 | [[projects]] 131 | branch = "master" 132 | name = "golang.org/x/crypto" 133 | packages = ["ssh/terminal"] 134 | revision = "9de5f2eaf759b4c4550b3db39fed2e9e5f86f45c" 135 | 136 | [[projects]] 137 | branch = "master" 138 | name = "golang.org/x/sys" 139 | packages = [ 140 | "unix", 141 | "windows" 142 | ] 143 | revision = "37707fdb30a5b38865cfb95e5aab41707daec7fd" 144 | 145 | [[projects]] 146 | name = "golang.org/x/text" 147 | packages = [ 148 | "internal/gen", 149 | "internal/triegen", 150 | "internal/ucd", 151 | "transform", 152 | "unicode/cldr", 153 | "unicode/norm" 154 | ] 155 | revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" 156 | version = "v0.3.0" 157 | 158 | [[projects]] 159 | branch = "v2" 160 | name = "gopkg.in/yaml.v2" 161 | packages = ["."] 162 | revision = "d670f9405373e636a5a2765eea47fac0c9bc91a4" 163 | 164 | [solve-meta] 165 | analyzer-name = "dep" 166 | analyzer-version = 1 167 | inputs-digest = "0cd4b0fb858b550e04e72c7e12a4240acf5f606484728c17a2fc735942e0ead7" 168 | solver-name = "gps-cdcl" 169 | solver-version = 1 170 | -------------------------------------------------------------------------------- /Gopkg.toml: -------------------------------------------------------------------------------- 1 | [[constraint]] 2 | name = "github.com/sirupsen/logrus" 3 | version = "1.0.4" 4 | 5 | [[constraint]] 6 | branch = "master" 7 | name = "github.com/spf13/cobra" 8 | 9 | [[constraint]] 10 | branch = "master" 11 | name = "github.com/prometheus/common" 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018 reBuy reCommerce GmbH 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PACKAGE=github.com/rebuy-de/exporter-merger 2 | 3 | include golang.mk 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # exporter-merger 2 | 3 | > :warning: We archived this project because we don't use it anymore and in 4 | > the meantime some other solutions for the problem described in 5 | > [prometheus/prometheus#3756](https://github.com/prometheus/prometheus/issues/3756) 6 | > became available. 7 | 8 | [![license](https://img.shields.io/github/license/rebuy-de/exporter-merger.svg)]() 9 | [![GitHub release](https://img.shields.io/github/release/rebuy-de/exporter-merger.svg)]() 10 | 11 | Merges Prometheus metrics from multiple sources. 12 | 13 | > **Development Status** *exporter-merger* is in an early development phase. 14 | > Expect incompatible changes and abandoment at any time. 15 | 16 | ## But Why?! 17 | 18 | > [prometheus/prometheus#3756](https://github.com/prometheus/prometheus/issues/3756) 19 | 20 | ## Usage 21 | 22 | *exporter-merger* needs a configuration file. Currently, nothing but URLs are accepted: 23 | 24 | ```yaml 25 | exporters: 26 | - url: http://localhost:9100/metrics 27 | - url: http://localhost:9101/metrics 28 | ``` 29 | 30 | To start the exporter: 31 | 32 | ``` 33 | exporter-merger --config-path merger.yaml --listen-port 8080 34 | ``` 35 | 36 | ### Environment variables 37 | 38 | Alternatively configuration can be passed via environment variables, here is relevant part of `exporter-merger -h` output: 39 | ``` 40 | --listen-port int Listen port for the HTTP server. (ENV:MERGER_PORT) (default 8080) 41 | --url stringSlice URL to scrape. Can be speficied multiple times. (ENV:MERGER_URLS,space-seperated) 42 | 43 | ``` 44 | 45 | ## Kubernetes 46 | 47 | The exporter-merger is supposed to run as a sidecar. Here is an example config with [nginx-exporter](https://github.com/rebuy-de/nginx-exporter): 48 | 49 | ```yaml 50 | apiVersion: apps/v1 51 | kind: Deployment 52 | 53 | metadata: 54 | name: my-nginx 55 | labels: 56 | app: my-nginx 57 | 58 | spec: 59 | selector: 60 | matchLabels: 61 | app: my-nginx 62 | 63 | template: 64 | metadata: 65 | name: my-nginx 66 | labels: 67 | app: my-nginx 68 | annotations: 69 | prometheus.io/scrape: "true" 70 | prometheus.io/port: "8080" 71 | 72 | spec: 73 | containers: 74 | - name: "nginx" 75 | image: "my-nginx" # nginx image with modified config file 76 | 77 | volumeMounts: 78 | - name: mtail 79 | mountPath: /var/log/nginx/mtail 80 | 81 | - name: nginx-exporter 82 | image: quay.io/rebuy/nginx-exporter:v1.1.0 83 | ports: 84 | - containerPort: 9397 85 | env: 86 | - name: NGINX_ACCESS_LOGS 87 | value: /var/log/nginx/mtail/access.log 88 | - name: NGINX_STATUS_URI 89 | value: http://localhost:8888/nginx_status 90 | volumeMounts: 91 | - name: mtail 92 | mountPath: /var/log/nginx/mtail 93 | 94 | - name: exporter-merger 95 | image: quay.io/rebuy/exporter-merger:v0.2.0 96 | ports: 97 | - containerPort: 8080 98 | env: 99 | # space-separated list of URLs 100 | - name: MERGER_URLS 101 | value: http://localhost:9000/prometheus/metrics http://localhost:9397/metrics 102 | # default exposed port, change only if need other than default 8080 103 | # - name: MERGER_PORT 104 | # value: 8080 105 | ``` 106 | 107 | ## Planned Features 108 | 109 | * Allow transforming of metrics from backend exporters. 110 | * eg add a prefix to the metric names 111 | * eg add labels to the metrics 112 | * Allow dynamic adding of exporters. 113 | -------------------------------------------------------------------------------- /cmd/command.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "os" 7 | "strings" 8 | 9 | log "github.com/sirupsen/logrus" 10 | "github.com/spf13/cobra" 11 | "github.com/spf13/viper" 12 | ) 13 | 14 | func NewRootCommand() *cobra.Command { 15 | app := new(App) 16 | 17 | cmd := &cobra.Command{ 18 | Use: "exporter-merger", 19 | Short: "merges Prometheus metrics from multiple sources", 20 | Run: app.run, 21 | PersistentPreRun: func(cmd *cobra.Command, args []string) { 22 | if app.viper.GetBool("verbose") { 23 | log.SetLevel(log.DebugLevel) 24 | } else { 25 | log.SetLevel(log.InfoLevel) 26 | } 27 | }, 28 | } 29 | 30 | app.Bind(cmd) 31 | 32 | cmd.AddCommand(NewVersionCommand()) 33 | 34 | return cmd 35 | } 36 | 37 | type App struct { 38 | viper *viper.Viper 39 | } 40 | 41 | func (app *App) Bind(cmd *cobra.Command) { 42 | app.viper = viper.New() 43 | app.viper.SetEnvPrefix("MERGER") 44 | app.viper.AutomaticEnv() 45 | 46 | configPath := cmd.PersistentFlags().StringP( 47 | "config-path", "c", "", 48 | "Path to the configuration file.") 49 | cobra.OnInitialize(func() { 50 | if configPath != nil && *configPath != "" { 51 | config, err := ReadConfig(*configPath) 52 | if err != nil { 53 | log.WithField("error", err).Errorf("failed to load config file '%s'", *configPath) 54 | os.Exit(1) 55 | return 56 | } 57 | 58 | urls := []string{} 59 | for _, e := range config.Exporters { 60 | urls = append(urls, e.URL) 61 | } 62 | app.viper.SetDefault("urls", strings.Join(urls, " ")) 63 | } 64 | }) 65 | 66 | cmd.PersistentFlags().Int( 67 | "listen-port", 8080, 68 | "Listen port for the HTTP server. (ENV:MERGER_PORT)") 69 | app.viper.BindPFlag("port", cmd.PersistentFlags().Lookup("listen-port")) 70 | 71 | cmd.PersistentFlags().Int( 72 | "exporters-timeout", 10, 73 | "HTTP client timeout for connecting to exporters. (ENV:MERGER_EXPORTERSTIMEOUT)") 74 | app.viper.BindPFlag("exporterstimeout", cmd.PersistentFlags().Lookup("exporters-timeout")) 75 | 76 | cmd.PersistentFlags().BoolP( 77 | "verbose", "v", false, 78 | "Include debug messages to output (ENV:MERGER_VERBOSE)") 79 | app.viper.BindPFlag("verbose", cmd.PersistentFlags().Lookup("verbose")) 80 | 81 | cmd.PersistentFlags().StringSlice( 82 | "url", nil, 83 | "URL to scrape. Can be speficied multiple times. (ENV:MERGER_URLS,space-seperated)") 84 | app.viper.BindPFlag("urls", cmd.PersistentFlags().Lookup("url")) 85 | } 86 | 87 | func (app *App) run(cmd *cobra.Command, args []string) { 88 | http.Handle("/metrics", Handler{ 89 | Exporters: app.viper.GetStringSlice("urls"), 90 | ExportersHTTPTimeout: app.viper.GetInt("exporterstimeout"), 91 | }) 92 | 93 | port := app.viper.GetInt("port") 94 | log.Infof("starting HTTP server on port %d", port) 95 | err := http.ListenAndServe(fmt.Sprintf(":%d", port), nil) 96 | if err != nil { 97 | log.Fatal(err) 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /cmd/config.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | 7 | "github.com/pkg/errors" 8 | log "github.com/sirupsen/logrus" 9 | "gopkg.in/yaml.v2" 10 | ) 11 | 12 | type Config struct { 13 | Exporters []Exporter 14 | } 15 | 16 | type Exporter struct { 17 | URL string 18 | } 19 | 20 | func ReadConfig(path string) (*Config, error) { 21 | var err error 22 | 23 | raw, err := ioutil.ReadFile(path) 24 | if err != nil { 25 | return nil, errors.WithStack(err) 26 | } 27 | 28 | config := new(Config) 29 | err = yaml.Unmarshal(raw, config) 30 | if err != nil { 31 | return nil, errors.Wrapf(err, "failed to parse %s", path) 32 | } 33 | 34 | log.WithFields(log.Fields{ 35 | "content": fmt.Sprintf("%#v", config), 36 | "path": path, 37 | }).Debug("loaded config file") 38 | 39 | return config, nil 40 | } 41 | -------------------------------------------------------------------------------- /cmd/handler.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "io" 5 | "net/http" 6 | "sort" 7 | "sync" 8 | "time" 9 | 10 | prom "github.com/prometheus/client_model/go" 11 | "github.com/prometheus/common/expfmt" 12 | log "github.com/sirupsen/logrus" 13 | ) 14 | 15 | type Handler struct { 16 | Exporters []string 17 | ExportersHTTPTimeout int 18 | } 19 | 20 | func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 21 | log.WithFields(log.Fields{ 22 | "RequestURI": r.RequestURI, 23 | "UserAgent": r.UserAgent(), 24 | }).Debug("handling new request") 25 | h.Merge(w) 26 | } 27 | 28 | func (h Handler) Merge(w io.Writer) { 29 | mfs := map[string]*prom.MetricFamily{} 30 | 31 | responses := make([]map[string]*prom.MetricFamily, 1024) 32 | responsesMu := sync.Mutex{} 33 | httpClientTimeout := time.Second * time.Duration(h.ExportersHTTPTimeout) 34 | 35 | wg := sync.WaitGroup{} 36 | for _, url := range h.Exporters { 37 | wg.Add(1) 38 | go func(u string) { 39 | defer wg.Done() 40 | log.WithField("url", u).Debug("getting remote metrics") 41 | httpClient := http.Client{Timeout: httpClientTimeout} 42 | resp, err := httpClient.Get(u) 43 | if err != nil { 44 | log.WithField("url", u).Errorf("HTTP connection failed: %v", err) 45 | return 46 | } 47 | defer resp.Body.Close() 48 | 49 | tp := new(expfmt.TextParser) 50 | part, err := tp.TextToMetricFamilies(resp.Body) 51 | if err != nil { 52 | log.WithField("url", u).Errorf("Parse response body to metrics: %v", err) 53 | return 54 | } 55 | responsesMu.Lock() 56 | responses = append(responses, part) 57 | responsesMu.Unlock() 58 | }(url) 59 | } 60 | wg.Wait() 61 | 62 | for _, part := range responses { 63 | for n, mf := range part { 64 | mfo, ok := mfs[n] 65 | if ok { 66 | mfo.Metric = append(mfo.Metric, mf.Metric...) 67 | } else { 68 | mfs[n] = mf 69 | } 70 | 71 | } 72 | } 73 | 74 | names := []string{} 75 | for n := range mfs { 76 | names = append(names, n) 77 | } 78 | sort.Strings(names) 79 | 80 | enc := expfmt.NewEncoder(w, expfmt.FmtText) 81 | for _, n := range names { 82 | err := enc.Encode(mfs[n]) 83 | if err != nil { 84 | log.Error(err) 85 | return 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /cmd/handler_test.go: -------------------------------------------------------------------------------- 1 | package cmd_test 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "net/http/httptest" 7 | "reflect" 8 | "sort" 9 | "testing" 10 | 11 | "github.com/prometheus/common/expfmt" 12 | "github.com/rebuy-de/exporter-merger/cmd" 13 | log "github.com/sirupsen/logrus" 14 | ) 15 | 16 | func Equal(a, b []float64) bool { 17 | if len(a) != len(b) { 18 | return false 19 | } 20 | for i, v := range a { 21 | if v != b[i] { 22 | return false 23 | } 24 | } 25 | return true 26 | } 27 | 28 | func testExporter(t testing.TB, content string) (string, func()) { 29 | t.Helper() 30 | 31 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 32 | fmt.Fprintln(w, content) 33 | })) 34 | 35 | return ts.URL, ts.Close 36 | } 37 | 38 | func TestHandler(t *testing.T) { 39 | log.SetLevel(log.DebugLevel) 40 | 41 | te1, deferrer := testExporter(t, 42 | "foo{} 1\nconflict 2\nshared{meh=\"a\"} 3") 43 | defer deferrer() 44 | 45 | te2, deferrer := testExporter(t, 46 | "bar{} 4\nconflict 5\nshared{meh=\"b\"} 6") 47 | defer deferrer() 48 | 49 | exporters := []string{ 50 | te1, 51 | te2, 52 | } 53 | 54 | server := httptest.NewServer(cmd.Handler{ 55 | Exporters: exporters, 56 | }) 57 | defer server.Close() 58 | 59 | resp, err := http.Get(server.URL) 60 | if err != nil { 61 | t.Fatal(err) 62 | } 63 | if resp.StatusCode != 200 { 64 | t.Fatalf("Received non-200 response: %d\n", resp.StatusCode) 65 | } 66 | 67 | // want := `# TYPE bar untyped 68 | // bar 4 69 | // # TYPE conflict untyped 70 | // conflict 2 71 | // conflict 5 72 | // # TYPE foo untyped 73 | // foo 1 74 | // # TYPE shared untyped 75 | // shared{meh="a"} 3 76 | // shared{meh="b"} 6 77 | // ` 78 | // have, err := ioutil.ReadAll(resp.Body) 79 | // if err != nil { 80 | // t.Fatal(err) 81 | // } 82 | 83 | eFmt := new(expfmt.TextParser) 84 | part, err := eFmt.TextToMetricFamilies(resp.Body) 85 | 86 | fooWanted := 1.0 87 | var foo float64 88 | 89 | barWanted := 4.0 90 | var bar float64 91 | 92 | var conflictWanted sort.Float64Slice = []float64{2.0, 5.0} 93 | var conflict sort.Float64Slice = make([]float64, 0) 94 | 95 | sharedWanted := map[string]float64{"a": 3.0, "b": 6.0} 96 | shared := make(map[string]float64) 97 | 98 | for n, mf := range part { 99 | if n == "bar" { 100 | bar = mf.GetMetric()[0].GetUntyped().GetValue() 101 | } 102 | 103 | if n == "foo" { 104 | foo = mf.GetMetric()[0].GetUntyped().GetValue() 105 | } 106 | 107 | if n == "conflict" { 108 | for _, metric := range mf.GetMetric() { 109 | conflict = append(conflict, metric.GetUntyped().GetValue()) 110 | } 111 | } 112 | 113 | if n == "shared" { 114 | for _, metric := range mf.GetMetric() { 115 | label := metric.GetLabel()[0].GetValue() 116 | value := metric.GetUntyped().GetValue() 117 | shared[label] = value 118 | } 119 | } 120 | } 121 | 122 | if bar != barWanted { 123 | t.Errorf("bar is %f but wanted %f", bar, barWanted) 124 | } 125 | 126 | if foo != 1.0 { 127 | t.Errorf("foo is %f but wanted %f", foo, fooWanted) 128 | } 129 | 130 | conflictWanted.Sort() 131 | conflict.Sort() 132 | 133 | if !Equal(conflict, conflictWanted) { 134 | t.Errorf("conflict is %v but wanted %v", conflict, conflictWanted) 135 | } 136 | 137 | if !reflect.DeepEqual(shared, sharedWanted) { 138 | t.Errorf("shared is %v but wanted %v", shared, sharedWanted) 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /cmd/version.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/spf13/cobra" 7 | ) 8 | 9 | var ( 10 | BuildVersion = "unknown" 11 | BuildDate = "unknown" 12 | BuildHash = "unknown" 13 | BuildEnvironment = "unknown" 14 | ) 15 | 16 | func NewVersionCommand() *cobra.Command { 17 | cmd := &cobra.Command{ 18 | Use: "version", 19 | Short: "shows version of this application", 20 | Run: func(cmd *cobra.Command, args []string) { 21 | fmt.Printf("version: %s\n", BuildVersion) 22 | fmt.Printf("build date: %s\n", BuildDate) 23 | fmt.Printf("scm hash: %s\n", BuildHash) 24 | fmt.Printf("environment: %s\n", BuildEnvironment) 25 | }, 26 | } 27 | 28 | return cmd 29 | } 30 | -------------------------------------------------------------------------------- /golang.mk: -------------------------------------------------------------------------------- 1 | # Source: https://github.com/rebuy-de/golang-template 2 | # Version: 1.3.1 3 | # Dependencies: 4 | # * Glide 5 | # * gocov (https://github.com/axw/gocov) 6 | # * gocov-html (https://github.com/matm/gocov-html) 7 | 8 | NAME=$(notdir $(PACKAGE)) 9 | 10 | BUILD_VERSION=$(shell git describe --always --dirty --tags | tr '-' '.' ) 11 | BUILD_DATE=$(shell date) 12 | BUILD_HASH=$(shell git rev-parse HEAD) 13 | BUILD_MACHINE=$(shell echo $$HOSTNAME) 14 | BUILD_USER=$(shell whoami) 15 | 16 | BUILD_FLAGS=-ldflags "\ 17 | -X '$(PACKAGE)/cmd.BuildVersion=$(BUILD_VERSION)' \ 18 | -X '$(PACKAGE)/cmd.BuildDate=$(BUILD_DATE)' \ 19 | -X '$(PACKAGE)/cmd.BuildHash=$(BUILD_HASH)' \ 20 | -X '$(PACKAGE)/cmd.BuildEnvironment=$(BUILD_USER)@$(BUILD_MACHINE)' \ 21 | " 22 | 23 | GOFILES=$(shell find . -type f -name '*.go' -not -path "./vendor/*") 24 | GOPKGS=$(shell go list ./...) 25 | 26 | default: build 27 | 28 | Gopkg.lock: Gopkg.toml 29 | dep ensure 30 | touch Gopkg.lock 31 | 32 | vendor: Gopkg.lock Gopkg.toml 33 | dep ensure 34 | touch vendor 35 | 36 | format: 37 | gofmt -s -w $(GOFILES) 38 | 39 | vet: 40 | go vet $(GOPKGS) 41 | 42 | lint: 43 | $(foreach pkg,$(GOPKGS),golint $(pkg);) 44 | 45 | test_gopath: 46 | test $$(go list) = "$(PACKAGE)" 47 | 48 | test_packages: vendor 49 | go test $(GOPKGS) 50 | 51 | test_format: 52 | gofmt -l $(GOFILES) 53 | 54 | test: test_gopath test_format vet lint test_packages 55 | 56 | cov: 57 | gocov test -v $(GOPKGS) \ 58 | | gocov-html > coverage.html 59 | 60 | build: vendor 61 | go build \ 62 | $(BUILD_FLAGS) \ 63 | -o $(NAME)-$(BUILD_VERSION)-$(shell go env GOOS)-$(shell go env GOARCH) 64 | ln -sf $(NAME)-$(BUILD_VERSION)-$(shell go env GOOS)-$(shell go env GOARCH) $(NAME) 65 | 66 | xc: 67 | GOOS=linux GOARCH=amd64 make build 68 | GOOS=darwin GOARCH=amd64 make build 69 | 70 | install: test 71 | go install \ 72 | $(BUILD_FLAGS) 73 | 74 | clean: 75 | rm -f $(NAME)* 76 | 77 | .PHONY: build install test 78 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/rebuy-de/exporter-merger/cmd" 5 | log "github.com/sirupsen/logrus" 6 | ) 7 | 8 | func main() { 9 | if err := cmd.NewRootCommand().Execute(); err != nil { 10 | log.Fatal(err) 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /merger.yaml: -------------------------------------------------------------------------------- 1 | exporters: 2 | - url: http://localhost:9100/metrics 3 | - url: http://localhost:9101/metrics 4 | --------------------------------------------------------------------------------