├── .gitignore
├── Makefile
├── README.md
├── app.go
├── app_test.go
└── docs
    └── images
        └── grafana_setup.png


/.gitignore:
--------------------------------------------------------------------------------
1 | app
2 | 


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
 1 | # Go parameters
 2 | GOCMD=go
 3 | BINARY_NAME=app
 4 | GOBUILD=$(GOCMD) build
 5 | GOCLEAN=$(GOCMD) clean
 6 | GOFMT=$(GOCMD) fmt
 7 | GOGET=$(GOCMD) get
 8 | GOLIST=$(GOCMD) list
 9 | GOTEST=$(GOCMD) test
10 | GOVET=$(GOCMD) vet
11 | BINARY_UNIX=$(BINARY_NAME)_unix
12 | CWD=`pwd`
13 | 
14 | all: lint test build
15 | 
16 | build:
17 | 	$(GOBUILD) -o $(BINARY_NAME) -v
18 | 
19 | test:
20 | 	go test $(VERBOSE) -race `go list ./... | grep -v /vendor/`
21 | 
22 | lint:
23 | 	FILES=`go list ./... | grep -v /vendor/`
24 | 	$(GOVET) $(FILES)
25 | 	$(GOFMT) $(FILES)
26 | 
27 | clean:
28 | 	$(GOCLEAN)
29 | 	rm -f $(BINARY_NAME)
30 | 	rm -f $(BINARY_UNIX)
31 | 
32 | run:
33 | 	$(GOBUILD) -o $(BINARY_NAME) -v ./...
34 | 	./$(BINARY_NAME)
35 | 
36 | # Cross compilation
37 | build-linux:
38 | 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 $(GOBUILD) -o $(BINARY_UNIX) -v
39 | docker-build:
40 | 	docker build -t promproxy .
41 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # This project is not being maintained
 2 | 
 3 | It works, but I've found [Promxy](https://github.com/jacksontj/promxy) which handles more edge cases than this, so I've moved to that. You should, too!
 4 | 
 5 | # PromProxy
 6 | 
 7 | A daemon that receives Prometheus API calls from a system such as Grafana, sends that request to multiple Prometheus instances, and aggregates the responses.
 8 | 
 9 | Systems like Grafana don't work well when your data is split up across multiple non overlapping Prometheus servers. This daemon attempts to trick the requester into thinking that there is just one server, but the requests to go many.
10 | 
11 | # Getting started
12 | 
13 | ## Building
14 | 
15 | Before starting, you should have [installed go](https://golang.org/doc/install) and ensure that your [`$GOPATH`](https://github.com/golang/go/wiki/SettingGOPATH) environment variable is also set.
16 | 
17 | ### Get the source code
18 | 
19 | ```
20 | $ go get https://github.com/swalberg/promproxy
21 | ```
22 | 
23 | ### Build the app
24 | 
25 | ```
26 | $ cd $GOPATH/src/github.com/swalberg/promproxy
27 | $ make
28 | ```
29 | 
30 | Upon success, you will have the binary named `app` in your current directory
31 | 
32 | ## Running
33 | 
34 | ### Launching
35 | 
36 | At this time, there are limited configuration options, so running the app is simple.
37 | 
38 | Launch it with an array of Prometheus servers as arguments
39 | 
40 | ```
41 | $ ./app http://192.168.0.20:9090 http://192.168.0.30:9090 http://192.168.0.40:9090
42 | ```
43 | 
44 | ### Testing
45 | 
46 | Once running, the app will be listening at `http://0.0.0.0:6379`
47 | 
48 | Perform a test query against the `label` endpoint
49 | 
50 | ```
51 | $ curl http://localhost:6789/api/v1/label/job/values
52 | {"status":"success","data":["prometheus"]}
53 | ```
54 | ### Grafana
55 | 
56 | After promproxy is running, setup (or change) your `Prometheus` data source in Grafana to point to promproxy instead of a prometheus server.
57 | 
58 | <img src="docs/images/grafana_setup.png" width="400">
59 | 
60 | ## API Limitations
61 | 
62 | Currently, only the following endpoints of the Prometheus API are supported:
63 | 
64 | - label
65 | - series
66 | - query_range
67 | 
68 | These endpoints should provide sufficient functionality for building dashboards in [Grafana](https://grafana.com/)
69 | 


--------------------------------------------------------------------------------
/app.go:
--------------------------------------------------------------------------------
  1 | package main
  2 | 
  3 | import (
  4 | 	"encoding/json"
  5 | 	"fmt"
  6 | 	"io/ioutil"
  7 | 	"log"
  8 | 	"net/http"
  9 | 	"os"
 10 | 	"sort"
 11 | 	"strings"
 12 | 	"time"
 13 | 
 14 | 	"github.com/prometheus/common/model"
 15 | )
 16 | 
 17 | type ErrorType string
 18 | 
 19 | type ApiResponse struct {
 20 | 	Status    string          `json:"status"`
 21 | 	Data      json.RawMessage `json:"data"`
 22 | 	ErrorType ErrorType       `json:"errorType,omitempty"`
 23 | 	Error     string          `json:"error,omitempty"`
 24 | }
 25 | 
 26 | type QueryResult struct {
 27 | 	Type   model.ValueType `json:"resultType"`
 28 | 	Result json.RawMessage `json:"result"`
 29 | 
 30 | 	// The decoded value.
 31 | 	//	v model.Value
 32 | }
 33 | 
 34 | type LabelResult []*string
 35 | 
 36 | //Define a new structure that represents out API response (response status and body)
 37 | type HTTPResponse struct {
 38 | 	status string
 39 | 	//matrix model.Matrix
 40 | 	result []byte
 41 | }
 42 | 
 43 | func (r ApiResponse) Successful() bool {
 44 | 	return string(r.Status) == "success"
 45 | }
 46 | 
 47 | func main() {
 48 | 	h := http.HandlerFunc(proxyHandler)
 49 | 	log.Fatal(http.ListenAndServe(":6789", h))
 50 | }
 51 | 
 52 | func proxyHandler(w http.ResponseWriter, r *http.Request) {
 53 | 	log.Println("Received:", r.URL)
 54 | 	urlParts := strings.Split(r.RequestURI, `/`)
 55 | 	api := strings.Split(urlParts[3], `?`)[0]
 56 | 
 57 | 	var ch chan HTTPResponse = make(chan HTTPResponse)
 58 | 	servers := os.Args[1:]
 59 | 	for _, server := range servers {
 60 | 		go DoHTTPGet(fmt.Sprintf("%s%s", server, r.URL), ch)
 61 | 	}
 62 | 
 63 | 	remote := make([]ApiResponse, 0)
 64 | 	for range servers {
 65 | 		response := <-ch
 66 | 		if strings.HasPrefix(response.status, `200`) {
 67 | 			remote = append(remote, ParseResponse(response.result))
 68 | 		} else {
 69 | 			log.Println("Ignoring response with status", response.status)
 70 | 		}
 71 | 	}
 72 | 	merged := Merge(api, remote)
 73 | 	asJson, err := json.Marshal(merged)
 74 | 	if err != nil {
 75 | 		log.Println("error marshalling back", err)
 76 | 	}
 77 | 	fmt.Fprintf(w, "%s", asJson)
 78 | }
 79 | 
 80 | func DoHTTPGet(url string, ch chan<- HTTPResponse) {
 81 | 	timeout := time.Duration(5 * time.Second)
 82 | 	client := http.Client{
 83 | 		Timeout: timeout,
 84 | 	}
 85 | 	log.Println("Getting", url)
 86 | 	httpResponse, err := client.Get(url)
 87 | 	if err != nil {
 88 | 		log.Println("Error in response", err)
 89 | 		ch <- HTTPResponse{`failed`, []byte(err.Error())}
 90 | 	} else {
 91 | 		defer httpResponse.Body.Close()
 92 | 		httpBody, _ := ioutil.ReadAll(httpResponse.Body)
 93 | 		//Send an HTTPResponse back to the channel
 94 | 		ch <- HTTPResponse{httpResponse.Status, httpBody}
 95 | 	}
 96 | }
 97 | 
 98 | func ParseResponse(body []byte) ApiResponse {
 99 | 	var ar ApiResponse
100 | 	err := json.Unmarshal(body, &ar)
101 | 
102 | 	if err != nil {
103 | 		log.Println("Error unmarshalling JSON api response", err, body)
104 | 		return ApiResponse{Status: "error"}
105 | 	}
106 | 
107 | 	return ar
108 | }
109 | 
110 | func Merge(api string, responses []ApiResponse) ApiResponse {
111 | 	var qr QueryResult
112 | 	var ar ApiResponse
113 | 
114 | 	if len(responses) == 0 {
115 | 		log.Println("No responses received")
116 | 		return ApiResponse{Status: "failure"}
117 | 	} else {
118 | 		ar = responses[0]
119 | 	}
120 | 
121 | 	switch api {
122 | 	case `label`:
123 | 		return MergeArrays(responses)
124 | 	case `series`:
125 | 		return MergeSeries(responses)
126 | 	case `query_range`:
127 | 		err := json.Unmarshal(ar.Data, &qr)
128 | 		if err != nil {
129 | 			log.Println("Error unmarshalling JSON query result", err, string(ar.Data))
130 | 			log.Println("Full response:", string(ar.Data))
131 | 			return ApiResponse{}
132 | 		}
133 | 
134 | 		switch qr.Type {
135 | 		case model.ValMatrix:
136 | 			return MergeMatrices(responses)
137 | 		default:
138 | 			log.Println("Did not recognize the response type of", qr.Type)
139 | 
140 | 		}
141 | 	}
142 | 
143 | 	log.Println("Full response:", string(ar.Data))
144 | 	return ApiResponse{}
145 | }
146 | 
147 | func MergeSeries(responses []ApiResponse) ApiResponse {
148 | 	merged := make([]map[string]string, 0)
149 | 
150 | 	for _, ar := range responses {
151 | 		var result []map[string]string
152 | 		err := json.Unmarshal(ar.Data, &result)
153 | 		if err != nil {
154 | 			log.Println("Unmarshal problem got", err)
155 | 		}
156 | 		for _, r := range result {
157 | 			merged = append(merged, r)
158 | 		}
159 | 	}
160 | 	log.Println("result", merged)
161 | 
162 | 	m, err := json.Marshal(merged)
163 | 	if err != nil {
164 | 		log.Println("error marshalling series back", err)
165 | 	}
166 | 
167 | 	return ApiResponse{Status: "success", Data: m}
168 | }
169 | 
170 | func MergeArrays(responses []ApiResponse) ApiResponse {
171 | 	set := make(map[string]struct{})
172 | 
173 | 	for _, ar := range responses {
174 | 		var labels LabelResult
175 | 		err := json.Unmarshal(ar.Data, &labels)
176 | 		if err != nil {
177 | 			log.Println("Error unmarshalling labels", err)
178 | 		}
179 | 		for _, label := range labels {
180 | 			set[*label] = struct{}{}
181 | 		}
182 | 	}
183 | 	keys := make([]string, 0, len(set))
184 | 	for k := range set {
185 | 		keys = append(keys, k)
186 | 	}
187 | 	sort.Strings(keys)
188 | 
189 | 	m, err := json.Marshal(keys)
190 | 	if err != nil {
191 | 		log.Println("error marshalling labels back", err)
192 | 	}
193 | 
194 | 	return ApiResponse{Status: "success", Data: m}
195 | 
196 | }
197 | 
198 | func MergeMatrices(responses []ApiResponse) ApiResponse {
199 | 	samples := make([]*model.SampleStream, 0)
200 | 	for _, r := range responses {
201 | 		matrix := ExtractMatrix(r)
202 | 		for _, s := range matrix {
203 | 			samples = append(samples, s)
204 | 		}
205 | 	}
206 | 
207 | 	mj, _ := json.Marshal(samples)
208 | 
209 | 	qr := QueryResult{model.ValMatrix, mj}
210 | 	qrj, _ := json.Marshal(qr)
211 | 
212 | 	r := ApiResponse{Status: "success", Data: qrj}
213 | 	return r
214 | 
215 | }
216 | 
217 | func ExtractMatrix(ar ApiResponse) model.Matrix {
218 | 	var qr QueryResult
219 | 	err := json.Unmarshal(ar.Data, &qr)
220 | 
221 | 	if err != nil {
222 | 		log.Println("Error unmarshalling JSON query result", err, string(ar.Data))
223 | 	}
224 | 
225 | 	var m model.Matrix
226 | 	err = json.Unmarshal(qr.Result, &m)
227 | 	if err != nil {
228 | 		log.Println("Error unmarshalling a matrix", err, string(qr.Result))
229 | 	}
230 | 	return m
231 | }
232 | 


--------------------------------------------------------------------------------
/app_test.go:
--------------------------------------------------------------------------------
  1 | package main
  2 | 
  3 | import (
  4 | 	"encoding/json"
  5 | 	"strings"
  6 | 	"testing"
  7 | )
  8 | 
  9 | var rError = []byte(`{"status":"error","data":{}}`)
 10 | var matrix1 = []byte(`{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"__name__":"prometheus_tsdb_blocks_loaded","group":"prometheus","instance":"la3stgprom01","job":"prometheus"},"values":[[1532712425,"24"],[1532712455,"24"],[1532712485,"24"],[1532712515,"24"],[1532712545,"24"],[1532712575,"24"],[1532712605,"24"],[1532712635,"24"],[1532712665,"24"],[1532712695,"24"],[1532712725,"24"],[1532712755,"24"],[1532712785,"24"],[1532712815,"24"],[1532712845,"24"],[1532712875,"24"],[1532712905,"24"],[1532712935,"24"],[1532712965,"24"],[1532712995,"24"],[1532713025,"24"],[1532713055,"24"],[1532713085,"24"],[1532713115,"24"],[1532713145,"24"],[1532713175,"24"],[1532713205,"24"],[1532713235,"24"],[1532713265,"24"],[1532713295,"24"],[1532713325,"24"],[1532713355,"24"],[1532713385,"24"],[1532713415,"24"],[1532713445,"24"],[1532713475,"24"],[1532713505,"24"],[1532713535,"24"],[1532713565,"24"],[1532713595,"24"],[1532713625,"24"],[1532713655,"24"],[1532713685,"24"],[1532713715,"24"],[1532713745,"24"],[1532713775,"24"]]}]}}`)
 11 | var matrix2 = []byte(`{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"__name__":"prometheus_tsdb_blocks_loaded","group":"prometheus","instance":"lv1stgprom01","job":"prometheus"},"values":[[1532712425,"24"],[1532712455,"24"],[1532712485,"24"],[1532712515,"24"],[1532712545,"24"],[1532712575,"24"],[1532712605,"24"],[1532712635,"24"],[1532712665,"24"],[1532712695,"24"],[1532712725,"24"],[1532712755,"24"],[1532712785,"24"],[1532712815,"24"],[1532712845,"24"],[1532712875,"24"],[1532712905,"24"],[1532712935,"24"],[1532712965,"24"],[1532712995,"24"],[1532713025,"24"],[1532713055,"24"],[1532713085,"24"],[1532713115,"24"],[1532713145,"24"],[1532713175,"24"],[1532713205,"24"],[1532713235,"24"],[1532713265,"24"],[1532713295,"24"],[1532713325,"24"],[1532713355,"24"],[1532713385,"24"],[1532713415,"24"],[1532713445,"24"],[1532713475,"24"],[1532713505,"24"],[1532713535,"24"],[1532713565,"24"],[1532713595,"24"],[1532713625,"24"],[1532713655,"24"],[1532713685,"24"],[1532713715,"24"],[1532713745,"24"],[1532713775,"24"]]}]}}`)
 12 | var merged = `{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"__name__":"prometheus_tsdb_blocks_loaded","group":"prometheus","instance":"la3stgprom01","job":"prometheus"},"values":[[1532712425,"24"],[1532712455,"24"],[1532712485,"24"],[1532712515,"24"],[1532712545,"24"],[1532712575,"24"],[1532712605,"24"],[1532712635,"24"],[1532712665,"24"],[1532712695,"24"],[1532712725,"24"],[1532712755,"24"],[1532712785,"24"],[1532712815,"24"],[1532712845,"24"],[1532712875,"24"],[1532712905,"24"],[1532712935,"24"],[1532712965,"24"],[1532712995,"24"],[1532713025,"24"],[1532713055,"24"],[1532713085,"24"],[1532713115,"24"],[1532713145,"24"],[1532713175,"24"],[1532713205,"24"],[1532713235,"24"],[1532713265,"24"],[1532713295,"24"],[1532713325,"24"],[1532713355,"24"],[1532713385,"24"],[1532713415,"24"],[1532713445,"24"],[1532713475,"24"],[1532713505,"24"],[1532713535,"24"],[1532713565,"24"],[1532713595,"24"],[1532713625,"24"],[1532713655,"24"],[1532713685,"24"],[1532713715,"24"],[1532713745,"24"],[1532713775,"24"]]},{"metric":{"__name__":"prometheus_tsdb_blocks_loaded","group":"prometheus","instance":"lv1stgprom01","job":"prometheus"},"values":[[1532712425,"24"],[1532712455,"24"],[1532712485,"24"],[1532712515,"24"],[1532712545,"24"],[1532712575,"24"],[1532712605,"24"],[1532712635,"24"],[1532712665,"24"],[1532712695,"24"],[1532712725,"24"],[1532712755,"24"],[1532712785,"24"],[1532712815,"24"],[1532712845,"24"],[1532712875,"24"],[1532712905,"24"],[1532712935,"24"],[1532712965,"24"],[1532712995,"24"],[1532713025,"24"],[1532713055,"24"],[1532713085,"24"],[1532713115,"24"],[1532713145,"24"],[1532713175,"24"],[1532713205,"24"],[1532713235,"24"],[1532713265,"24"],[1532713295,"24"],[1532713325,"24"],[1532713355,"24"],[1532713385,"24"],[1532713415,"24"],[1532713445,"24"],[1532713475,"24"],[1532713505,"24"],[1532713535,"24"],[1532713565,"24"],[1532713595,"24"],[1532713625,"24"],[1532713655,"24"],[1532713685,"24"],[1532713715,"24"],[1532713745,"24"],[1532713775,"24"]]}]}}`
 13 | var labelSet1 = []byte(`{"status":"success","data":["ALERTS","go_gc_duration_seconds","go_gc_duration_seconds_count"]}`)
 14 | var labelSet2 = []byte(`{"status":"success","data":["ALERTS","go_gc_duration_seconds","testing"]}`)
 15 | var labelMerged = `{"status":"success","data":["ALERTS","go_gc_duration_seconds","go_gc_duration_seconds_count","testing"]}`
 16 | 
 17 | var series1 = []byte(`{"status":"success","data":[{"__name__":"mysql_up","dc":"la3","environment":"stg","instance":"la3stgbicubedb01","job":"bicube_mysql-db"},{"__name__":"mysql_up","dc":"la3","environment":"stg","instance":"la3stgbicubedb02","job":"bicube_mysql-db"}]}`)
 18 | var series2 = []byte(`{"status":"success","data":[{"__name__":"mysql_up","dc":"lv1","environment":"stg","instance":"lv1stgbicubedb01","job":"bicube_mysql-db"},{"__name__":"mysql_up","dc":"lv1","environment":"stg","instance":"lv1stgbicubedb02","job":"bicube_mysql-db"}]}`)
 19 | 
 20 | func TestExtractMatrix(t *testing.T) {
 21 | 	response := ParseResponse(matrix1)
 22 | 
 23 | 	matrix := ExtractMatrix(response)
 24 | 	if len(matrix) != 1 {
 25 | 		t.Errorf("Expected to see 1 time series, got %d", len(matrix))
 26 | 	}
 27 | }
 28 | 
 29 | func TestParseResponseError(t *testing.T) {
 30 | 	response := ParseResponse(rError)
 31 | 	if response.Successful() {
 32 | 		t.Error("expected success to be false but it wasn`t")
 33 | 	}
 34 | }
 35 | 
 36 | func TestParseResponse(t *testing.T) {
 37 | 	response := ParseResponse(matrix1)
 38 | 	if !response.Successful() {
 39 | 		t.Error("expected success to be true but it wasn`t")
 40 | 	}
 41 | }
 42 | 
 43 | func TestMergeSeries(t *testing.T) {
 44 | 	p1 := ParseResponse(series1)
 45 | 	p2 := ParseResponse(series2)
 46 | 	var two = []ApiResponse{p1, p2}
 47 | 
 48 | 	m1 := Merge(`series`, two)
 49 | 	got, err := json.Marshal(m1)
 50 | 	if err != nil {
 51 | 		t.Errorf("Unexpected error marshalling response: %s", err)
 52 | 	}
 53 | 
 54 | 	i := strings.Count(string(got), `instance`)
 55 | 	if i != 4 {
 56 | 		t.Errorf("Expected to see 4 instances but got %d from %s", i, got)
 57 | 	}
 58 | }
 59 | 
 60 | func TestMergeArray(t *testing.T) {
 61 | 	p1 := ParseResponse(labelSet1)
 62 | 	p2 := ParseResponse(labelSet2)
 63 | 	var two = []ApiResponse{p1, p2}
 64 | 	m1 := Merge(`label`, two)
 65 | 	got, err := json.Marshal(m1)
 66 | 	if err != nil {
 67 | 		t.Errorf("Unexpected error unmarshalling response: %s", err)
 68 | 	}
 69 | 	if labelMerged != string(got) {
 70 | 		t.Errorf("Expected %s but got %s", labelMerged, got)
 71 | 	}
 72 | }
 73 | 
 74 | func TestMergeMatrix(t *testing.T) {
 75 | 	p1 := ParseResponse(matrix1)
 76 | 	p2 := ParseResponse(matrix2)
 77 | 	var two = []ApiResponse{p1, p2}
 78 | 	m1 := Merge(`query_range`, two)
 79 | 
 80 | 	matrix := ExtractMatrix(m1)
 81 | 	if m1.Status != "success" {
 82 | 		t.Errorf("Expected success status, got %s", m1.Status)
 83 | 	}
 84 | 
 85 | 	var qr QueryResult
 86 | 	err := json.Unmarshal(m1.Data, &qr)
 87 | 	if err != nil {
 88 | 		t.Errorf("Expected to unmarshall the QueryResult, got %s", err)
 89 | 	}
 90 | 
 91 | 	if len(matrix) != 2 {
 92 | 		t.Errorf("Expected to see 2 time series, got %d", len(matrix))
 93 | 	}
 94 | }
 95 | 
 96 | func BenchmarkMergeMatrix(b *testing.B) {
 97 | 	for i := 0; i < b.N; i++ {
 98 | 		var args = []ApiResponse{
 99 | 			ParseResponse(matrix1),
100 | 			ParseResponse(matrix2),
101 | 		}
102 | 		Merge(`query_range`, args)
103 | 	}
104 | }
105 | 


--------------------------------------------------------------------------------
/docs/images/grafana_setup.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/swalberg/promproxy/967c34ad936a4096266f6ec103fd20f77c785fcd/docs/images/grafana_setup.png


--------------------------------------------------------------------------------