├── .dockerignore
├── .github
    └── FUNDING.yml
├── .gitignore
├── Dockerfile
├── LICENSE
├── README.md
├── config.yaml
├── example
    ├── data-gen
    │   ├── go.mod
    │   ├── main.go
    │   └── sample.Dockerfile
    ├── docker-compose.yaml
    ├── grafana
    │   ├── dashboards
    │   │   ├── ds_prometheus.yaml
    │   │   └── example.json
    │   └── datasources
    │   │   └── prometheus.yaml
    └── prometheus.yml
├── go.mod
├── go.sum
├── images
    ├── gopher.png
    └── grafana.png
├── main.go
├── pkg
    ├── config
    │   ├── config.go
    │   └── config_test.go
    ├── query
    │   ├── count.go
    │   ├── mongo.go
    │   ├── mongo_test.go
    │   ├── sql.go
    │   └── sql_test.go
    └── scheduler
    │   ├── scheduler.go
    │   └── scheduler_test.go
└── test
    ├── invalid_config.yaml
    └── test_config.yaml


/.dockerignore:
--------------------------------------------------------------------------------
1 | test/
2 | example/


--------------------------------------------------------------------------------
/.github/FUNDING.yml:
--------------------------------------------------------------------------------
1 | # These are supported funding model platforms
2 | 
3 | custom: https://www.buymeacoffee.com/yolossn
4 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | data/*
2 | .env
3 | .vscode/*


--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM golang:1.13
 2 | 
 3 | LABEL maintainer="nssvlr@gmail.com"
 4 | 
 5 | WORKDIR /app
 6 | 
 7 | # COPY and download dependencies
 8 | COPY ./go.mod .
 9 | RUN go mod download
10 | 
11 | 
12 | COPY . .
13 | 
14 | RUN go build -o app
15 | 
16 | CMD ["./app"]
17 | 
18 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2020 Santhosh Nagaraj
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | <h1 align="center">query2metric</h1>
  2 | <p align="center">A tool to run db queries in defined frequency and expose the count as prometheus metrics.</p>
  3 | <p align="center">
  4 |     <img src="https://github.com/yolossn/query2metric/blob/master/images/gopher.png" height="200px"/>
  5 | </p>
  6 | 
  7 | ## Why ?
  8 | 
  9 | Product metrics play an important role in understanding product adoption and historic metrics helps answer many questions about a product (for eg: which day of the week do I get the most signups). One common thing is that most of these metrics are extracted by querying the databases. The tool takes queries and time frequency as configuration and runs the queries in the specified intervals and exposes the output as prometheus metrics.
 10 | 
 11 | ## Example
 12 | 
 13 | Create a config.yaml file.
 14 | 
 15 | > config.yaml
 16 | 
 17 | ```yml
 18 | connections:
 19 |   - name: mongodb1
 20 |     type: MONGO
 21 |     connectionStringFromEnv: MONGO_CONN
 22 |     metrics:
 23 |       - name: active_user_count
 24 |         helpString: users in the product
 25 |         database: test
 26 |         collection: test
 27 |         query: '{"is_active":true}'
 28 |         time: 10
 29 |       - name: total_user_count
 30 |         helpString: users in the product
 31 |         database: test
 32 |         collection: test
 33 |         query: ""
 34 |         time: 120
 35 |   - name: postgres1
 36 |     type: SQL
 37 |     connectionStringFromEnv: POSTGRES_CONN
 38 |     metrics:
 39 |       - name: template_count
 40 |         helpString: products in the db
 41 |         query: select * from templates
 42 |         time: 2
 43 |       - name: active_template_count
 44 |         helpString: products in the db
 45 |         query: error
 46 |         time: 4
 47 | ```
 48 | 
 49 | Along with the metrics defined, the success and failure count of queries are also exposed as prometheus metrics.
 50 | 
 51 | `query2metric_success_count` - No of successful queries coverted to metrics.
 52 | 
 53 | `query2metric_error_count` - No of errors when converting query to metric.
 54 | 
 55 | Note: Errors can occur due to invalid queries or connection issues to the db, one can use the logs to debug the issues.
 56 | 
 57 | ## How to use ?
 58 | 
 59 | At present the tool supports mongo and sql queries. Just create a config.yaml file and run the code.
 60 | 
 61 | ### Mongo
 62 | 
 63 | set `type` as `MONGO` and metrics as given in example with `query`,`time` (in seconds) etc.
 64 | 
 65 | ```yml
 66 | connections:
 67 | - name: mongodb1
 68 |     type: MONGO
 69 |     connectionStringFromEnv: MONGO_CONN
 70 |     metrics:
 71 |       - name: active_user_count
 72 |         helpString: users in the product
 73 |         database: test
 74 |         collection: test
 75 |         query: '{"is_active":true}'
 76 |         time: 10
 77 | ```
 78 | 
 79 | ### SQL
 80 | 
 81 | set `type` as `SQL` and metrics as give in example.
 82 | 
 83 | ```yml
 84 | connections:
 85 |   - name: postgres1
 86 |     type: SQL
 87 |     connectionStringFromEnv: POSTGRES_CONN
 88 |     metrics:
 89 |       - name: template_count
 90 |         helpString: products in the db
 91 |         query: select * from templates
 92 |         time: 2
 93 | ```
 94 | 
 95 | ## Run example using docker
 96 | 
 97 | You can run the example along with prometheus and grafana using docker.
 98 | 
 99 | docker-compose.yaml
100 | 
101 | > docker-compose up
102 | 
103 | metrics output: [localhost:8090/metrics](http://localhost:8090/metrics).
104 | 
105 | prometheus dashboard: [localhost:9090/graph](http://localhost:9090/graph).
106 | 
107 | grafana dashboard: [localhost:3000/d/qqTN2unMk/example?orgId=1](http://localhost:3000/d/qqTN2unMk/example?orgId=1).
108 | 
109 | Example Output:
110 | 
111 | <p align="center">
112 |   <img width="720" height="354" src="https://github.com/yolossn/query2metric/blob/master/images/grafana.png">
113 | </p>
114 | 
115 | ## Credits
116 | 
117 | - Logo credit [gopherize.me](gopherize.me)
118 | 


--------------------------------------------------------------------------------
/config.yaml:
--------------------------------------------------------------------------------
 1 | connections:
 2 |   - name: mongodb1
 3 |     type: MONGO
 4 |     connectionStringFromEnv: MONGO_CONN
 5 |     metrics:
 6 |       - name: active_user_count
 7 |         helpString: users in the product
 8 |         database: test
 9 |         collection: test
10 |         query: '{"is_active":true}'
11 |         time: 10
12 |       - name: total_user_count
13 |         helpString: users in the product
14 |         database: test
15 |         collection: test
16 |         query: ""
17 |         time: 120
18 |   - name: postgres1
19 |     type: SQL
20 |     connectionStringFromEnv: POSTGRES_CONN
21 |     metrics:
22 |       - name: template_count
23 |         helpString: products in the db
24 |         query: select * from templates
25 |         time: 2
26 |       - name: active_template_count
27 |         helpString: products in the db
28 |         query: error
29 |         time: 4
30 | 


--------------------------------------------------------------------------------
/example/data-gen/go.mod:
--------------------------------------------------------------------------------
 1 | module github.com/yolossn/query2metric
 2 | 
 3 | go 1.13
 4 | 
 5 | require (
 6 | 	github.com/go-sql-driver/mysql v1.5.0
 7 | 	github.com/lib/pq v1.7.0
 8 | 	github.com/mongodb/mongo-tools-common v4.0.1+incompatible
 9 | 	github.com/pkg/errors v0.8.1
10 | 	github.com/prometheus/client_golang v1.4.0
11 | 	github.com/prometheus/client_model v0.2.0
12 | 	github.com/sirupsen/logrus v1.4.2
13 | 	github.com/smartystreets/goconvey v1.6.4 // indirect
14 | 	github.com/stretchr/testify v1.4.0
15 | 	github.com/xo/dburl v0.0.0-20200124232849-e9ec94f52bc3
16 | 	go.mongodb.org/mongo-driver v1.3.5
17 | 	gopkg.in/yaml.v2 v2.3.0
18 | )
19 | 


--------------------------------------------------------------------------------
/example/data-gen/main.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"context"
 5 | 	"log"
 6 | 	"os"
 7 | 	"time"
 8 | 
 9 | 	_ "github.com/lib/pq"
10 | 	"github.com/xo/dburl"
11 | 	"go.mongodb.org/mongo-driver/mongo"
12 | 	"go.mongodb.org/mongo-driver/mongo/options"
13 | )
14 | 
15 | // This script is to add random data to the db
16 | // to demonstrate the output/
17 | 
18 | func main() {
19 | 	time.Sleep(time.Duration(10) * time.Second)
20 | 	ctx := context.Background()
21 | 	// postgres connection
22 | 	postgresconnectionEnv := "POSTGRES_CONN"
23 | 	postgresconnectionStr := os.Getenv(postgresconnectionEnv)
24 | 	db, err := dburl.Open(postgresconnectionStr)
25 | 	if err != nil {
26 | 		log.Fatal(err)
27 | 	}
28 | 
29 | 	// mongo connection
30 | 	mongoconnectionEnv := "MONGO_CONN"
31 | 	mongoconnectionString := os.Getenv(mongoconnectionEnv)
32 | 	mongoClient, err := mongo.NewClient(options.Client().ApplyURI(mongoconnectionString))
33 | 	if err != nil {
34 | 		log.Fatal(err)
35 | 	}
36 | 	err = mongoClient.Connect(ctx)
37 | 	if err != nil {
38 | 		log.Fatal(err)
39 | 	}
40 | 
41 | 	dropTableStmt := `DROP TABLE IF EXISTS templates`
42 | 	_, err = db.Exec(dropTableStmt)
43 | 	if err != nil {
44 | 		log.Fatal(err)
45 | 	}
46 | 	// create table in postgres
47 | 	createTableStmt := `CREATE TABLE templates(name TEXT, is_active BOOLEAN)`
48 | 	_, err = db.Exec(createTableStmt)
49 | 	if err != nil {
50 | 		log.Fatal(err)
51 | 	}
52 | 
53 | 	// Insert rows
54 | 	insertStmt := `INSERT INTO templates (name, is_active) VALUES ($1, $2)`
55 | 	_, err = db.Exec(insertStmt, "test1", true)
56 | 	if err != nil {
57 | 		log.Fatal(err)
58 | 	}
59 | 	// Insert documents in mongo
60 | 	coll := mongoClient.Database("test").Collection("test")
61 | 	_, err = coll.InsertOne(ctx, map[string]interface{}{"name": "test", "is_active": true})
62 | 	if err != nil {
63 | 		log.Fatal(err)
64 | 	}
65 | 	_, err = coll.InsertOne(ctx, map[string]interface{}{"name": "test1", "is_active": false})
66 | 	if err != nil {
67 | 		log.Fatal(err)
68 | 	}
69 | 
70 | 	time.Sleep(time.Duration(3) * time.Second)
71 | 	_, err = coll.InsertOne(ctx, map[string]interface{}{"name": "test2", "is_active": false})
72 | 	if err != nil {
73 | 		log.Fatal(err)
74 | 	}
75 | 	_, err = db.Exec(insertStmt, "test2", true)
76 | 	if err != nil {
77 | 		log.Fatal(err)
78 | 	}
79 | }
80 | 


--------------------------------------------------------------------------------
/example/data-gen/sample.Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM golang:1.13
 2 | 
 3 | LABEL maintainer="nssvlr@gmail.com"
 4 | 
 5 | WORKDIR /app
 6 | 
 7 | # COPY and download dependencies
 8 | COPY ./go.mod .
 9 | RUN go mod download
10 | 
11 | 
12 | COPY . .
13 | 
14 | RUN go build -o app
15 | 
16 | CMD ["./app"]
17 | 
18 | 


--------------------------------------------------------------------------------
/example/docker-compose.yaml:
--------------------------------------------------------------------------------
 1 | version: "3.8"
 2 | 
 3 | networks:
 4 |   example-network:
 5 | 
 6 | services:
 7 |   query2metric:
 8 |     build:
 9 |       context: ..
10 |     depends_on:
11 |       - mongo
12 |       - postgres
13 |       - sample_data
14 |     ports:
15 |       - 8090:8090
16 |     networks:
17 |       - example-network
18 |     environment:
19 |       MONGO_CONN: mongodb://mongo:27017/test?connect=direct
20 |       POSTGRES_CONN: postgresql://postuser:postpass@postgres/test?sslmode=disable
21 | 
22 |   sample_data:
23 |     build:
24 |       context: data-gen
25 |       dockerfile: sample.Dockerfile
26 |     depends_on:
27 |       - mongo
28 |       - postgres
29 |     networks:
30 |       - example-network
31 |     environment:
32 |       MONGO_CONN: mongodb://mongo:27017/test?connect=direct
33 |       POSTGRES_CONN: postgresql://postuser:postpass@postgres/test?sslmode=disable
34 | 
35 |   mongo:
36 |     image: mongo:4.2
37 |     networks:
38 |       - example-network
39 |     environment:
40 |       MONGO_INITDB_DATABASE: test
41 |     healthcheck:
42 |       test: echo 'db.runCommand("ping").ok' | mongo mongo:27017/test --quiet
43 |       interval: 10s
44 |       timeout: 10s
45 |       retries: 5
46 |       start_period: 40s
47 | 
48 |   postgres:
49 |     image: postgres
50 |     networks:
51 |       - example-network
52 |     environment:
53 |       POSTGRES_USER: postuser
54 |       POSTGRES_PASSWORD: postpass
55 |       POSTGRES_DB: test
56 |     healthcheck:
57 |       test: ["CMD-SHELL", "pg_isready -U postuser"]
58 |       interval: 10s
59 |       timeout: 5s
60 |       retries: 5
61 | 
62 |   prometheus:
63 |     image: prom/prometheus
64 |     volumes:
65 |       - ./prometheus.yml:/etc/prometheus/prometheus.yml
66 |     command:
67 |       - "--config.file=/etc/prometheus/prometheus.yml"
68 |     ports:
69 |       - 9090:9090
70 |     networks:
71 |       - example-network
72 |     restart: always
73 | 
74 |   grafana:
75 |     image: grafana/grafana
76 |     depends_on:
77 |       - prometheus
78 |     ports:
79 |       - 3000:3000
80 |     volumes:
81 |       - ./grafana/:/etc/grafana/provisioning/
82 |     networks:
83 |       - example-network
84 |     restart: always
85 | 


--------------------------------------------------------------------------------
/example/grafana/dashboards/ds_prometheus.yaml:
--------------------------------------------------------------------------------
 1 | # https://github.com/ruanbekker/docker-prometheus-grafana/blob/master/grafana/provisioning/dashboards/ds_prometheus.yml
 2 | apiVersion: 1
 3 | 
 4 | providers:
 5 |   - name: "Prometheus"
 6 |     orgId: 1
 7 |     folder: ""
 8 |     type: file
 9 |     disableDeletion: false
10 |     editable: true
11 |     options:
12 |       path: /etc/grafana/provisioning/dashboards
13 | 


--------------------------------------------------------------------------------
/example/grafana/dashboards/example.json:
--------------------------------------------------------------------------------
  1 | {
  2 |   "annotations": {
  3 |     "list": [
  4 |       {
  5 |         "builtIn": 1,
  6 |         "datasource": "-- Grafana --",
  7 |         "enable": true,
  8 |         "hide": true,
  9 |         "iconColor": "rgba(0, 211, 255, 1)",
 10 |         "name": "Annotations & Alerts",
 11 |         "type": "dashboard"
 12 |       }
 13 |     ]
 14 |   },
 15 |   "editable": true,
 16 |   "gnetId": null,
 17 |   "graphTooltip": 0,
 18 |   "id": 1,
 19 |   "links": [],
 20 |   "panels": [
 21 |     {
 22 |       "aliasColors": {},
 23 |       "bars": false,
 24 |       "dashLength": 10,
 25 |       "dashes": false,
 26 |       "datasource": null,
 27 |       "fill": 1,
 28 |       "fillGradient": 0,
 29 |       "gridPos": {
 30 |         "h": 9,
 31 |         "w": 12,
 32 |         "x": 0,
 33 |         "y": 0
 34 |       },
 35 |       "hiddenSeries": false,
 36 |       "id": 4,
 37 |       "legend": {
 38 |         "avg": false,
 39 |         "current": false,
 40 |         "max": false,
 41 |         "min": false,
 42 |         "show": true,
 43 |         "total": false,
 44 |         "values": false
 45 |       },
 46 |       "lines": true,
 47 |       "linewidth": 1,
 48 |       "nullPointMode": "null",
 49 |       "options": {
 50 |         "dataLinks": []
 51 |       },
 52 |       "percentage": false,
 53 |       "pointradius": 2,
 54 |       "points": false,
 55 |       "renderer": "flot",
 56 |       "seriesOverrides": [],
 57 |       "spaceLength": 10,
 58 |       "stack": false,
 59 |       "steppedLine": false,
 60 |       "targets": [
 61 |         {
 62 |           "expr": "mongodb1_total_user_count",
 63 |           "interval": "",
 64 |           "legendFormat": "",
 65 |           "refId": "A"
 66 |         }
 67 |       ],
 68 |       "thresholds": [],
 69 |       "timeFrom": null,
 70 |       "timeRegions": [],
 71 |       "timeShift": null,
 72 |       "title": "Total user count",
 73 |       "tooltip": {
 74 |         "shared": true,
 75 |         "sort": 0,
 76 |         "value_type": "individual"
 77 |       },
 78 |       "type": "graph",
 79 |       "xaxis": {
 80 |         "buckets": null,
 81 |         "mode": "time",
 82 |         "name": null,
 83 |         "show": true,
 84 |         "values": []
 85 |       },
 86 |       "yaxes": [
 87 |         {
 88 |           "format": "short",
 89 |           "label": null,
 90 |           "logBase": 1,
 91 |           "max": null,
 92 |           "min": null,
 93 |           "show": true
 94 |         },
 95 |         {
 96 |           "format": "short",
 97 |           "label": null,
 98 |           "logBase": 1,
 99 |           "max": null,
100 |           "min": null,
101 |           "show": true
102 |         }
103 |       ],
104 |       "yaxis": {
105 |         "align": false,
106 |         "alignLevel": null
107 |       }
108 |     },
109 |     {
110 |       "aliasColors": {},
111 |       "bars": false,
112 |       "dashLength": 10,
113 |       "dashes": false,
114 |       "datasource": null,
115 |       "description": "",
116 |       "fill": 1,
117 |       "fillGradient": 0,
118 |       "gridPos": {
119 |         "h": 9,
120 |         "w": 12,
121 |         "x": 12,
122 |         "y": 0
123 |       },
124 |       "hiddenSeries": false,
125 |       "id": 2,
126 |       "legend": {
127 |         "avg": false,
128 |         "current": false,
129 |         "max": false,
130 |         "min": false,
131 |         "show": true,
132 |         "total": false,
133 |         "values": false
134 |       },
135 |       "lines": true,
136 |       "linewidth": 1,
137 |       "nullPointMode": "null",
138 |       "options": {
139 |         "dataLinks": []
140 |       },
141 |       "percentage": false,
142 |       "pointradius": 2,
143 |       "points": false,
144 |       "renderer": "flot",
145 |       "seriesOverrides": [],
146 |       "spaceLength": 10,
147 |       "stack": false,
148 |       "steppedLine": false,
149 |       "targets": [
150 |         {
151 |           "expr": "postgres1_template_count",
152 |           "interval": "",
153 |           "legendFormat": "",
154 |           "refId": "A"
155 |         }
156 |       ],
157 |       "thresholds": [],
158 |       "timeFrom": null,
159 |       "timeRegions": [],
160 |       "timeShift": null,
161 |       "title": "Active Template Count",
162 |       "tooltip": {
163 |         "shared": true,
164 |         "sort": 0,
165 |         "value_type": "individual"
166 |       },
167 |       "transparent": true,
168 |       "type": "graph",
169 |       "xaxis": {
170 |         "buckets": null,
171 |         "mode": "time",
172 |         "name": null,
173 |         "show": true,
174 |         "values": []
175 |       },
176 |       "yaxes": [
177 |         {
178 |           "format": "short",
179 |           "label": null,
180 |           "logBase": 1,
181 |           "max": null,
182 |           "min": null,
183 |           "show": true
184 |         },
185 |         {
186 |           "format": "short",
187 |           "label": null,
188 |           "logBase": 1,
189 |           "max": null,
190 |           "min": null,
191 |           "show": true
192 |         }
193 |       ],
194 |       "yaxis": {
195 |         "align": false,
196 |         "alignLevel": null
197 |       }
198 |     },
199 |     {
200 |       "cacheTimeout": null,
201 |       "datasource": null,
202 |       "gridPos": {
203 |         "h": 8,
204 |         "w": 12,
205 |         "x": 0,
206 |         "y": 9
207 |       },
208 |       "id": 6,
209 |       "links": [],
210 |       "options": {
211 |         "colorMode": "value",
212 |         "fieldOptions": {
213 |           "calcs": ["mean"],
214 |           "defaults": {
215 |             "mappings": [
216 |               {
217 |                 "id": 0,
218 |                 "op": "=",
219 |                 "text": "N/A",
220 |                 "type": 1,
221 |                 "value": "null"
222 |               }
223 |             ],
224 |             "nullValueMode": "connected",
225 |             "thresholds": {
226 |               "mode": "absolute",
227 |               "steps": [
228 |                 {
229 |                   "color": "green",
230 |                   "value": null
231 |                 }
232 |               ]
233 |             },
234 |             "unit": "none"
235 |           },
236 |           "overrides": [],
237 |           "values": false
238 |         },
239 |         "graphMode": "area",
240 |         "justifyMode": "auto",
241 |         "orientation": "horizontal"
242 |       },
243 |       "pluginVersion": "6.7.3",
244 |       "targets": [
245 |         {
246 |           "expr": "query2metric_success_count",
247 |           "instant": true,
248 |           "interval": "",
249 |           "legendFormat": "",
250 |           "refId": "A"
251 |         }
252 |       ],
253 |       "timeFrom": null,
254 |       "timeShift": null,
255 |       "title": "successful queries to metrics",
256 |       "type": "stat"
257 |     },
258 |     {
259 |       "datasource": null,
260 |       "gridPos": {
261 |         "h": 8,
262 |         "w": 12,
263 |         "x": 12,
264 |         "y": 9
265 |       },
266 |       "id": 8,
267 |       "options": {
268 |         "colorMode": "value",
269 |         "fieldOptions": {
270 |           "calcs": ["mean"],
271 |           "defaults": {
272 |             "mappings": [],
273 |             "thresholds": {
274 |               "mode": "absolute",
275 |               "steps": [
276 |                 {
277 |                   "color": "green",
278 |                   "value": null
279 |                 },
280 |                 {
281 |                   "color": "red",
282 |                   "value": 80
283 |                 }
284 |               ]
285 |             }
286 |           },
287 |           "overrides": [],
288 |           "values": false
289 |         },
290 |         "graphMode": "area",
291 |         "justifyMode": "auto",
292 |         "orientation": "auto"
293 |       },
294 |       "pluginVersion": "6.7.3",
295 |       "targets": [
296 |         {
297 |           "expr": "query2metric_error_count",
298 |           "instant": true,
299 |           "interval": "",
300 |           "legendFormat": "",
301 |           "refId": "A"
302 |         }
303 |       ],
304 |       "timeFrom": null,
305 |       "timeShift": null,
306 |       "title": "Errors in query to metric",
307 |       "type": "stat"
308 |     }
309 |   ],
310 |   "schemaVersion": 22,
311 |   "style": "dark",
312 |   "tags": [],
313 |   "templating": {
314 |     "list": []
315 |   },
316 |   "time": {
317 |     "from": "now-30m",
318 |     "to": "now"
319 |   },
320 |   "timepicker": {},
321 |   "timezone": "",
322 |   "title": "Example",
323 |   "uid": "qqTN2unMk",
324 |   "variables": {
325 |     "list": []
326 |   },
327 |   "version": 4
328 | }
329 | 


--------------------------------------------------------------------------------
/example/grafana/datasources/prometheus.yaml:
--------------------------------------------------------------------------------
 1 | apiVersion: 1
 2 | datasources:
 3 |   - name: Prometheus
 4 |     type: prometheus
 5 |     access: proxy
 6 |     orgId: 1
 7 |     url: http://prometheus:9090
 8 |     password:
 9 |     user:
10 |     database:
11 |     basicAuth: false
12 |     basicAuthUser:
13 |     basicAuthPassword:
14 |     withCredentials:
15 |     isDefault: true
16 |     jsonData:
17 |       graphiteVersion: "1.1"
18 |       tlsAuth: false
19 |       tlsAuthWithCACert: false
20 |     secureJsonData:
21 |       tlsCACert: "..."
22 |       tlsClientCert: "..."
23 |       tlsClientKey: "..."
24 |     version: 1
25 |     editable: true
26 | 


--------------------------------------------------------------------------------
/example/prometheus.yml:
--------------------------------------------------------------------------------
1 | global:
2 |   scrape_interval: 15s
3 | 
4 | scrape_configs:
5 |   - job_name: app_metrics
6 |     static_configs:
7 |       - targets: ["query2metric:8090"]
8 | 


--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
 1 | module github.com/yolossn/query2metric
 2 | 
 3 | go 1.13
 4 | 
 5 | require (
 6 | 	github.com/go-sql-driver/mysql v1.5.0
 7 | 	github.com/lib/pq v1.7.0
 8 | 	github.com/mongodb/mongo-tools-common v4.0.1+incompatible
 9 | 	github.com/pkg/errors v0.8.1
10 | 	github.com/prometheus/client_golang v1.4.0
11 | 	github.com/prometheus/client_model v0.2.0
12 | 	github.com/sirupsen/logrus v1.4.2
13 | 	github.com/smartystreets/goconvey v1.6.4 // indirect
14 | 	github.com/stretchr/testify v1.4.0
15 | 	github.com/xo/dburl v0.0.0-20200124232849-e9ec94f52bc3
16 | 	go.mongodb.org/mongo-driver v1.3.5
17 | 	gopkg.in/yaml.v2 v2.3.0
18 | )
19 | 


--------------------------------------------------------------------------------
/go.sum:
--------------------------------------------------------------------------------
  1 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
  2 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
  3 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
  4 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
  5 | github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
  6 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
  7 | github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
  8 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
  9 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 10 | github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
 11 | github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 12 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 13 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 14 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 15 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 16 | github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 17 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 18 | github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 19 | github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
 20 | github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 21 | github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 22 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 23 | github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
 24 | github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY=
 25 | github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg=
 26 | github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
 27 | github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
 28 | github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs=
 29 | github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
 30 | github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
 31 | github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk=
 32 | github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28=
 33 | github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo=
 34 | github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk=
 35 | github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw=
 36 | github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360=
 37 | github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg=
 38 | github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE=
 39 | github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8=
 40 | github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
 41 | github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
 42 | github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
 43 | github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
 44 | github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
 45 | github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
 46 | github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
 47 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 48 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 49 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 50 | github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 51 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 52 | github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 53 | github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 54 | github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 55 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 56 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 57 | github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 58 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 59 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 60 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 61 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 62 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 63 | github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
 64 | github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 65 | github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 66 | github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 67 | github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 68 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 69 | github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
 70 | github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
 71 | github.com/klauspost/compress v1.9.5 h1:U+CaK85mrNNb4k8BNOfgJtJ/gr6kswUCFj6miSzVC6M=
 72 | github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 73 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 74 | github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
 75 | github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 76 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 77 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 78 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 79 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 80 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 81 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 82 | github.com/lib/pq v1.7.0 h1:h93mCPfUSkaul3Ka/VG8uZdmW1uMHDGxzu0NWHuJmHY=
 83 | github.com/lib/pq v1.7.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 84 | github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
 85 | github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
 86 | github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 87 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 88 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 89 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 90 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 91 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 92 | github.com/mongodb/mongo-tools-common v4.0.1+incompatible h1:xSo5bgWUzIpQ4pMdXXzX4JYQtoQIuq54cwI/gMb1sMs=
 93 | github.com/mongodb/mongo-tools-common v4.0.1+incompatible/go.mod h1:T0AGIWI5PvxSzqt5zsR5zlcHl52KGeAzhoAnSvvpEtU=
 94 | github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 95 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 96 | github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
 97 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 98 | github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 99 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
100 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
101 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
102 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
103 | github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
104 | github.com/prometheus/client_golang v1.4.0 h1:YVIb/fVcOTMSqtqZWSKnHpSLBxu8DKgxq8z6RuBZwqI=
105 | github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
106 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
107 | github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
108 | github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
109 | github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
110 | github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
111 | github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U=
112 | github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
113 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
114 | github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
115 | github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8=
116 | github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
117 | github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
118 | github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
119 | github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
120 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
121 | github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
122 | github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
123 | github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
124 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
125 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
126 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
127 | github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
128 | github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
129 | github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
130 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
131 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
132 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
133 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
134 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
135 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
136 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
137 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
138 | github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
139 | github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
140 | github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk=
141 | github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
142 | github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc h1:n+nNi93yXLkJvKwXNP9d55HC7lGK4H/SRcwB5IaUZLo=
143 | github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
144 | github.com/xo/dburl v0.0.0-20200124232849-e9ec94f52bc3 h1:NC3CI7do3KHtiuYhk1CdS9V2qS3jNa7Fs2Afcnnt+IE=
145 | github.com/xo/dburl v0.0.0-20200124232849-e9ec94f52bc3/go.mod h1:A47W3pdWONaZmXuLZgfKLAVgUY0qvfTRM5vVDKS40S4=
146 | go.mongodb.org/mongo-driver v1.3.5 h1:S0ZOruh4YGHjD7JoN7mIsTrNjnQbOjrmgrx6l6pZN7I=
147 | go.mongodb.org/mongo-driver v1.3.5/go.mod h1:Ual6Gkco7ZGQw8wE1t4tLnvBsf6yVSM60qW6TgOeJ5c=
148 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
149 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
150 | golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
151 | golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5 h1:8dUaAV7K4uHsF56JQWkprecIQKdPHtR9jCHF5nB8uzc=
152 | golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
153 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
154 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
155 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
156 | golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
157 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
158 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
159 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
160 | golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
161 | golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
162 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
163 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
164 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
165 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
166 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
167 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
168 | golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
169 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
170 | golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
171 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
172 | golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
173 | golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU=
174 | golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
175 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
176 | golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
177 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
178 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
179 | golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
180 | golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
181 | golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
182 | golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
183 | golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
184 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
185 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
186 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
187 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
188 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
189 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
190 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
191 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
192 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
193 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
194 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
195 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
196 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
197 | gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
198 | gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
199 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
200 | 


--------------------------------------------------------------------------------
/images/gopher.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/yolossn/query2metric/29118351f26affcddb7085625d7c8a98c0de536b/images/gopher.png


--------------------------------------------------------------------------------
/images/grafana.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/yolossn/query2metric/29118351f26affcddb7085625d7c8a98c0de536b/images/grafana.png


--------------------------------------------------------------------------------
/main.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"net/http"
 5 | 
 6 | 	"github.com/prometheus/client_golang/prometheus/promhttp"
 7 | 	log "github.com/sirupsen/logrus"
 8 | 	"github.com/yolossn/query2metric/pkg/config"
 9 | 	"github.com/yolossn/query2metric/pkg/scheduler"
10 | )
11 | 
12 | func main() {
13 | 	conf, err := config.FromFile("config.yaml")
14 | 	if err != nil {
15 | 		log.Fatal("Error reading config file,err:", err)
16 | 	}
17 | 
18 | 	configRunner := scheduler.FromConfig(*conf)
19 | 	err = configRunner.Start()
20 | 	if err != nil {
21 | 		log.Fatal("Error in runner,err:", err)
22 | 	}
23 | 
24 | 	http.Handle("/metrics", promhttp.Handler())
25 | 	log.Fatal(http.ListenAndServe(":8090", nil))
26 | 
27 | }
28 | 


--------------------------------------------------------------------------------
/pkg/config/config.go:
--------------------------------------------------------------------------------
 1 | package config
 2 | 
 3 | import (
 4 | 	"io/ioutil"
 5 | 
 6 | 	"github.com/pkg/errors"
 7 | 	"gopkg.in/yaml.v2"
 8 | )
 9 | 
10 | const MONGO = "MONGO"
11 | const SQL = "SQL"
12 | 
13 | type Metric struct {
14 | 	Name       string `yaml:"name"`
15 | 	HelpString string `yaml:"helpString"`
16 | 	Database   string `yaml:"database"`
17 | 	Collection string `yaml:"collection"`
18 | 	Query      string `yaml:"query"`
19 | 	Time       int64  `yaml:"time"`
20 | }
21 | 
22 | type Connection struct {
23 | 	Name             string   `yaml:"name"`
24 | 	Type             string   `yaml:"type"`
25 | 	Subtype          string   `yaml:"subtype"`
26 | 	ConnectionString string   `yaml:"connectionStringFromEnv"`
27 | 	Metrics          []Metric `yaml:"metrics"`
28 | }
29 | 
30 | type Config struct {
31 | 	Connections []Connection `yaml:"connections"`
32 | }
33 | 
34 | func FromFile(file string) (*Config, error) {
35 | 	content, err := ioutil.ReadFile(file)
36 | 	if err != nil {
37 | 		return nil, errors.Wrap(err, "Error reading file")
38 | 	}
39 | 
40 | 	conf := Config{}
41 | 	err = yaml.Unmarshal(content, &conf)
42 | 	if err != nil {
43 | 		return nil, errors.Wrap(err, "Error unmarshalling config")
44 | 	}
45 | 	return &conf, nil
46 | }
47 | 


--------------------------------------------------------------------------------
/pkg/config/config_test.go:
--------------------------------------------------------------------------------
 1 | package config_test
 2 | 
 3 | import (
 4 | 	"testing"
 5 | 
 6 | 	"github.com/stretchr/testify/require"
 7 | 	"github.com/yolossn/query2metric/pkg/config"
 8 | )
 9 | 
10 | func TestFromFile(t *testing.T) {
11 | 	t.Parallel()
12 | 
13 | 	// File exists
14 | 	conf, err := config.FromFile("../../test/test_config.yaml")
15 | 	require.NoError(t, err)
16 | 
17 | 	require.Equal(t, len(conf.Connections), 2)
18 | 
19 | 	conf, err = config.FromFile("../../test/not_exists.yaml")
20 | 	require.Error(t, err)
21 | 	require.Nil(t, conf)
22 | 
23 | 	conf, err = config.FromFile("../../test/invalid_config.yaml")
24 | 	require.Error(t, err)
25 | 	require.Nil(t, conf)
26 | }
27 | 


--------------------------------------------------------------------------------
/pkg/query/count.go:
--------------------------------------------------------------------------------
 1 | package query
 2 | 
 3 | import (
 4 | 	"github.com/pkg/errors"
 5 | 	"github.com/yolossn/query2metric/pkg/config"
 6 | )
 7 | 
 8 | type CountQuery interface {
 9 | 	Count(metric config.Metric) (int64, error)
10 | }
11 | 
12 | var ENV_NOT_SET = errors.New("connnectionString is empty")
13 | 


--------------------------------------------------------------------------------
/pkg/query/mongo.go:
--------------------------------------------------------------------------------
 1 | package query
 2 | 
 3 | import (
 4 | 	"context"
 5 | 	"encoding/json"
 6 | 	"os"
 7 | 
 8 | 	"github.com/mongodb/mongo-tools-common/bsonutil"
 9 | 	"github.com/pkg/errors"
10 | 	"github.com/yolossn/query2metric/pkg/config"
11 | 	"go.mongodb.org/mongo-driver/mongo"
12 | 	"go.mongodb.org/mongo-driver/mongo/options"
13 | )
14 | 
15 | type mongoQuery struct {
16 | 	connection string
17 | 	client     *mongo.Client
18 | }
19 | 
20 | func NewMongoConn(connnectionURL string) (CountQuery, error) {
21 | 	connnectionString := os.Getenv(connnectionURL)
22 | 	if connnectionString == "" {
23 | 		return nil, ENV_NOT_SET
24 | 	}
25 | 
26 | 	ctx := context.Background()
27 | 
28 | 	mongoClient, err := mongo.NewClient(options.Client().ApplyURI(connnectionString))
29 | 	if err != nil {
30 | 		return nil, errors.Wrap(err, "Connection error")
31 | 	}
32 | 
33 | 	err = mongoClient.Connect(ctx)
34 | 	if err != nil {
35 | 		return nil, errors.Wrap(err, "Connection error")
36 | 	}
37 | 
38 | 	return &mongoQuery{connnectionURL, mongoClient}, err
39 | }
40 | 
41 | func (m *mongoQuery) Count(metric config.Metric) (int64, error) {
42 | 
43 | 	ctx := context.Background()
44 | 	query := map[string]interface{}{}
45 | 	if metric.Query != "" {
46 | 		err := json.Unmarshal([]byte(metric.Query), &query)
47 | 		if err != nil {
48 | 			return 0, err
49 | 		}
50 | 	}
51 | 	bsonQuery, err := bsonutil.ConvertLegacyExtJSONValueToBSON(query)
52 | 	if err != nil {
53 | 		return 0, err
54 | 	}
55 | 	count, err := m.client.Database(metric.Database).Collection(metric.Collection).CountDocuments(ctx, bsonQuery)
56 | 	if err != nil {
57 | 		return 0, err
58 | 	}
59 | 	return count, nil
60 | }
61 | 


--------------------------------------------------------------------------------
/pkg/query/mongo_test.go:
--------------------------------------------------------------------------------
  1 | package query_test
  2 | 
  3 | import (
  4 | 	"context"
  5 | 	"os"
  6 | 	"testing"
  7 | 
  8 | 	"github.com/stretchr/testify/require"
  9 | 	"github.com/yolossn/query2metric/pkg/config"
 10 | 	"github.com/yolossn/query2metric/pkg/query"
 11 | 	"go.mongodb.org/mongo-driver/mongo"
 12 | 	"go.mongodb.org/mongo-driver/mongo/options"
 13 | )
 14 | 
 15 | func TestNewMongoConn(t *testing.T) {
 16 | 	t.Parallel()
 17 | 
 18 | 	// Env check
 19 | 	connectionEnv := "TEST_INVALID_MONGO_CONN"
 20 | 
 21 | 	conn, err := query.NewMongoConn(connectionEnv)
 22 | 	require.Error(t, err)
 23 | 	require.Equal(t, err, query.ENV_NOT_SET)
 24 | 	require.Nil(t, conn)
 25 | 
 26 | 	// invalid URI check
 27 | 	connectionStr := "mongo://test@123:27017/not_exists"
 28 | 
 29 | 	conn, err = newMongoConnectionHelper(t, connectionEnv, connectionStr)
 30 | 	require.Error(t, err)
 31 | 	require.Nil(t, conn)
 32 | 
 33 | 	// valid connect check
 34 | 	connectionStr = "mongodb://test@123:27017/not_exists?directConnection=blah"
 35 | 
 36 | 	conn, err = newMongoConnectionHelper(t, connectionEnv, connectionStr)
 37 | 	require.NoError(t, err)
 38 | 	require.NotNil(t, conn)
 39 | 
 40 | }
 41 | 
 42 | func TestMongoCount(t *testing.T) {
 43 | 
 44 | 	ctx := context.Background()
 45 | 	connectionEnv := "TEST_MONGO_CONN"
 46 | 	connectionString := os.Getenv(connectionEnv)
 47 | 
 48 | 	// Setup
 49 | 
 50 | 	mongoClient, err := mongo.NewClient(options.Client().ApplyURI(connectionString))
 51 | 	require.NotNil(t, mongoClient)
 52 | 	require.NoError(t, err)
 53 | 
 54 | 	err = mongoClient.Connect(ctx)
 55 | 	require.NoError(t, err)
 56 | 
 57 | 	coll := mongoClient.Database("yolo_db").Collection("test")
 58 | 
 59 | 	// Clean
 60 | 	err = coll.Drop(ctx)
 61 | 	require.NoError(t, err)
 62 | 
 63 | 	// Test Data
 64 | 	_, err = coll.InsertOne(ctx, map[string]interface{}{"name": "test", "is_active": true})
 65 | 	require.NoError(t, err)
 66 | 	_, err = coll.InsertOne(ctx, map[string]interface{}{"name": "test1", "is_active": false})
 67 | 	require.NoError(t, err)
 68 | 
 69 | 	// New Conn
 70 | 	conn, err := newMongoConnectionHelper(t, connectionEnv, connectionString)
 71 | 	require.NoError(t, err)
 72 | 	require.NotNil(t, conn)
 73 | 
 74 | 	// Test
 75 | 	// Positive test cases
 76 | 	testQuery := config.Metric{
 77 | 		Database:   "yolo_db",
 78 | 		Collection: "test",
 79 | 		Query:      `{"is_active":true}`,
 80 | 	}
 81 | 	out, err := conn.Count(testQuery)
 82 | 	require.Equal(t, int64(1), out)
 83 | 	require.NoError(t, err)
 84 | 
 85 | 	testQuery.Query = ""
 86 | 	out, err = conn.Count(testQuery)
 87 | 	require.Equal(t, int64(2), out)
 88 | 	require.NoError(t, err)
 89 | 
 90 | 	// negative test cases
 91 | 	testQuery.Query = "{test:'invalidjson'"
 92 | 	out, err = conn.Count(testQuery)
 93 | 	require.Equal(t, int64(0), out)
 94 | 	require.Error(t, err)
 95 | 
 96 | 	testQuery.Query = `{"test":{"$count":"{"}}`
 97 | 	out, err = conn.Count(testQuery)
 98 | 	require.Equal(t, int64(0), out)
 99 | 	require.Error(t, err)
100 | 
101 | 	testQuery.Query = `{"test":{"$count":"{"}}`
102 | 	out, err = conn.Count(testQuery)
103 | 	require.Equal(t, int64(0), out)
104 | 	require.Error(t, err)
105 | 
106 | }
107 | 
108 | func newMongoConnectionHelper(t *testing.T, connectionEnv, connectionString string) (query.CountQuery, error) {
109 | 	t.Helper()
110 | 
111 | 	err := os.Setenv(connectionEnv, connectionString)
112 | 	require.NoError(t, err)
113 | 
114 | 	return query.NewMongoConn(connectionEnv)
115 | }
116 | 


--------------------------------------------------------------------------------
/pkg/query/sql.go:
--------------------------------------------------------------------------------
 1 | package query
 2 | 
 3 | import (
 4 | 	"database/sql"
 5 | 	"fmt"
 6 | 	"os"
 7 | 
 8 | 	_ "github.com/go-sql-driver/mysql"
 9 | 	_ "github.com/lib/pq"
10 | 	"github.com/pkg/errors"
11 | 	"github.com/xo/dburl"
12 | 	"github.com/yolossn/query2metric/pkg/config"
13 | )
14 | 
15 | type sqlQuery struct {
16 | 	connection string
17 | 	db         *sql.DB
18 | }
19 | 
20 | func NewSQLQuery(connnectionURL string) (CountQuery, error) {
21 | 	connnectionString := os.Getenv(connnectionURL)
22 | 	if connnectionString == "" {
23 | 		return nil, ENV_NOT_SET
24 | 	}
25 | 	db, err := dburl.Open(connnectionString)
26 | 	if err != nil {
27 | 		return nil, errors.Wrap(err, "Error in establishing connection to db")
28 | 	}
29 | 
30 | 	return &sqlQuery{connnectionURL, db}, nil
31 | }
32 | 
33 | func (s *sqlQuery) Count(metric config.Metric) (int64, error) {
34 | 	if metric.Query == "" {
35 | 		return 0, errors.New("Query is empty")
36 | 	}
37 | 
38 | 	countQuery := fmt.Sprintf("select count(*) from (%s) as count_query", metric.Query)
39 | 	row, err := s.db.Query(countQuery)
40 | 	if err != nil {
41 | 		return 0, errors.Wrap(err, "Error running query")
42 | 	}
43 | 	defer row.Close()
44 | 	var out int64
45 | 	if row.Next() {
46 | 		err := row.Scan(&out)
47 | 		if err != nil {
48 | 			return 0, errors.Wrap(err, "Error running query")
49 | 		}
50 | 	}
51 | 	return out, nil
52 | }
53 | 


--------------------------------------------------------------------------------
/pkg/query/sql_test.go:
--------------------------------------------------------------------------------
  1 | package query_test
  2 | 
  3 | import (
  4 | 	"os"
  5 | 	"testing"
  6 | 
  7 | 	"github.com/stretchr/testify/require"
  8 | 	"github.com/xo/dburl"
  9 | 	"github.com/yolossn/query2metric/pkg/config"
 10 | 	"github.com/yolossn/query2metric/pkg/query"
 11 | )
 12 | 
 13 | func TestNewSQLConn(t *testing.T) {
 14 | 	t.Parallel()
 15 | 
 16 | 	// Env check
 17 | 	connectionEnv := "TEST_INVALID_SQL_CONN"
 18 | 
 19 | 	conn, err := query.NewSQLQuery(connectionEnv)
 20 | 	require.Error(t, err)
 21 | 	require.Equal(t, err, query.ENV_NOT_SET)
 22 | 	require.Nil(t, conn)
 23 | 
 24 | 	// Invalid URI
 25 | 	connectionStr := "postg://127.0.0.1/sdffsdf"
 26 | 	conn, err = newSQLConnectionHelper(t, connectionEnv, connectionStr)
 27 | 	require.Error(t, err)
 28 | 	require.Nil(t, conn)
 29 | 
 30 | 	// Valid URI
 31 | 	connectionStr = os.Getenv("TEST_SQL_CONN")
 32 | 	conn, err = newSQLConnectionHelper(t, connectionEnv, connectionStr)
 33 | 	require.NoError(t, err)
 34 | 	require.NotNil(t, conn)
 35 | 
 36 | }
 37 | 
 38 | func TestSQLCount(t *testing.T) {
 39 | 
 40 | 	// Setup
 41 | 	connectionEnv := "TEST_SQL_CONN"
 42 | 	connectionStr := os.Getenv(connectionEnv)
 43 | 	db, err := dburl.Open(connectionStr)
 44 | 	require.NotNil(t, db)
 45 | 	require.NoError(t, err)
 46 | 
 47 | 	// Clean
 48 | 	dropTableStmt := `DROP TABLE IF EXISTS test`
 49 | 	res, err := db.Exec(dropTableStmt)
 50 | 	require.NoError(t, err)
 51 | 	require.NotNil(t, res)
 52 | 
 53 | 	// Test Data
 54 | 	createTableStmt := `CREATE TABLE test(name TEXT, is_active BOOLEAN)`
 55 | 	_, err = db.Exec(createTableStmt)
 56 | 	require.NoError(t, err)
 57 | 
 58 | 	insertStmt := `INSERT INTO test (name, is_active) VALUES ($1, $2)`
 59 | 	_, err = db.Exec(insertStmt, "test1", true)
 60 | 	require.NoError(t, err)
 61 | 	_, err = db.Exec(insertStmt, "test2", false)
 62 | 	require.NoError(t, err)
 63 | 
 64 | 	// New Conn
 65 | 	conn, err := newSQLConnectionHelper(t, connectionEnv, connectionStr)
 66 | 	require.NoError(t, err)
 67 | 	require.NotNil(t, conn)
 68 | 
 69 | 	// Test
 70 | 	// Positive test cases
 71 | 	testQuery := config.Metric{
 72 | 		Query: `select * from test where is_active = true`,
 73 | 	}
 74 | 	out, err := conn.Count(testQuery)
 75 | 	require.Equal(t, int64(1), out)
 76 | 	require.NoError(t, err)
 77 | 
 78 | 	testQuery.Query = `select * from test`
 79 | 	out, err = conn.Count(testQuery)
 80 | 	require.Equal(t, int64(2), out)
 81 | 	require.NoError(t, err)
 82 | 
 83 | 	testQuery.Query = ""
 84 | 	out, err = conn.Count(testQuery)
 85 | 	require.Error(t, err)
 86 | 	require.Equal(t, int64(0), out)
 87 | 
 88 | 	// Negative test cases
 89 | 	testQuery.Query = `test`
 90 | 	out, err = conn.Count(testQuery)
 91 | 	require.Error(t, err)
 92 | 	require.Equal(t, int64(0), out)
 93 | 
 94 | }
 95 | 
 96 | func newSQLConnectionHelper(t *testing.T, connectionEnv, connectionString string) (query.CountQuery, error) {
 97 | 	t.Helper()
 98 | 
 99 | 	err := os.Setenv(connectionEnv, connectionString)
100 | 	require.NoError(t, err)
101 | 
102 | 	return query.NewSQLQuery(connectionEnv)
103 | }
104 | 


--------------------------------------------------------------------------------
/pkg/scheduler/scheduler.go:
--------------------------------------------------------------------------------
  1 | package scheduler
  2 | 
  3 | import (
  4 | 	"time"
  5 | 
  6 | 	"github.com/pkg/errors"
  7 | 	"github.com/prometheus/client_golang/prometheus"
  8 | 	log "github.com/sirupsen/logrus"
  9 | 	"github.com/yolossn/query2metric/pkg/config"
 10 | 	"github.com/yolossn/query2metric/pkg/query"
 11 | )
 12 | 
 13 | func init() {
 14 | 	log.SetFormatter(&log.JSONFormatter{})
 15 | 	log.SetLevel(log.DebugLevel)
 16 | }
 17 | 
 18 | type Scheduler struct {
 19 | 	conf config.Config
 20 | }
 21 | 
 22 | func (s Scheduler) Start() error {
 23 | 	errorChan := make(chan bool, 1)
 24 | 	successChan := make(chan bool, 1)
 25 | 	for _, conn := range s.conf.Connections {
 26 | 		var dbConnection query.CountQuery
 27 | 		var err error
 28 | 		switch conn.Type {
 29 | 		case config.MONGO:
 30 | 			dbConnection, err = query.NewMongoConn(conn.ConnectionString)
 31 | 			if err != nil {
 32 | 				return err
 33 | 			}
 34 | 		case config.SQL:
 35 | 			dbConnection, err = query.NewSQLQuery(conn.ConnectionString)
 36 | 			if err != nil {
 37 | 				return err
 38 | 			}
 39 | 		default:
 40 | 			continue
 41 | 		}
 42 | 
 43 | 		for _, metric := range conn.Metrics {
 44 | 
 45 | 			gaugeMetric := prometheus.NewGauge(
 46 | 				prometheus.GaugeOpts{
 47 | 					Namespace: conn.Name,
 48 | 					Name:      metric.Name,
 49 | 					Help:      metric.HelpString,
 50 | 				},
 51 | 			)
 52 | 			err = prometheus.Register(gaugeMetric)
 53 | 			if err != nil {
 54 | 				return errors.Wrap(err, "Error registering metric")
 55 | 			}
 56 | 			ticker := time.NewTicker(time.Duration(metric.Time) * time.Second)
 57 | 			run(ticker, gaugeMetric, dbConnection, metric, successChan, errorChan)
 58 | 		}
 59 | 	}
 60 | 
 61 | 	go errorCounter(errorChan)
 62 | 	go successCounter(successChan)
 63 | 
 64 | 	return nil
 65 | }
 66 | 
 67 | func run(tick *time.Ticker, gauge prometheus.Gauge, quer query.CountQuery, metric config.Metric, successChan, errorChan chan bool) {
 68 | 
 69 | 	go func() {
 70 | 		for {
 71 | 			select {
 72 | 			case <-tick.C:
 73 | 				out, err := quer.Count(metric)
 74 | 				if err != nil {
 75 | 					errorChan <- true
 76 | 					log.WithFields(log.Fields{"db": metric.Database, "metric": metric.Name, "query": metric.Query}).Error(err)
 77 | 				} else {
 78 | 					gauge.Set(float64(out))
 79 | 					successChan <- true
 80 | 					log.WithFields(log.Fields{"db": metric.Database, "metric": metric.Name, "query": metric.Query}).Debug("success")
 81 | 				}
 82 | 			}
 83 | 		}
 84 | 	}()
 85 | }
 86 | 
 87 | func FromConfig(conf config.Config) Scheduler {
 88 | 	return Scheduler{conf}
 89 | }
 90 | 
 91 | func errorCounter(errorChan chan bool) {
 92 | 
 93 | 	errorCounter := prometheus.NewCounter(
 94 | 		prometheus.CounterOpts{
 95 | 			Namespace: "query2metric",
 96 | 			Name:      "error_count",
 97 | 			Help:      "No of errors when converting query to metric",
 98 | 		},
 99 | 	)
100 | 
101 | 	prometheus.MustRegister(errorCounter)
102 | 	for {
103 | 		switch {
104 | 		case <-errorChan:
105 | 			errorCounter.Inc()
106 | 		}
107 | 	}
108 | 
109 | }
110 | 
111 | func successCounter(successChan chan bool) {
112 | 
113 | 	successCounter := prometheus.NewCounter(
114 | 		prometheus.CounterOpts{
115 | 			Namespace: "query2metric",
116 | 			Name:      "success_count",
117 | 			Help:      "No of successful queries coverted to metrics",
118 | 		},
119 | 	)
120 | 
121 | 	prometheus.MustRegister(successCounter)
122 | 
123 | 	for {
124 | 		switch {
125 | 		case <-successChan:
126 | 			successCounter.Inc()
127 | 		}
128 | 	}
129 | 
130 | }
131 | 


--------------------------------------------------------------------------------
/pkg/scheduler/scheduler_test.go:
--------------------------------------------------------------------------------
  1 | package scheduler_test
  2 | 
  3 | import (
  4 | 	"os"
  5 | 	"testing"
  6 | 	"time"
  7 | 
  8 | 	"github.com/prometheus/client_golang/prometheus"
  9 | 	dto "github.com/prometheus/client_model/go"
 10 | 	"github.com/stretchr/testify/require"
 11 | 	"github.com/xo/dburl"
 12 | 	"github.com/yolossn/query2metric/pkg/config"
 13 | 	"github.com/yolossn/query2metric/pkg/scheduler"
 14 | )
 15 | 
 16 | func TestFromConfig(t *testing.T) {
 17 | 	t.Parallel()
 18 | 	config := config.Config{
 19 | 		Connections: []config.Connection{
 20 | 			{
 21 | 				Name: "test2",
 22 | 			},
 23 | 		},
 24 | 	}
 25 | 	sh := scheduler.FromConfig(config)
 26 | 	require.NotNil(t, sh)
 27 | }
 28 | 
 29 | func TestSchedulerStart(t *testing.T) {
 30 | 	t.Parallel()
 31 | 
 32 | 	conf, err := config.FromFile("../../test/test_config.yaml")
 33 | 	require.NoError(t, err)
 34 | 	sch := scheduler.FromConfig(*conf)
 35 | 	sch.Start()
 36 | 
 37 | 	// Setup
 38 | 	connectionEnv := "TEST_SQL_CONN"
 39 | 	connectionStr := os.Getenv(connectionEnv)
 40 | 	db, err := dburl.Open(connectionStr)
 41 | 	require.NotNil(t, db)
 42 | 	require.NoError(t, err)
 43 | 
 44 | 	dropTableStmt := `DROP TABLE IF EXISTS queries`
 45 | 	res, err := db.Exec(dropTableStmt)
 46 | 	require.NoError(t, err)
 47 | 	require.NotNil(t, res)
 48 | 
 49 | 	// Test Data
 50 | 	createTableStmt := `CREATE TABLE queries(name TEXT, is_active BOOLEAN)`
 51 | 	_, err = db.Exec(createTableStmt)
 52 | 	require.NoError(t, err)
 53 | 
 54 | 	insertStmt := `INSERT INTO queries (name, is_active) VALUES ($1, $2)`
 55 | 	_, err = db.Exec(insertStmt, "test1", true)
 56 | 	require.NoError(t, err)
 57 | 	_, err = db.Exec(insertStmt, "test2", false)
 58 | 	require.NoError(t, err)
 59 | 
 60 | 	// Test
 61 | 	time.Sleep(time.Duration(5) * time.Second)
 62 | 	out, err := prometheus.DefaultGatherer.Gather()
 63 | 	require.NotNil(t, out)
 64 | 	require.NoError(t, err)
 65 | 
 66 | 	metric := filterMetrics(t, out, []string{"postgres1_template_count"})
 67 | 	require.NotNil(t, metric)
 68 | 	require.Equal(t, 1, len(metric))
 69 | 	require.NotZero(t, metric[0].Metric[0].Gauge.Value)
 70 | 
 71 | 	currentValue := metric[0].Metric[0].Gauge.Value
 72 | 	// add more data
 73 | 	_, err = db.Exec(insertStmt, "test3", false)
 74 | 	require.NoError(t, err)
 75 | 
 76 | 	// wait for the metric to be recalculated from query
 77 | 	time.Sleep(time.Duration(5) * time.Second)
 78 | 	out, err = prometheus.DefaultGatherer.Gather()
 79 | 	require.NotNil(t, out)
 80 | 	require.NoError(t, err)
 81 | 
 82 | 	metric = filterMetrics(t, out, []string{"postgres1_template_count"})
 83 | 	require.NotNil(t, metric)
 84 | 	require.Equal(t, 1, len(metric))
 85 | 	require.NotZero(t, metric[0].Metric[0].Gauge.Value)
 86 | 	// check the value is not equal to old value
 87 | 	require.NotEqual(t, *currentValue, *metric[0].Metric[0].Gauge.Value)
 88 | 
 89 | 	// TearDown
 90 | 	res, err = db.Exec(dropTableStmt)
 91 | 	require.NoError(t, err)
 92 | 	require.NotNil(t, res)
 93 | 
 94 | }
 95 | 
 96 | func filterMetrics(t *testing.T, metrics []*dto.MetricFamily, names []string) []*dto.MetricFamily {
 97 | 
 98 | 	t.Helper()
 99 | 
100 | 	var filtered []*dto.MetricFamily
101 | 	for _, m := range metrics {
102 | 		drop := true
103 | 		for _, name := range names {
104 | 			if m.GetName() == name {
105 | 				drop = false
106 | 				break
107 | 			}
108 | 		}
109 | 		if !drop {
110 | 			filtered = append(filtered, m)
111 | 		}
112 | 	}
113 | 	return filtered
114 | }
115 | 


--------------------------------------------------------------------------------
/test/invalid_config.yaml:
--------------------------------------------------------------------------------
 1 | connections:
 2 |   - name: mongodb1
 3 |     type: MONGO
 4 |     connectionStringFromEnv: MONGO_CONN
 5 | metrics:
 6 |       - name: active_user_count
 7 |         helpString: users in the product
 8 |         database: yolo_db
 9 |         collection: user
10 |         query: '{"is_active":true}'
11 |         time: 60
12 |       - name: total_user_count
13 |         helpString: users in the product
14 |         database: yolo_db
15 |         collection: user
16 |         query: ""
17 |         time: 120
18 |   - name: postgres1
19 |     type: SQL
20 |     connectionStringFromEnv: POSTGRES_CONN
21 |     metrics:
22 |       - name: template_count
23 |         helpString: products in the db
24 |         query: select * from templates
25 |         time: 2
26 |       - name: active_template_count
27 |         helpString: products in the db
28 |         query: error
29 |         time: 4
30 | 


--------------------------------------------------------------------------------
/test/test_config.yaml:
--------------------------------------------------------------------------------
 1 | connections:
 2 |   - name: mongodb1
 3 |     type: MONGO
 4 |     connectionStringFromEnv: TEST_MONGO_CONN
 5 |     metrics:
 6 |       - name: active_user_count
 7 |         helpString: users in the product
 8 |         database: yolo_db
 9 |         collection: user
10 |         query: '{"is_active":true}'
11 |         time: 60
12 |       - name: total_user_count
13 |         helpString: users in the product
14 |         database: yolo_db
15 |         collection: user
16 |         query: ""
17 |         time: 120
18 |   - name: postgres1
19 |     type: SQL
20 |     connectionStringFromEnv: TEST_SQL_CONN
21 |     metrics:
22 |       - name: template_count
23 |         helpString: products in the db
24 |         query: select * from queries
25 |         time: 2
26 | 


--------------------------------------------------------------------------------