├── .github └── workflows │ └── build.yml ├── .gitignore ├── .goreleaser.yml ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── TODO.md ├── collector.go ├── config.go ├── config.sample.toml ├── docker-compose.yml ├── examples ├── docker-compose.yaml └── queries.sql ├── exporter.go ├── go.mod ├── go.sum ├── main.go ├── models.go ├── store ├── db.go └── manager.go └── utils.go /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | on: [push] 3 | jobs: 4 | 5 | build: 6 | name: Build 7 | runs-on: ubuntu-latest 8 | steps: 9 | 10 | - name: Set up Go 1.12 11 | uses: actions/setup-go@v1 12 | with: 13 | go-version: 1.12 14 | id: go 15 | 16 | - name: Check out code into the Go module directory 17 | uses: actions/checkout@v1 18 | 19 | - name: Get dependencies 20 | run: | 21 | go get -v -t -d ./... 22 | if [ -f Gopkg.toml ]; then 23 | curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh 24 | dep ensure 25 | fi 26 | 27 | - name: Build 28 | run: go build -v . 29 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, build with `go test -c` 9 | *.test 10 | dist/ 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | config.toml 14 | store-exporter -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | env: 2 | - GO111MODULE=on 3 | - CGO_ENABLED=0 4 | - RELEASE_BUILDS=dist/store-exporter_darwin_amd64/store-exporter dist/store-exporter_linux_amd64/store-exporter dist/store-exporter_windows_amd64//store-exporter.exe 5 | 6 | builds: 7 | - binary: store-exporter 8 | goos: 9 | - windows 10 | - darwin 11 | - linux 12 | goarch: 13 | - amd64 14 | ldflags: 15 | - -s -w -X "main.buildVersion={{ .Tag }} ({{ .ShortCommit }} {{ .Date }})" 16 | 17 | archives: 18 | - format: tar.gz 19 | files: 20 | - config.sample.toml 21 | - README.md 22 | - LICENSE 23 | dockers: 24 | # You can have multiple Docker images. 25 | - 26 | # GOOS of the built binary that should be used. 27 | goos: linux 28 | # GOARCH of the built binary that should be used. 29 | goarch: amd64 30 | # GOARM of the built binary that should be used. 31 | goarm: '' 32 | # Name templates of the built binaries that should be used. 33 | binaries: 34 | - store-exporter 35 | # Templates of the Docker image names. 36 | image_templates: 37 | - "mrkaran/store-exporter:latest" 38 | - "mrkaran/store-exporter:{{ .Tag }}" 39 | # Skips the docker push. Could be useful if you also do draft releases. 40 | # If set to auto, the release will not be pushed to the docker repository 41 | # in case there is an indicator for prerelease in the tag e.g. v1.0.0-rc1 42 | # Defaults to false. 43 | skip_push: false 44 | # Path to the Dockerfile (from the project root). 45 | dockerfile: Dockerfile 46 | # If your Dockerfile copies files other than the binary itself, 47 | # you should list them here as well. 48 | # Note that goreleaser will create the same structure inside the temporary 49 | # folder, so if you add `foo/bar.json` here, on your Dockerfile you can 50 | # `COPY foo/bar.json /whatever.json`. 51 | # Also note that the paths here are relative to the folder in which 52 | # goreleaser is being run. 53 | # This field does not support wildcards, you can add an entire folder here 54 | # and use wildcards when you `COPY`/`ADD` in your Dockerfile. 55 | extra_files: 56 | - config.sample.toml -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | store-exporter uses GitHub to manage reviews of pull requests. 4 | 5 | * If you have a trivial fix or improvement, go ahead and create a pull request, 6 | addressing (with `@...`) the author of this repository, in the description of the pull request. 7 | 8 | * If you plan to do something more involved, first discuss your ideas 9 | on [Github Issues](https://github.com/mr-karan/store-exporter/issues). 10 | This will avoid unnecessary work and surely give you and us a good deal 11 | of inspiration. 12 | 13 | * Relevant coding style guidelines are the [Go Code Review 14 | Comments](https://code.google.com/p/go-wiki/wiki/CodeReviewComments) 15 | and the _Formatting and style_ section of Peter Bourgon's [Go: Best 16 | Practices for Production 17 | Environments](http://peter.bourgon.org/go-in-production/#formatting-and-style). 18 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:latest AS deploy 2 | RUN apk --no-cache add ca-certificates 3 | COPY store-exporter / 4 | COPY config.sample.toml /etc/store-exporter/config.toml 5 | VOLUME ["/etc/store-exporter"] 6 | CMD ["./store-exporter", "--config", "/etc/store-exporter/config.toml"] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Karan Sharma 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY : build run fresh test clean 2 | 3 | BIN := store-exporter 4 | 5 | HASH := $(shell git rev-parse --short HEAD) 6 | COMMIT_DATE := $(shell git show -s --format=%ci ${HASH}) 7 | BUILD_DATE := $(shell date '+%Y-%m-%d %H:%M:%S') 8 | VERSION := ${HASH} (${COMMIT_DATE}) 9 | 10 | 11 | build: 12 | go build -o ${BIN} -ldflags="-X 'main.buildVersion=${VERSION}' -X 'main.buildDate=${BUILD_DATE}'" 13 | 14 | run: 15 | ./store-exporter 16 | 17 | fresh: clean build run 18 | 19 | test: 20 | go test 21 | 22 | clean: 23 | go clean 24 | - rm -f ${BIN} -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # store-exporter 4 | _Utility to extract metrics from arbitary data stores in Prometheus format_ 5 | 6 | ## Overview 7 | 8 | Export your custom app metrics from external data _stores_ like PostgreSQL, MySQL, Redis(coming soon!) 9 | 10 | ## Features 11 | 12 | - Extract column names from results and expose them as custom metric labels. 13 | - Ability to register multiple jobs with different stores. 14 | 15 | ## Table of Contents 16 | 17 | - [Getting Started](#getting-started) 18 | - [Motivation](#motivation) 19 | - [Installation](#installation) 20 | - [Quickstart](#quickstart) 21 | - [Sending a sample scrape request](#testing-a-sample-alert) 22 | 23 | - [Advanced Section](#advanced-section) 24 | - [Configuration options](#configuation-options) 25 | - [Setting up Prometheus](#setting-up-prometheus) 26 | 27 | 28 | ### Motivation 29 | 30 | `store-exporter` loads SQL query file and fetches the data from DB and transforms the result in Prometheus text format. A lot of times, it is undesirable to add instrumentation right in your app for the following reasons: 31 | 32 | - Your app doesn't have any HTTP server, but to just extract metrics you've to invoke HTTP server. 33 | - Your app cares about being _fast_ in which case adding any external library penalises performance. 34 | - You don't want to mix the app logic with the metric collection/exposition logic. 35 | 36 | In all the above cases, it is more suitable to take a [Sidecar approach](https://docs.microsoft.com/en-us/azure/architecture/patterns/sidecar), where you query for metrics from an external persistent store your app maintains. This utility just makes it easier for anyone to write custom SQL queries and expose metrics without having to worry about Prometheus format/exposition logic. You can run a single binary anywhere in your cluster environment which has access to the external store which exposes the metrics on an HTTP server confirming to Prometheus metric format. 37 | 38 | 39 | ### Installation 40 | 41 | There are multiple ways of installing `store-exporter`. 42 | 43 | ### Running as docker container 44 | 45 | [mrkaran/store-exporter](https://hub.docker.com/r/mrkaran/store-exporter) 46 | 47 | `docker run -p 9609:9609 -v /etc/store-exporter/config.toml:/etc/store-exporter/config.toml mrkaran/store-exporter:latest` 48 | 49 | ### Precompiled binaries 50 | 51 | Precompiled binaries for released versions are available in the [_Releases_ section](https://github.com/mr-karan/store-exporter/releases/). 52 | 53 | ### Compiling the binary 54 | 55 | You can checkout the source code and build manually: 56 | 57 | ```bash 58 | git clone https://github.com/mr-karan/store-exporter.git 59 | cd store-exporter 60 | make build 61 | cp config.sample.toml config.toml 62 | ./store-exporter 63 | ``` 64 | 65 | ### Quickstart 66 | 67 | ```sh 68 | mkdir store-exporter && cd store-exporter/ # copy the binary and config.sample in this folder 69 | cp config.toml.sample config.toml # change the settings like server address, job metadata, db credentials etc. 70 | ./store-exporter # this command starts a web server and is ready to collect metrics. 71 | ``` 72 | 73 | ### Testing a sample scrape request 74 | 75 | You can send a `GET` request to `/metrics` and see the following metrics in Prometheus format: 76 | 77 | ```bash 78 | # HELP job_name_basicname this is such a great help text 79 | # TYPE job_name_basicname gauge 80 | job_name_basicname{job="myjob",pg_db_blks_hit="74400",pg_db_tup_inserted="120"} 13713 81 | # HELP job_name_verybasic_name this is such a great help text again 82 | # TYPE job_name_verybasic_name gauge 83 | job_name_verybasic_name{job="myjob",pg_db_conflicts="0",pg_db_temp_bytes="0"} 40 84 | # HELP version Version of store-exporter 85 | # TYPE version gauge 86 | version{build="846771f (2019-08-28 10:28:07 +0530)"} 1 87 | ``` 88 | 89 | ## Advanced Section 90 | 91 | ### Configuration Options 92 | 93 | - **[server]** 94 | - **address**: Port which the server listens to. Default is *9608* 95 | - **name**: _Optional_, human identifier for the server. 96 | - **read_timeout**: Duration (in milliseconds) for the request body to be fully read) Read this [blog](https://blog.cloudflare.com/the-complete-guide-to-golang-net-http-timeouts/) for more info. 97 | - **write_timeout**: Duration (in milliseconds) for the response body to be written. 98 | 99 | - **[app]** 100 | - **log_level**: "production" for all `INFO` level logs. If you want to enable verbose logging use "debug". 101 | - **jobs** 102 | - **name**: Unique identifier for the job. 103 | -- **store**: Config options for the store 104 | - **db**: Type of SQL DB. Supported values: [postgres, mysql]. 105 | - **dsn**: Connection URL to the DB. 106 | - **query**: Path to SQL file. 107 | - **max_open_connections**: Max open connections to the DB. 108 | - **max_idle_connections**: Max idle connections maintained in the connection pool. 109 | - **metrics**: 110 | - **namespace**: Unique identifier for the metric, prepended in each metric name. 111 | - **help**: Helptext for the metric 112 | - **query**: Name of the query mapped in `sql` file, used to query the db for this metric. 113 | - **labels**: List of additional column names fetched from the DB, to be used in metric as key/value pairs. 114 | - **columns**: Each column name creates a separate Prometheus `metric` and the corresponding value fetched from the store is used as the metric value. 115 | 116 | **NOTE**: You can use `--config` flag to supply a custom config file path while running `store-exporter`. 117 | 118 | ### Setting up Prometheus 119 | 120 | You can add the following config under `scrape_configs` in Prometheus' configuration. 121 | 122 | ```yaml 123 | - job_name: 'store-exporter' 124 | metrics_path: '/metrics' 125 | static_configs: 126 | - targets: ['localhost:9610'] 127 | labels: 128 | service: my-app-metrics 129 | ``` 130 | 131 | Validate your setup by querying `version` to check if store-exporter is discovered by Prometheus: 132 | 133 | ```plain 134 | `version{build="846771f (2019-08-28 10:28:07 +0530)"} 1` 135 | ``` 136 | 137 | ## Contribution 138 | 139 | PRs on Feature Requests, Bug fixes are welcome. Feel free to open an issue and have a discussion first. Contributions on more external stores are also welcome and encouraged. 140 | 141 | Read [CONTRIBUTING.md](CONTRIBUTING.md) for more details. 142 | 143 | ## License 144 | 145 | [MIT](license) 146 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | - Redis store 2 | - Metric type (currently only gaugevec) -------------------------------------------------------------------------------- /collector.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "fmt" 4 | 5 | // constructMetricData takes a map of column names with corresponding values and returns in 6 | // format of Prometheus metric value and lables 7 | func constructMetricData(data map[string]interface{}, column string, labels []string) (float64, []string, error) { 8 | var ( 9 | value float64 10 | err error 11 | ) 12 | // if column name is in the result set, get the value 13 | if i, ok := data[column]; ok { 14 | value = getFloatValue(i) 15 | } 16 | labelValues := []string{} 17 | // iterate over labels and extract the value as k/v pair to construct labels 18 | for _, label := range labels { 19 | // fallback empty value in case column name doesn't match from config with the result set 20 | // This is done to ensure label cardinality 21 | lv := "" 22 | if i, ok := data[label]; ok { 23 | lv = getStringValue(i) 24 | // in case column name matches but type is incorrect, return error to let the user know 25 | // and not silently fail. 26 | if lv == "" { 27 | err = fmt.Errorf("Column: %s must be type text/string", label) 28 | } 29 | } 30 | labelValues = append(labelValues, lv) 31 | } 32 | return value, labelValues, err 33 | } 34 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | log "github.com/sirupsen/logrus" 8 | 9 | "github.com/knadh/koanf" 10 | "github.com/knadh/koanf/parsers/toml" 11 | "github.com/knadh/koanf/providers/file" 12 | flag "github.com/spf13/pflag" 13 | ) 14 | 15 | // initConfig initializes the app's configuration manager 16 | // and loads disk and command line configuration values. 17 | func initConfig() config { 18 | var cfg = config{} 19 | var koanf = koanf.New(".") 20 | // Parse Command Line Flags. 21 | // --config flag to specify the location of config file on fs 22 | flagSet := flag.NewFlagSet("config", flag.ContinueOnError) 23 | flagSet.Usage = func() { 24 | fmt.Println(flagSet.FlagUsages()) 25 | os.Exit(0) 26 | } 27 | // Config Location. 28 | flagSet.StringSlice("config", []string{}, "Path to a config file to load. This can be specified multiple times and the config files will be merged in order") 29 | // Process flags. 30 | failOnReadConfigErr(flagSet.Parse(os.Args[1:])) 31 | // Read default config file. Won't throw the error yet. 32 | vErr := koanf.Load(file.Provider("config.toml"), toml.Parser()) 33 | // Load the config files provided in the commandline if there are any. 34 | cFiles, _ := flagSet.GetStringSlice("config") 35 | for _, c := range cFiles { 36 | if err := koanf.Load(file.Provider(c), toml.Parser()); err != nil { 37 | log.Fatalf("error loading config file: %v", err) 38 | } 39 | } 40 | // If no default config is read and no additional config is supplied, exit. 41 | if vErr != nil { 42 | if len(cFiles) == 0 { 43 | log.Fatalf("no config was read: %v", vErr) 44 | } 45 | } 46 | // Read the configuration and load it to internal struct. 47 | failOnReadConfigErr(koanf.Unmarshal("", &cfg)) 48 | return cfg 49 | } 50 | 51 | func failOnReadConfigErr(err error) { 52 | if err != nil { 53 | log.Fatalf("error reading config: %v.", err) 54 | } 55 | } -------------------------------------------------------------------------------- /config.sample.toml: -------------------------------------------------------------------------------- 1 | [app] 2 | # Logging Level. Supported values are `production` and `debug`. 3 | log_level = "debug" 4 | # List of jobs to be registered to collect metrics for. 5 | [[app.jobs]] 6 | # Unique Identifier for a Job. 7 | name="dbstats" 8 | # Supported external stores are `postgres`, `mysql` and `sqlite3`. 9 | [app.jobs.store] 10 | db="postgres" 11 | # Path to SQL file to be used for all metric queries for this job. 12 | query="examples/queries.sql" 13 | # DSN connection string to authenticate and create a connection to external store. (For sqlite3, it's just the path to db file) 14 | dsn="postgres://postgres:postgres@localhost:9432/postgres?sslmode=disable" 15 | max_open_connections=3 16 | max_idle_connections=5 17 | # List of metric names to be collected in one job. 18 | [[app.jobs.metrics]] 19 | namespace="wowmy" 20 | # Query name defined in `query` file. 21 | query = "get-max" 22 | # Help text for the metric 23 | help="this is such a great help text" 24 | # List of additional labels to add to the metric. Extra care must be taken to ensure the labels are unique 25 | # to ensure label consistency. If uniqueness is not maintained, Prometheus will treat two metrics as different 26 | # and any kind of aggregation won't be possible in that case. The value of the label key is fetched from the SQL 27 | # result. 28 | labels=["abc"] 29 | # Each column name constructs a metric with the name as the column name and the corresponding column value 30 | # becomes the metric value. 31 | columns=["pg_db_blks_hit","pg_db_tup_inserted"] # Additional metrics constructed for each column 32 | 33 | [server] 34 | address = ":9610" 35 | name = "store-exporter" 36 | read_timeout=8000 37 | write_timeout=8000 38 | max_body_size=40000% 39 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | # NOTE: This docker-compose.yml is meant to be just an example guideline 2 | # on how you can achieve the same. It is not intented to run out of the box 3 | # and you must edit the below configurations to suit your needs. 4 | 5 | version: "3.7" 6 | 7 | services: 8 | app: 9 | restart: unless-stopped 10 | image: mrkaran/store-exporter:latest 11 | ports: 12 | - "9610:9610" 13 | networks: 14 | - store-exporter 15 | volumes: 16 | - type: bind 17 | source: /etc/store-exporter/ 18 | target: /etc/store-exporter/ 19 | 20 | networks: 21 | store-exporter: 22 | -------------------------------------------------------------------------------- /examples/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | # NOTE: This docker-compose.yml is meant to be just an example guideline 2 | # on how you can achieve the same. It is not intented to run out of the box 3 | # and you must edit the below configurations to suit your needs. 4 | 5 | version: "3.7" 6 | 7 | x-db-defaults: &db-defaults 8 | image: postgres:11 9 | ports: 10 | - "9432:5432" 11 | networks: 12 | - store-exporter 13 | environment: 14 | - POSTGRES_PASSWORD=postgres 15 | - POSTGRES_USER=postgres 16 | - POSTGRES_DB=postgres 17 | restart: unless-stopped 18 | 19 | services: 20 | db: 21 | <<: *db-defaults 22 | volumes: 23 | - type: volume 24 | source: store-exporter-test-data 25 | target: /var/lib/postgresql/data 26 | 27 | networks: 28 | store-exporter: 29 | 30 | volumes: 31 | store-exporter-test-data: 32 | -------------------------------------------------------------------------------- /examples/queries.sql: -------------------------------------------------------------------------------- 1 | -- queries.sql 2 | 3 | -- name: get-max 4 | SELECT 5 | datname AS datname, 6 | numbackends AS pg_db_numbackends, 7 | xact_commit AS pg_db_xact_commit, 8 | xact_rollback AS pg_db_xact_rollback, 9 | blks_read AS pg_db_blks_read, 10 | blks_hit::text AS pg_db_blks_hit, 11 | tup_returned AS pg_db_tup_returned, 12 | tup_fetched AS pg_db_tup_fetched, 13 | tup_inserted AS pg_db_tup_inserted, 14 | tup_updated AS pg_db_tup_updated, 15 | tup_deleted AS pg_db_tup_deleted, 16 | conflicts AS pg_db_conflicts, 17 | temp_bytes AS pg_db_temp_bytes, 18 | deadlocks AS pg_db_deadlocks, 19 | blk_read_time AS pg_db_blk_read_time, 20 | blk_write_time AS pg_db_blk_write_time 21 | FROM pg_stat_database 22 | 23 | 24 | -- name: get-total 25 | SELECT * 26 | FROM foo 27 | WHERE bar = $1; 28 | -------------------------------------------------------------------------------- /exporter.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | 7 | "github.com/mr-karan/store-exporter/store" 8 | "github.com/prometheus/client_golang/prometheus" 9 | ) 10 | 11 | // NewExporter returns an initialized `Exporter`. 12 | func (hub *Hub) NewExporter(job *Job) (*Exporter, error) { 13 | manager, err := store.NewManager(job.Store) 14 | if err != nil { 15 | hub.logger.Errorf("Error initializing database manager: %s", err) 16 | return nil, err 17 | } 18 | return &Exporter{ 19 | Mutex: sync.Mutex{}, 20 | manager: manager, 21 | job: job, 22 | hub: hub, 23 | up: prometheus.NewDesc( 24 | prometheus.BuildFQName(job.Name, "", "up"), 25 | "Could the data source be reached.", 26 | nil, 27 | nil, 28 | ), 29 | version: prometheus.NewDesc( 30 | prometheus.BuildFQName(job.Name, "", "version"), 31 | "Version of store-exporter", 32 | []string{"build"}, 33 | nil, 34 | ), 35 | }, nil 36 | } 37 | 38 | // sendSafeMetric is a concurrent safe method to send metrics to a channel. Since we are collecting metrics from AWS API, there might be possibility where 39 | // a timeout occurs from Prometheus' collection context and the channel is closed but Goroutines running in background can still 40 | // send metrics to this closed channel which would result in panic and crash. To solve that we use context and check if the channel is not closed 41 | // and only send the metrics in that case. Else it logs the error and returns in a safe way. 42 | func (hub *Hub) sendSafeMetric(ctx context.Context, ch chan<- prometheus.Metric, metric prometheus.Metric) error { 43 | // Check if collection context is finished 44 | select { 45 | case <-ctx.Done(): 46 | // don't send metrics, instead return in a "safe" way 47 | hub.logger.Errorf("Attempted to send metrics to a closed channel after collection context had finished: %s", metric) 48 | return ctx.Err() 49 | default: // continue 50 | } 51 | // Send metrics if collection context is still open 52 | ch <- metric 53 | return nil 54 | } 55 | 56 | // Describe describes all the metrics ever exported by the exporter. It implements `prometheus.Collector`. 57 | func (p *Exporter) Describe(ch chan<- *prometheus.Desc) { 58 | ch <- p.version 59 | ch <- p.up 60 | } 61 | 62 | // Collect is called by the Prometheus registry when collecting 63 | // metrics. This method may be called concurrently and must therefore be 64 | // implemented in a concurrency safe way. It implements `prometheus.Collector` 65 | func (p *Exporter) Collect(ch chan<- prometheus.Metric) { 66 | // Initialize context to keep track of the collection. 67 | ctx, cancel := context.WithCancel(context.Background()) 68 | defer cancel() 69 | // Lock the exporter for one iteration of collection as `Collect` can be called concurrently. 70 | p.Lock() 71 | defer p.Unlock() 72 | p.hub.logger.Debugf("Collecting metric data for job: %v", p.job.Name) 73 | for _, m := range p.job.Metrics { 74 | p.collectMetrics(ctx, ch, m) 75 | } 76 | // Send default metrics data. 77 | p.hub.sendSafeMetric(ctx, ch, prometheus.MustNewConstMetric(p.version, prometheus.GaugeValue, 1, p.hub.version)) 78 | } 79 | 80 | // collectMetrics fetches data from external stores and sends as Prometheus metrics 81 | func (p *Exporter) collectMetrics(ctx context.Context, ch chan<- prometheus.Metric, metric Metric) { 82 | p.hub.logger.Debugf("Querying the store for metrics with query: %v", metric.Query) 83 | data, err := p.manager.FetchResults(metric.Query) 84 | if err != nil { 85 | p.hub.logger.Errorf("Error while fetching result from DB: %v", err) 86 | return 87 | } 88 | p.hub.logger.Debugf("Fetched data from db for job: %v metric: %v", p.job.Name, metric.Namespace) 89 | for _, col := range metric.Columns { 90 | value, labelValues, err := constructMetricData(data, col, metric.Labels) 91 | if err != nil { 92 | p.hub.logger.Errorf("Error while converting results to metrics: %v", err) 93 | return 94 | } 95 | // Create metrics on the fly 96 | metricDesc := createMetricDesc(metric.Namespace, col, p.job.Name, metric.Help, metric.Labels) 97 | p.hub.sendSafeMetric(ctx, ch, prometheus.MustNewConstMetric(metricDesc, prometheus.GaugeValue, value, labelValues...)) 98 | } 99 | return 100 | } 101 | 102 | // createMetricDesc returns an intialized prometheus.Desc instance 103 | func createMetricDesc(namespace string, metricName string, jobName string, helpText string, additionalLabels []string) *prometheus.Desc { 104 | // Default labels for any metric constructed with this function. 105 | var labels []string 106 | // Iterate through a slice of additional labels to be exported. 107 | for _, k := range additionalLabels { 108 | // Replace all tags with underscores if present to make it a valid Prometheus label name. 109 | labels = append(labels, replaceWithUnderscores(k)) 110 | } 111 | return prometheus.NewDesc( 112 | prometheus.BuildFQName(replaceWithUnderscores(namespace), "", replaceWithUnderscores(metricName)), 113 | helpText, 114 | labels, prometheus.Labels{"job": replaceWithUnderscores(jobName)}, 115 | ) 116 | } 117 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/mr-karan/store-exporter 2 | 3 | go 1.12 4 | 5 | require ( 6 | github.com/aws/aws-sdk-go v1.23.7 7 | github.com/go-sql-driver/mysql v1.4.0 8 | github.com/jmoiron/sqlx v1.2.0 9 | github.com/knadh/goyesql v2.0.0+incompatible 10 | github.com/knadh/koanf v0.4.4 11 | github.com/lib/pq v1.0.0 12 | github.com/mattn/go-sqlite3 v1.9.0 13 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 14 | github.com/modern-go/reflect2 v1.0.1 // indirect 15 | github.com/prometheus/client_golang v1.1.0 16 | github.com/sirupsen/logrus v1.4.2 17 | github.com/spf13/pflag v1.0.3 18 | ) 19 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 2 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 3 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 4 | github.com/aws/aws-sdk-go v1.23.7 h1:mbIEzk5n9DqZF11dqsO7KghlJU/wHn6w8KIGsBIFsmA= 5 | github.com/aws/aws-sdk-go v1.23.7/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= 6 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 7 | github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= 8 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 9 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 10 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 11 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 12 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 13 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 14 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= 15 | github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= 16 | github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= 17 | github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= 18 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 19 | github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= 20 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 21 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 22 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 23 | github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= 24 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 25 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 26 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 27 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 28 | github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= 29 | github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= 30 | github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= 31 | github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= 32 | github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 33 | github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 34 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= 35 | github.com/knadh/goyesql v2.0.0+incompatible h1:hJFJrU8kaiLmvYt9I/1k1AB7q+qRhHs/afzTfQ3eGqk= 36 | github.com/knadh/goyesql v2.0.0+incompatible/go.mod h1:W0tSzU8l7lYH1Fihj+bdQzkzOwvirrsMNHwkuY22qoY= 37 | github.com/knadh/koanf v0.4.4 h1:Pg+eR7wuJtCGHLeip31K20eJojjZ3lXE8ILQQGj2PTM= 38 | github.com/knadh/koanf v0.4.4/go.mod h1:Qd5yvXN39ZzjoRJdXMKN2QqHzQKhSx/K8fU5gyn4LPs= 39 | github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= 40 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 41 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 42 | github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= 43 | github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= 44 | github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4= 45 | github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= 46 | github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= 47 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 48 | github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= 49 | github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 50 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 51 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 52 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 53 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 54 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 55 | github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfSeg= 56 | github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= 57 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 58 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 59 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 60 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 61 | github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= 62 | github.com/prometheus/client_golang v1.1.0 h1:BQ53HtBmfOitExawJ6LokA4x8ov/z0SYYb0+HxJfRI8= 63 | github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= 64 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 65 | github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= 66 | github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 67 | github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 68 | github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo= 69 | github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= 70 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 71 | github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= 72 | github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE= 73 | github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= 74 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 75 | github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= 76 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 77 | github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= 78 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 79 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 80 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 81 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 82 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= 83 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 84 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 85 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 86 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 87 | golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 88 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 89 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 90 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 91 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 92 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 93 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= 94 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 95 | golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3 h1:4y9KwBHBgBNwDbtu44R5o1fdOCQUEXhbk/P4A9WmJq0= 96 | golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 97 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 98 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 99 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 100 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 101 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 102 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | 7 | "github.com/prometheus/client_golang/prometheus" 8 | "github.com/prometheus/client_golang/prometheus/promhttp" 9 | "github.com/sirupsen/logrus" 10 | ) 11 | 12 | var ( 13 | // injected during build 14 | buildVersion = "unknown" 15 | buildDate = "unknown" 16 | ) 17 | 18 | func initLogger(config cfgApp) *logrus.Logger { 19 | // Initialize logger 20 | logger := logrus.New() 21 | logger.SetFormatter(&logrus.TextFormatter{ 22 | FullTimestamp: true, 23 | }) 24 | // Set logger level 25 | switch level := config.LogLevel; level { 26 | case "debug": 27 | logger.SetLevel(logrus.DebugLevel) 28 | logger.Debug("verbose logging enabled") 29 | default: 30 | logger.SetLevel(logrus.InfoLevel) 31 | } 32 | return logger 33 | } 34 | 35 | func main() { 36 | var ( 37 | config = initConfig() 38 | logger = initLogger(config.App) 39 | ) 40 | // Initialize hub which contains initializations of app level objects 41 | hub := &Hub{ 42 | config: config, 43 | logger: logger, 44 | version: buildVersion, 45 | } 46 | hub.logger.Infof("booting store-exporter-version:%v", buildVersion) 47 | // Initialize prometheus registry to register metrics and collectors. 48 | r := prometheus.NewRegistry() 49 | // Fetch all jobs listed in config and register with the registry. 50 | for _, job := range hub.config.App.Jobs { 51 | // This is to avoid all copies of `exporter` getting updated by the last `job` memory address 52 | // you instantiate with, since we pass `job` as a pointer to the struct. 53 | j := job 54 | // Initialize the exporter. Exporter is a collection of metrics to be exported. 55 | exporter, err := hub.NewExporter(&j) 56 | if err != nil { 57 | hub.logger.Panicf("exporter initialization failed for %s : %s", job.Name, err) 58 | } 59 | // Register the exporters with our custom registry. Panics in case of failure. 60 | r.MustRegister(exporter) 61 | hub.logger.Debugf("registration of metrics for job %s success", job.Name) 62 | } 63 | // Default index handler. 64 | handleIndex := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 65 | w.WriteHeader(http.StatusOK) 66 | w.Write([]byte("Welcome to store-exporter. Visit /metrics to scrape prometheus metrics.")) 67 | }) 68 | // Initialize router and define all endpoints. 69 | router := http.NewServeMux() 70 | router.Handle("/", handleIndex) 71 | router.Handle("/metrics", promhttp.HandlerFor(r, promhttp.HandlerOpts{})) 72 | // Initialize server. 73 | server := &http.Server{ 74 | Addr: hub.config.Server.Address, 75 | Handler: router, 76 | ReadTimeout: hub.config.Server.ReadTimeout * time.Millisecond, 77 | WriteTimeout: hub.config.Server.WriteTimeout * time.Millisecond, 78 | } 79 | // Start the server. Blocks the main thread. 80 | hub.logger.Infof("starting server listening on %v", hub.config.Server.Address) 81 | if err := server.ListenAndServe(); err != nil { 82 | hub.logger.Fatalf("error starting server: %v", err) 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /models.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | 7 | "github.com/mr-karan/store-exporter/store" 8 | "github.com/prometheus/client_golang/prometheus" 9 | "github.com/sirupsen/logrus" 10 | ) 11 | 12 | // Hub represents the structure for all app wide functions and structs 13 | type Hub struct { 14 | logger *logrus.Logger 15 | config config 16 | version string 17 | } 18 | 19 | // cfgApp represents the structure to hold App specific configuration. 20 | type cfgApp struct { 21 | LogLevel string `koanf:"log_level"` 22 | Jobs []Job `koanf:"jobs"` 23 | } 24 | 25 | // cfgServer represents the structure to hold Server specific configuration 26 | type cfgServer struct { 27 | Name string `koanf:"name"` 28 | Address string `koanf:"address"` 29 | ReadTimeout time.Duration `koanf:"read_timeout"` 30 | WriteTimeout time.Duration `koanf:"write_timeout"` 31 | MaxBodySize int `koanf:"max_body_size"` 32 | } 33 | 34 | // config represents the structure to hold configuration loaded from an external data source. 35 | type config struct { 36 | App cfgApp `koanf:"app"` 37 | Server cfgServer `koanf:"server"` 38 | } 39 | 40 | // Job represents a list of scrape jobs with additional config for the target. 41 | type Job struct { 42 | Name string `koanf:"name"` 43 | Store store.Store `koanf:"store"` 44 | Metrics []Metric `koanf:"metrics"` 45 | } 46 | 47 | // Exporter represents the structure to hold Prometheus Descriptors. It implements prometheus.Collector 48 | type Exporter struct { 49 | sync.Mutex // Lock exporter to protect from concurrent scrapes. 50 | hub *Hub // To access logger and other app wide config. 51 | job *Job // Holds the Job metadata. 52 | manager store.Manager // Implements Manager interface which is a set of methods to interact with dataset. 53 | up *prometheus.Desc // Represents if a scrape was successful or not. 54 | version *prometheus.Desc // Represents verion of the exporter. 55 | } 56 | 57 | // Metric represents the structure to hold details about constructing a Prometheus.Metric 58 | type Metric struct { 59 | Namespace string `koanf:"namespace"` 60 | Query string `koanf:"query"` 61 | Columns []string `koanf:"columns"` 62 | Help string `koanf:"help"` 63 | Labels []string `koanf:"labels"` 64 | } 65 | -------------------------------------------------------------------------------- /store/db.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/jmoiron/sqlx" 7 | "github.com/knadh/goyesql" 8 | ) 9 | 10 | // DBClient represents the structure to hold DB Client object required to create DB session and 11 | // query and fetch results. 12 | type DBClient struct { 13 | Conn *sqlx.DB 14 | Queries goyesql.Queries 15 | } 16 | 17 | // DBClientOpts represents additional options to use for DB Client 18 | type DBClientOpts struct { 19 | QueryFile string 20 | MaxIdleConns int 21 | MaxOpenConns int 22 | } 23 | 24 | // NewDBClient initializes a connection object with the databse. 25 | func NewDBClient(db string, dsn string, opts *DBClientOpts) (Manager, error) { 26 | conn, err := sqlx.Connect(db, dsn) 27 | if err != nil { 28 | return nil, err 29 | } 30 | // Connection Pool grows unbounded, have some sane defaults. 31 | conn.SetMaxIdleConns(opts.MaxIdleConns) 32 | conn.SetMaxOpenConns(opts.MaxOpenConns) 33 | // Load queries 34 | if opts.QueryFile == "" { 35 | return nil, fmt.Errorf("error initialising DB Manager: Path to query file not provided") 36 | } 37 | queries := goyesql.MustParseFile(opts.QueryFile) 38 | 39 | return &DBClient{ 40 | Conn: conn, 41 | Queries: queries, 42 | }, nil 43 | } 44 | 45 | // FetchResults executes the query and parses the result 46 | func (client *DBClient) FetchResults(query string) (map[string]interface{}, error) { 47 | q, ok := client.Queries[query] 48 | if !ok { 49 | return nil, fmt.Errorf("No query mapped to: %s", query) 50 | } 51 | row := client.Conn.QueryRowx(q.Query) 52 | results := make(map[string]interface{}) 53 | err := row.MapScan(results) // connection is closed automatically here. Read more: https://jmoiron.github.io/sqlx/#queryrow 54 | if err != nil { 55 | return nil, err 56 | } 57 | return results, err 58 | } 59 | -------------------------------------------------------------------------------- /store/manager.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "fmt" 5 | 6 | _ "github.com/go-sql-driver/mysql" // Imports mysql driver 7 | _ "github.com/lib/pq" // Imports postgresql driver 8 | _ "github.com/mattn/go-sqlite3" // Imports sqlite driver 9 | ) 10 | 11 | // Store represents additional options for the external store 12 | type Store struct { 13 | MaxOpenConnections int `koanf:"max_open_connections"` 14 | MaxIdleConnections int `koanf:"max_idle_connections"` 15 | DB string `koanf:"db"` 16 | DSN string `koanf:"dsn"` 17 | QueryFile string `koanf:"query"` 18 | } 19 | 20 | // Manager represents the set of methods used to interact with the db. 21 | type Manager interface { 22 | FetchResults(string) (map[string]interface{}, error) 23 | } 24 | 25 | // DBConnOpts represents additonal parameters to create a DB Client 26 | 27 | // NewManager instantiates an object of Manager based on the params 28 | func NewManager(store Store) (Manager, error) { 29 | switch dbType := store.DB; dbType { 30 | case "postgres", "mysql": 31 | return NewDBClient(store.DB, store.DSN, &DBClientOpts{ 32 | QueryFile: store.QueryFile, 33 | MaxIdleConns: store.MaxIdleConnections, 34 | MaxOpenConns: store.MaxOpenConnections, 35 | }) 36 | case "sqlite3": 37 | return NewDBClient(store.DB, store.DSN, &DBClientOpts{ 38 | QueryFile: store.QueryFile, 39 | MaxIdleConns: store.MaxIdleConnections, 40 | MaxOpenConns: store.MaxOpenConnections, 41 | }) 42 | // TODO case "redis": 43 | default: 44 | return nil, fmt.Errorf("Error fetching results: Unknown db type") 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /utils.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "strconv" 5 | "strings" 6 | ) 7 | 8 | // replaceWithUnderscores replaces `-` with `_` 9 | func replaceWithUnderscores(text string) string { 10 | replacer := strings.NewReplacer(" ", "_", ",", "_", "\t", "_", ",", "_", "/", "_", "\\", "_", ".", "_", "-", "_", ":", "_", "=", "_") 11 | return replacer.Replace(text) 12 | } 13 | 14 | // getStringValue converts supported data types to string 15 | func getStringValue(i interface{}) string { 16 | switch v := i.(type) { 17 | case string: 18 | return v 19 | case []uint8: 20 | return string(v) 21 | case float64: 22 | return strconv.FormatFloat(v, 'f', 6, 64) 23 | case int: 24 | return strconv.Itoa(v) 25 | case int64: 26 | return strconv.FormatInt(v, 10) 27 | default: 28 | return "" 29 | } 30 | } 31 | 32 | // getStringValue converts supported data types to float64 33 | func getFloatValue(i interface{}) float64 { 34 | var value float64 35 | switch f := i.(type) { 36 | case int: 37 | value = float64(f) 38 | case int32: 39 | value = float64(f) 40 | case int64: 41 | value = float64(f) 42 | case uint: 43 | value = float64(f) 44 | case uint32: 45 | value = float64(f) 46 | case uint64: 47 | value = float64(f) 48 | case float32: 49 | value = float64(f) 50 | case float64: 51 | value = float64(f) 52 | case []uint8: 53 | val, err := strconv.ParseFloat(string(f), 64) 54 | if err != nil { 55 | return value 56 | } 57 | value = val 58 | case string: 59 | val, err := strconv.ParseFloat(f, 64) 60 | if err != nil { 61 | return value 62 | } 63 | value = val 64 | default: 65 | return value 66 | } 67 | return value 68 | } 69 | --------------------------------------------------------------------------------