├── .gitignore ├── .vscode └── launch.json ├── AUTHORS.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── README.md ├── config.xml ├── go.mod ├── img └── screen1.png ├── main.go ├── reader.go ├── schema.sql ├── srv.go ├── version.go └── writer.go /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode/ 2 | go.sum 3 | bin 4 | prom2click 5 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // 使用 IntelliSense 了解相关属性。 3 | // 悬停以查看现有属性的描述。 4 | // 欲了解更多信息，请访问: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Launch Package", 9 | "type": "go", 10 | "request": "launch", 11 | "mode": "auto", 12 | "program": "${fileDirname}" 13 | } 14 | ] 15 | } -------------------------------------------------------------------------------- /AUTHORS.md: -------------------------------------------------------------------------------- 1 | # Credits 2 | 3 | ## Development Lead 4 | 5 | - Kenny Freeman [s4z](https://github.com/s4z) 6 | 7 | ## Contributors 8 | 9 | None yet. Why not be the first? -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Contributions are welcome, and they are greatly appreciated! Every little bit helps, and credit will always be given. 4 | 5 | You can contribute in many ways: 6 | 7 | ## Types of Contributions 8 | 9 | ### Report Bugs 10 | 11 | Report bugs at https://github.com/s4z/prom2click/issues. 12 | 13 | If you are reporting a bug, please include: 14 | 15 | * Your operating system name and version. 16 | * Any details about your local setup that might be helpful in troubleshooting. 17 | * Detailed steps to reproduce the bug. 18 | 19 | ### Fix Bugs 20 | 21 | Look through the GitHub issues for bugs. Anything tagged with "bug" 22 | is open to whoever wants to implement it. 23 | 24 | ### Implement Features 25 | 26 | Look through the GitHub issues for features. Anything tagged with "feature" 27 | is open to whoever wants to implement it. 28 | 29 | ### Write Documentation 30 | 31 | prom2click could always use more documentation, whether as part of the 32 | official prom2click docs, in docstrings, or even on the web in blog posts, 33 | articles, and such. 34 | 35 | ### Submit Feedback 36 | 37 | The best way to send feedback is to file an issue at https://github.com/s4z/prom2click/issues. 38 | 39 | If you are proposing a feature: 40 | 41 | * Explain in detail how it would work. 42 | * Keep the scope as narrow as possible, to make it easier to implement. 43 | * Remember that this is a volunteer-driven project, and that contributions 44 | are welcome :) 45 | 46 | ## Get Started! 47 | 48 | Ready to contribute? Here's how to set up `prom2click` for local development. 49 | 50 | 1. Fork the `prom2click` repo on GitHub. 51 | 2. Clone your fork locally:: 52 | 53 | $ git clone git@github.com:your_name_here/prom2click.git 54 | 55 | 3. Create a branch for local development:: 56 | 57 | $ git checkout -b name-of-your-bugfix-or-feature 58 | 59 | Now you can make your changes locally. 60 | 61 | 4. When you're done making changes, check that your changes pass the tests:: 62 | 63 | $ make test 64 | 65 | 6. Commit your changes and push your branch to GitHub:: 66 | 67 | $ git add . 68 | $ git commit -m "Your detailed description of your changes." 69 | $ git push origin name-of-your-bugfix-or-feature 70 | 71 | 7. Submit a pull request through the GitHub website. 72 | 73 | Pull Request Guidelines 74 | ----------------------- 75 | 76 | Before you submit a pull request, check that it meets these guidelines: 77 | 78 | 1. The pull request should include tests. 79 | 2. If the pull request adds functionality, the docs should be updated. Put 80 | your new functionality into a function with a docstring, and add the 81 | feature to the list in README.md. -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:latest AS build-env 2 | ADD ./ /prom2click 3 | WORKDIR /prom2click 4 | RUN go mod tidy && CGO_ENABLED=0 go build -ldflags "-X main.GitCommit=${GIT_COMMIT}${GIT_DIRTY} -X main.VersionPrerelease=DEV" -o bin/prom2click 5 | 6 | FROM alpine 7 | RUN apk add -U tzdata 8 | RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime 9 | COPY --from=build-env /prom2click/bin/prom2click /usr/local/bin/prom2click 10 | RUN chmod +x /usr/local/bin/prom2click 11 | CMD ["prom2click"] 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # prom2click 2 | 3 | Prom2click is a Prometheus remote storage adapter for [Clickhouse](https://clickhouse.yandex/). This project is in the early stages, beta testers are welcome :). Consider it experimental - that said it is quite promising as a scalable and highly available remote storage for Prometheus. 4 | 5 | It's functional and writing metrics into Clickhouse in configurable batch sizes. Note that (currently) it is cpu hungry so you'll need a decent number of cores to sustain higher ingestion rates (eg. > hundreds of thousands/second). Also, it is missing some bits like doco, proper logging, tests and database error handling. 6 | 7 | If you've not heard of Clickhouse before it's a column oriented data store designed for real time analytic workloads on massive data sets (100's of tb+). It also happens to be pretty well suited for storing/retreiving time series data as it supports compression and has a Graphite type rollup on arbitrary tables/columns. 8 | 9 | 10 | ```console 11 | ./bin/prom2click --help 12 | Usage of ./bin/prom2click: 13 | -ch.batch int 14 | Clickhouse write batch size (n metrics). (default 8192) 15 | -ch.buffer int 16 | Maximum internal channel buffer size (n requests). (default 8192) 17 | -ch.db string 18 | The clickhouse database to write to. (default "metrics") 19 | -ch.dsn string 20 | The clickhouse server DSN to write to eg.tcp://host1:9000?username=user&password=qwerty&database=clicks&read_timeout=10&write_timeout=20&alt_hosts=host2:9000,host3:9000(see https://github.com/kshvakov/clickhouse). (default "tcp://127.0.0.1:9000?username=&password=&database=metrics&read_timeout=10&write_timeout=10&alt_hosts=") 21 | -ch.maxsamples int 22 | Maximum number of samples to return to Prometheus for a remote read request - the minimum accepted value is 50. Note: if you set this too low there can be issues displaying graphs in grafana. Increasing this will cause query times and memory utilization to grow. You'll probably need to experiment with this. (default 8192) 23 | -ch.minperiod int 24 | The minimum time range for Clickhouse time aggregation in seconds. (default 10) 25 | -ch.quantile float 26 | Quantile/Percentile for time series aggregation when the number of points exceeds ch.maxsamples. (default 0.75) 27 | -ch.table string 28 | The clickhouse table to write to. (default "samples") 29 | -log.format value 30 | Set the log target and format. Example: "logger:syslog?appname=bob&local=7" or "logger:stdout?json=true" (default "logger:stderr") 31 | -log.level value 32 | Only log messages with the given severity or above. Valid levels: [debug, info, warn, error, fatal] 33 | -version 34 | Version 35 | -web.address string 36 | Address to listen on for web endpoints. (default ":9201") 37 | -web.metrics string 38 | Address to listen on for metric requests. (default "/metrics") 39 | -web.timeout duration 40 | The timeout to use for HTTP requests and server shutdown. Defaults to 30s. (default 30s) 41 | -web.write string 42 | Address to listen on for remote write requests. (default "/write") 43 | ``` 44 | 45 | ## Getting started 46 | 47 | * [Install clickhouse](https://clickhouse.yandex/) 48 | * If you run ubuntu they have debs, otherwise.. well.. containers? (I'm running it in lxc with net=none on some Redhat based systems) 49 | 50 | * Configure Clickhouse graphite rollup settings in config.xml (bottom lines in [config.xml](https://github.com/s4z/prom2click/blob/master/config.xml)) 51 | 52 | * Goto [Tabix](http://ui.tabix.io/) for a quick and easy Clickhouse UI 53 | 54 | * Create clickhouse schema 55 | ```sql 56 | CREATE DATABASE IF NOT EXISTS metrics; 57 | CREATE TABLE IF NOT EXISTS metrics.samples 58 | ( 59 | date Date DEFAULT toDate(0), 60 | name String, 61 | tags Array(String), 62 | val Float64, 63 | ts DateTime, 64 | updated DateTime DEFAULT now() 65 | ) 66 | ENGINE = GraphiteMergeTree( 67 | date, (name, tags, ts), 8192, 'graphite_rollup' 68 | ); 69 | ``` 70 | * For a more resiliant setup you could setup shards, replicas and a distributed table 71 | * setup a Zookeeper cluster (or zetcd) 72 | * eg. for each clickhouse shard run two+ clickhouse servers and setup a ReplicatedGraphiteMergeTree on each with the same zk path and uniq replicas (eg. replace {replica} with the servers fqdn) 73 | * next create a distributed table that looks at the ReplicatedGraphiteMergeTrees 74 | * either define the {shard} and {replica} macros in your clickhouse server config or replace accordingly when you run the queries on each host 75 | * see: [Distributed](https://clickhouse.yandex/docs/en/table_engines/distributed.html) and [Replicated](https://clickhouse.yandex/docs/en/table_engines/replication.html) 76 | ```sql 77 | CREATE DATABASE IF NOT EXISTS metrics; 78 | CREATE TABLE IF NOT EXISTS metrics.samples 79 | ( 80 | date Date DEFAULT toDate(0), 81 | name String, 82 | tags Array(String), 83 | val Float64, 84 | ts DateTime, 85 | updated DateTime DEFAULT now() 86 | ) 87 | ENGINE = ReplicatedGraphiteMergeTree( 88 | '/clickhouse/tables/{shard}/metrics.samples', 89 | '{replica}', date, (name, tags, ts), 8192, 'graphite_rollup' 90 | ); 91 | 92 | CREATE TABLE IF NOT EXISTS metrics.dist 93 | ( 94 | date Date DEFAULT toDate(0), 95 | name String, 96 | tags Array(String), 97 | val Float64, 98 | ts DateTime, 99 | updated DateTime DEFAULT now() 100 | ) ENGINE = Distributed(metrics, metrics, samples, sipHash64(name)); 101 | ``` 102 | 103 | * Install/Configure [Grafana](https://grafana.com/) 104 | * (optional) Install the [Clickhouse Grafana Datasource](https://github.com/Vertamedia/clickhouse-grafana) Plugin 105 | ```console 106 | sudo grafana-cli plugins install vertamedia-clickhouse-datasource 107 | ``` 108 | * Install [Prometheus](https://prometheus.io/) and configure a remote read and write url 109 | ```yaml 110 | # Remote write configuration (for Graphite, OpenTSDB, InfluxDB or Clickhouse). 111 | remote_write: 112 | - url: "http://localhost:9201/write" 113 | remote_read: 114 | - url: "http://localhost:9201/read" 115 | 116 | ``` 117 | * Build prom2click and run it 118 | * Install go 1.21 above 119 | 120 | ```console 121 | $ go mod tidy 122 | $ go build -v -o bin/prom2click 123 | $ ./bin/prom2click 124 | ``` 125 | 126 | * Create a dashboard 127 | * This example was created with the Clickhouse datasource - you'll likely want to use the Prometheus data source though 128 | * Example template query 129 | ```sql 130 | SELECT DISTINCT(name) FROM metrics.samples 131 | ``` 132 | * Example query 133 | ```sql 134 | SELECT $timeSeries as t, median(val) FROM $table WHERE $timeFilter 135 | AND name = $metric GROUP BY t ORDER BY t 136 | ``` 137 | * Basic graph 138 | 139 | ![Alt text](./img/screen1.png "Dashboard Screen" ) 140 | 141 | ### Testing 142 | 143 | ``make test`` 144 | 145 | ..there are no tests (yet).. 146 | 147 | 148 | ### Misc Notes / Todo 149 | 150 | * add missing metrics (eg. read query metrics, nrows, failures, latency etc) 151 | * add a clickhouse metric exporter 152 | * possibly create a ticker to flush enqueued metrics at a constant rate 153 | * add logging support, remove prints 154 | * try the in-memory Buffer table engine to buffer writes 155 | * add proper db error handling 156 | * add tests 157 | -------------------------------------------------------------------------------- /config.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | warning 5 | /var/log/clickhouse-server/clickhouse-server.log 6 | /var/log/clickhouse-server/clickhouse-server.err.log 7 | 100M 8 | 10 9 | 10 | 8123 11 | 9000 12 | 9009 13 | shardnreplicanhostname 14 | 0.0.0.0 15 | 8192 16 | 3 17 | 100 18 | 8589934 19 | 5368709 20 | /var/lib/clickhouse/ 21 | /var/lib/clickhouse/tmp/ 22 | users.xml 23 | default 24 | default 25 | UTC 26 | 27 | 30 | 31 | 32 | 33 | 34 | 1 35 | 36 | 37 | true 38 | 39 | shard1replica1host 40 | 9000 41 | 42 | 43 | shard1replica2host 44 | 9000 45 | 46 | 47 | 48 | 49 | 1 50 | 51 | 52 | true 53 | 54 | shard2replica1host 55 | 9000 56 | 57 | 58 | shard2replica2host 59 | 9000 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | zkhost1 68 | 2181 69 | 70 | 71 | zkhost2 72 | 2181 73 | 74 | 75 | zkhost3 76 | 2181 77 | 78 | 79 | 80 | 81 | 3600 82 | 83 | system 84 | query_log
85 | 7500 86 | 87 | 88 | system 89 | part_log
90 | 91 | 7500 92 | 93 | *_dictionary.xml 94 | 95 | 96 | 97 | /clickhouse/task_queue 98 | 99 | 100 | 102 | 103 | tags 104 | ts 105 | val 106 | updated 107 | 108 | avg 109 | 110 | 0 111 | 10 112 | 113 | 114 | 86400 115 | 30 116 | 117 | 118 | 172800 119 | 300 120 | 121 | 122 | 123 | 124 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module prom2click 2 | -------------------------------------------------------------------------------- /img/screen1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iyacontrol/prom2click/f21211aef632c46c43a8eeddd61272d2906fd069/img/screen1.png -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | "time" 8 | 9 | "go.uber.org/zap" 10 | ) 11 | 12 | // a lot of this borrows directly from: 13 | // https://github.com/prometheus/prometheus/blob/master/documentation/examples/remote_storage/remote_storage_adapter/main.go 14 | 15 | type config struct { 16 | //tcp://host1:9000?username=user&password=qwerty&database=clicks&read_timeout=10&write_timeout=20&alt_hosts=host2:9000,host3:9000 17 | ChDSN string 18 | ChDB string 19 | ChTable string 20 | ChBatch int 21 | ChanSize int 22 | CHQuantile float64 23 | CHMaxSamples int 24 | CHMinPeriod int 25 | HTTPTimeout time.Duration 26 | HTTPAddr string 27 | HTTPWritePath string 28 | HTTPMetricsPath string 29 | } 30 | 31 | var ( 32 | versionFlag bool 33 | debug bool 34 | ) 35 | 36 | func main() { 37 | conf := parseFlags() 38 | 39 | if versionFlag { 40 | fmt.Println("Git Commit:", GitCommit) 41 | fmt.Println("Version:", Version) 42 | if VersionPrerelease != "" { 43 | fmt.Println("Version PreRelease:", VersionPrerelease) 44 | } 45 | return 46 | } 47 | 48 | var logger *zap.Logger 49 | if debug { 50 | logger, _ = zap.NewDevelopment() 51 | } else { 52 | logger, _ = zap.NewProduction() 53 | } 54 | 55 | defer logger.Sync() // flushes buffer, if any 56 | sugar := logger.Sugar() 57 | 58 | sugar.Info("Starting up..") 59 | 60 | srv, err := NewP2CServer(conf, sugar) 61 | if err != nil { 62 | sugar.Fatalf("could not create server: %s\n", err.Error()) 63 | } 64 | err = srv.Start() 65 | if err != nil { 66 | sugar.Fatalf("http server returned error: %s\n", err.Error()) 67 | } 68 | sugar.Info("Shutting down..") 69 | srv.Shutdown() 70 | sugar.Info("Exiting..") 71 | } 72 | 73 | func parseFlags() *config { 74 | cfg := new(config) 75 | 76 | // print version? 77 | flag.BoolVar(&versionFlag, "version", false, "Version") 78 | 79 | // turn on debug? 80 | flag.BoolVar(&debug, "debug", false, "turn on debug mode") 81 | 82 | // clickhouse dsn 83 | ddsn := "tcp://127.0.0.1:9000?username=&password=&database=metrics&" + 84 | "read_timeout=10&write_timeout=10&alt_hosts=" 85 | flag.StringVar(&cfg.ChDSN, "ch.dsn", ddsn, 86 | "The clickhouse server DSN to write to eg."+ 87 | "tcp://host1:9000?username=user&password=qwerty&database=clicks&"+ 88 | "read_timeout=10&write_timeout=20&alt_hosts=host2:9000,host3:9000"+ 89 | "(see https://github.com/kshvakov/clickhouse).", 90 | ) 91 | 92 | // clickhouse db 93 | flag.StringVar(&cfg.ChDB, "ch.db", "metrics", 94 | "The clickhouse database to write to.", 95 | ) 96 | 97 | // clickhouse table 98 | flag.StringVar(&cfg.ChTable, "ch.table", "samples", 99 | "The clickhouse table to write to.", 100 | ) 101 | 102 | // clickhouse insertion batch size 103 | flag.IntVar(&cfg.ChBatch, "ch.batch", 32768, 104 | "Clickhouse write batch size (n metrics).", 105 | ) 106 | 107 | // channel buffer size between http server => clickhouse writer(s) 108 | flag.IntVar(&cfg.ChanSize, "ch.buffer", 32768, 109 | "Maximum internal channel buffer size (n requests).", 110 | ) 111 | 112 | // quantile (eg. 0.9 for 90th) for aggregation of timeseries values from CH 113 | flag.Float64Var(&cfg.CHQuantile, "ch.quantile", 0.75, 114 | "Quantile/Percentile for time series aggregation when the number "+ 115 | "of points exceeds ch.maxsamples.", 116 | ) 117 | 118 | // maximum number of samples to return 119 | // todo: fixup strings.. yuck. 120 | flag.IntVar(&cfg.CHMaxSamples, "ch.maxsamples", 8192, 121 | "Maximum number of samples to return to Prometheus for a remote read "+ 122 | "request - the minimum accepted value is 50. "+ 123 | "Note: if you set this too low there can be issues displaying graphs in grafana. "+ 124 | "Increasing this will cause query times and memory utilization to grow. You'll "+ 125 | "probably need to experiment with this.", 126 | ) 127 | // need to ensure this isn't 0 - divide by 0.. 128 | if cfg.CHMaxSamples < 50 { 129 | fmt.Printf("Error: invalid ch.maxsamples of %d - minimum is 50\n", cfg.CHMaxSamples) 130 | os.Exit(1) 131 | } 132 | 133 | // http shutdown and request timeout 134 | flag.IntVar(&cfg.CHMinPeriod, "ch.minperiod", 10, 135 | "The minimum time range for Clickhouse time aggregation in seconds.", 136 | ) 137 | 138 | // http listen address 139 | flag.StringVar(&cfg.HTTPAddr, "web.address", ":9201", 140 | "Address to listen on for web endpoints.", 141 | ) 142 | 143 | // http prometheus remote write endpoint 144 | flag.StringVar(&cfg.HTTPWritePath, "web.write", "/write", 145 | "Address to listen on for remote write requests.", 146 | ) 147 | 148 | // http prometheus metrics endpoint 149 | flag.StringVar(&cfg.HTTPMetricsPath, "web.metrics", "/metrics", 150 | "Address to listen on for metric requests.", 151 | ) 152 | 153 | // http shutdown and request timeout 154 | flag.DurationVar(&cfg.HTTPTimeout, "web.timeout", 30*time.Second, 155 | "The timeout to use for HTTP requests and server shutdown. Defaults to 30s.", 156 | ) 157 | 158 | flag.Parse() 159 | 160 | return cfg 161 | } 162 | -------------------------------------------------------------------------------- /reader.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "database/sql" 6 | "errors" 7 | "fmt" 8 | "strings" 9 | 10 | clickhouse "github.com/ClickHouse/clickhouse-go/v2" 11 | "github.com/prometheus/common/model" 12 | "github.com/prometheus/prometheus/prompb" 13 | "go.uber.org/zap" 14 | ) 15 | 16 | var readerContent = []interface{}{"component", "reader"} 17 | 18 | type p2cReader struct { 19 | conf *config 20 | db *sql.DB 21 | logger *zap.SugaredLogger 22 | } 23 | 24 | // getTimePeriod return select and where SQL chunks relating to the time period -or- error 25 | func (r *p2cReader) getTimePeriod(query *prompb.Query) (string, string, error) { 26 | 27 | var tselSQL = "SELECT COUNT() AS CNT, (intDiv(toUInt32(ts), %d) * %d) * 1000 as t" 28 | var twhereSQL = "WHERE date >= toDate(%d) AND ts >= toDateTime(%d) AND ts <= toDateTime(%d)" 29 | var err error 30 | tstart := query.StartTimestampMs / 1000 31 | tend := query.EndTimestampMs / 1000 32 | 33 | // valid time period 34 | if tend < tstart { 35 | err = errors.New("Start time is after end time") 36 | return "", "", err 37 | } 38 | 39 | // need time period in seconds 40 | tperiod := tend - tstart 41 | 42 | // need to split time period into - also, don't divide by zero 43 | if r.conf.CHMaxSamples < 1 { 44 | err = fmt.Errorf(fmt.Sprintf("Invalid CHMaxSamples: %d", r.conf.CHMaxSamples)) 45 | return "", "", err 46 | } 47 | taggr := tperiod / int64(r.conf.CHMaxSamples) 48 | if taggr < int64(r.conf.CHMinPeriod) { 49 | taggr = int64(r.conf.CHMinPeriod) 50 | } 51 | 52 | selectSQL := fmt.Sprintf(tselSQL, taggr, taggr) 53 | whereSQL := fmt.Sprintf(twhereSQL, tstart, tstart, tend) 54 | 55 | return selectSQL, whereSQL, nil 56 | } 57 | 58 | func (r *p2cReader) getSQL(query *prompb.Query) (string, error) { 59 | // time related select sql, where sql chunks 60 | tselectSQL, twhereSQL, err := r.getTimePeriod(query) 61 | if err != nil { 62 | return "", err 63 | } 64 | 65 | // match sql chunk 66 | var mwhereSQL []string 67 | // build an sql statement chunk for each matcher in the query 68 | // yeah, this is a bit ugly.. 69 | for _, m := range query.Matchers { 70 | // __name__ is handled specially - match it directly 71 | // as it is stored in the name column (it's also in tags as __name__) 72 | // note to self: add name to index.. otherwise this will be slow.. 73 | if m.Name == model.MetricNameLabel { 74 | var whereAdd string 75 | switch m.Type { 76 | case prompb.LabelMatcher_EQ: 77 | whereAdd = fmt.Sprintf(` name='%s' `, strings.Replace(m.Value, `'`, `\'`, -1)) 78 | case prompb.LabelMatcher_NEQ: 79 | whereAdd = fmt.Sprintf(` name!='%s' `, strings.Replace(m.Value, `'`, `\'`, -1)) 80 | case prompb.LabelMatcher_RE: 81 | whereAdd = fmt.Sprintf(` match(name, %s) = 1 `, strings.Replace(m.Value, `/`, `\/`, -1)) 82 | case prompb.LabelMatcher_NRE: 83 | whereAdd = fmt.Sprintf(` match(name, %s) = 0 `, strings.Replace(m.Value, `/`, `\/`, -1)) 84 | } 85 | mwhereSQL = append(mwhereSQL, whereAdd) 86 | continue 87 | } 88 | 89 | switch m.Type { 90 | case prompb.LabelMatcher_EQ: 91 | var insql bytes.Buffer 92 | asql := "arrayExists(x -> x IN (%s), tags) = 1" 93 | // value appears to be | sep'd for multiple matches 94 | for i, val := range strings.Split(m.Value, "|") { 95 | if len(val) < 1 { 96 | continue 97 | } 98 | if i == 0 { 99 | istr := fmt.Sprintf(`'%s=%s' `, m.Name, strings.Replace(val, `'`, `\'`, -1)) 100 | insql.WriteString(istr) 101 | } else { 102 | istr := fmt.Sprintf(`,'%s=%s' `, m.Name, strings.Replace(val, `'`, `\'`, -1)) 103 | insql.WriteString(istr) 104 | } 105 | } 106 | wstr := fmt.Sprintf(asql, insql.String()) 107 | mwhereSQL = append(mwhereSQL, wstr) 108 | 109 | case prompb.LabelMatcher_NEQ: 110 | var insql bytes.Buffer 111 | asql := "arrayExists(x -> x IN (%s), tags) = 0" 112 | // value appears to be | sep'd for multiple matches 113 | for i, val := range strings.Split(m.Value, "|") { 114 | if len(val) < 1 { 115 | continue 116 | } 117 | if i == 0 { 118 | istr := fmt.Sprintf(`'%s=%s' `, m.Name, strings.Replace(val, `'`, `\'`, -1)) 119 | insql.WriteString(istr) 120 | } else { 121 | istr := fmt.Sprintf(`,'%s=%s' `, m.Name, strings.Replace(val, `'`, `\'`, -1)) 122 | insql.WriteString(istr) 123 | } 124 | } 125 | wstr := fmt.Sprintf(asql, insql.String()) 126 | mwhereSQL = append(mwhereSQL, wstr) 127 | 128 | case prompb.LabelMatcher_RE: 129 | asql := `arrayExists(x -> 1 == match(x, '^%s=%s'),tags) = 1` 130 | // we can't have ^ in the regexp since keys are stored in arrays of key=value 131 | if strings.HasPrefix(m.Value, "^") { 132 | val := strings.Replace(m.Value, "^", "", 1) 133 | val = strings.Replace(val, `/`, `\/`, -1) 134 | mwhereSQL = append(mwhereSQL, fmt.Sprintf(asql, m.Name, val)) 135 | } else { 136 | val := strings.Replace(m.Value, `/`, `\/`, -1) 137 | mwhereSQL = append(mwhereSQL, fmt.Sprintf(asql, m.Name, val)) 138 | } 139 | 140 | case prompb.LabelMatcher_NRE: 141 | asql := `arrayExists(x -> 1 == match(x, '^%s=%s'),tags) = 0` 142 | if strings.HasPrefix(m.Value, "^") { 143 | val := strings.Replace(m.Value, "^", "", 1) 144 | val = strings.Replace(val, `/`, `\/`, -1) 145 | mwhereSQL = append(mwhereSQL, fmt.Sprintf(asql, m.Name, val)) 146 | } else { 147 | val := strings.Replace(m.Value, `/`, `\/`, -1) 148 | mwhereSQL = append(mwhereSQL, fmt.Sprintf(asql, m.Name, val)) 149 | } 150 | } 151 | } 152 | 153 | // put select and where together with group by etc 154 | tempSQL := "%s, name, tags, quantile(%f)(val) as value FROM %s.%s %s AND %s GROUP BY t, name, tags ORDER BY t" 155 | sql := fmt.Sprintf(tempSQL, tselectSQL, r.conf.CHQuantile, r.conf.ChDB, r.conf.ChTable, twhereSQL, 156 | strings.Join(mwhereSQL, " AND ")) 157 | return sql, nil 158 | } 159 | 160 | func NewP2CReader(conf *config, sugar *zap.SugaredLogger) (*p2cReader, error) { 161 | var err error 162 | r := new(p2cReader) 163 | r.conf = conf 164 | r.logger = sugar 165 | r.db, err = sql.Open("clickhouse", r.conf.ChDSN) 166 | if err != nil { 167 | r.logger.With(readerContent...).Errorf("connecting to clickhouse: %s", err.Error()) 168 | return r, err 169 | } 170 | 171 | if err := r.db.Ping(); err != nil { 172 | if exception, ok := err.(*clickhouse.Exception); ok { 173 | r.logger.With(readerContent...).Errorf("[%d] %s \n%s\n", exception.Code, exception.Message, exception.StackTrace) 174 | } else { 175 | r.logger.With(readerContent...).Error(err.Error()) 176 | } 177 | return r, err 178 | } 179 | 180 | return r, nil 181 | } 182 | 183 | func (r *p2cReader) Read(req *prompb.ReadRequest) (*prompb.ReadResponse, error) { 184 | var err error 185 | var sqlStr string 186 | var rows *sql.Rows 187 | 188 | resp := prompb.ReadResponse{ 189 | Results: []*prompb.QueryResult{ 190 | {Timeseries: make([]*prompb.TimeSeries, 0, 0)}, 191 | }, 192 | } 193 | // need to map tags to timeseries to record samples 194 | var tsres = make(map[string]*prompb.TimeSeries) 195 | 196 | // for Debugfging/figuring out query format/etc 197 | rcount := 0 198 | for _, q := range req.Queries { 199 | // remove me.. 200 | r.logger.With(readerContent...).Debugf("\nquery: start: %d, end: %d", q.StartTimestampMs, q.EndTimestampMs) 201 | 202 | // get the select sql 203 | sqlStr, err = r.getSQL(q) 204 | r.logger.With(readerContent...).Debugf("query: running sql: %s", sqlStr) 205 | if err != nil { 206 | r.logger.With(readerContent...).Errorf("reader: getSQL: %s", err.Error()) 207 | return &resp, err 208 | } 209 | 210 | // get the select sql 211 | if err != nil { 212 | r.logger.With(readerContent...).Errorf("reader: getSQL: %s", err.Error()) 213 | return &resp, err 214 | } 215 | 216 | // todo: metrics on number of errors, rows, selects, timings, etc 217 | rows, err = r.db.Query(sqlStr) 218 | if err != nil { 219 | r.logger.With(readerContent...).Errorf("query failed: %s", sqlStr) 220 | r.logger.With(readerContent...).Errorf("query error: %s", err) 221 | return &resp, err 222 | } 223 | 224 | // build map of timeseries from sql result 225 | 226 | for rows.Next() { 227 | rcount++ 228 | var ( 229 | cnt int 230 | t int64 231 | name string 232 | tags []string 233 | value float64 234 | ) 235 | if err = rows.Scan(&cnt, &t, &name, &tags, &value); err != nil { 236 | r.logger.With(readerContent...).Errorf("scan: %s", err.Error()) 237 | } 238 | // remove this.. 239 | //fmt.Printf(fmt.Sprintf("%d,%d,%s,%s,%f\n", cnt, t, name, strings.Join(tags, ":"), value)) 240 | 241 | // borrowed from influx remote storage adapter - array sep 242 | key := strings.Join(tags, "\xff") 243 | ts, ok := tsres[key] 244 | if !ok { 245 | ts = &prompb.TimeSeries{ 246 | Labels: makeLabels(tags), 247 | } 248 | tsres[key] = ts 249 | } 250 | ts.Samples = append(ts.Samples, prompb.Sample{ 251 | Value: float64(value), 252 | Timestamp: t, 253 | }) 254 | } 255 | } 256 | 257 | // now add results to response 258 | for _, ts := range tsres { 259 | resp.Results[0].Timeseries = append(resp.Results[0].Timeseries, ts) 260 | } 261 | 262 | r.logger.With(readerContent...).Debugf("query: returning %d rows for %d queries", rcount, len(req.Queries)) 263 | 264 | return &resp, nil 265 | 266 | } 267 | 268 | func makeLabels(tags []string) []prompb.Label { 269 | lpairs := make([]prompb.Label, 0, len(tags)) 270 | // (currently) writer includes __name__ in tags so no need to add it here 271 | // may change this to save space later.. 272 | for _, tag := range tags { 273 | vals := strings.SplitN(tag, "=", 2) 274 | if len(vals) != 2 { 275 | fmt.Printf("Error unpacking tag key/val: %s\n", tag) 276 | continue 277 | } 278 | if vals[1] == "" { 279 | continue 280 | } 281 | lpairs = append(lpairs, prompb.Label{ 282 | Name: vals[0], 283 | Value: vals[1], 284 | }) 285 | } 286 | return lpairs 287 | } 288 | -------------------------------------------------------------------------------- /schema.sql: -------------------------------------------------------------------------------- 1 | # note: replace {shard} and {replica} and run on each server 2 | DROP TABLE IF EXISTS metrics.samples; 3 | CREATE TABLE IF NOT EXISTS metrics.samples 4 | ( 5 | date Date DEFAULT toDate(0), 6 | name String, 7 | tags Array(String), 8 | val Float64, 9 | ts DateTime, 10 | updated DateTime DEFAULT now() 11 | ) 12 | ENGINE = ReplicatedGraphiteMergeTree( 13 | '/clickhouse/tables/{shard}/metrics.samples', 14 | '{replica}', date, (name, tags, ts), 8192, 'graphite_rollup' 15 | ); 16 | 17 | DROP TABLE IF EXISTS metrics.dist; 18 | CREATE TABLE IF NOT EXISTS metrics.dist 19 | ( 20 | date Date DEFAULT toDate(0), 21 | name String, 22 | tags Array(String), 23 | val Float64, 24 | ts DateTime, 25 | updated DateTime DEFAULT now() 26 | ) ENGINE = Distributed(metrics, metrics, samples, sipHash64(name)); 27 | -------------------------------------------------------------------------------- /srv.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "net/http" 7 | "time" 8 | 9 | "github.com/golang/protobuf/proto" 10 | "github.com/golang/snappy" 11 | "github.com/prometheus/client_golang/prometheus" 12 | "github.com/prometheus/common/model" 13 | "github.com/prometheus/prometheus/prompb" 14 | 15 | //"github.com/prometheus/prometheus/storage/remote" 16 | "go.uber.org/zap" 17 | graceful "gopkg.in/tylerb/graceful.v1" 18 | ) 19 | 20 | type p2cRequest struct { 21 | name string 22 | tags []string 23 | val float64 24 | ts time.Time 25 | } 26 | 27 | type p2cServer struct { 28 | requests chan *p2cRequest 29 | mux *http.ServeMux 30 | conf *config 31 | writer *p2cWriter 32 | reader *p2cReader 33 | rx prometheus.Counter 34 | logger *zap.SugaredLogger 35 | } 36 | 37 | func NewP2CServer(conf *config, sugar *zap.SugaredLogger) (*p2cServer, error) { 38 | var err error 39 | c := new(p2cServer) 40 | c.requests = make(chan *p2cRequest, conf.ChanSize) 41 | c.mux = http.NewServeMux() 42 | c.conf = conf 43 | c.logger = sugar 44 | 45 | c.writer, err = NewP2CWriter(conf, c.requests, sugar) 46 | if err != nil { 47 | c.logger.Errorf("creating clickhouse writer: %s\n", err.Error()) 48 | return c, err 49 | } 50 | 51 | c.reader, err = NewP2CReader(conf, sugar) 52 | if err != nil { 53 | c.logger.Errorf("creating clickhouse reader: %s\n", err.Error()) 54 | return c, err 55 | } 56 | 57 | c.rx = prometheus.NewCounter( 58 | prometheus.CounterOpts{ 59 | Name: "received_samples_total", 60 | Help: "Total number of received samples.", 61 | }, 62 | ) 63 | prometheus.MustRegister(c.rx) 64 | 65 | c.mux.HandleFunc(c.conf.HTTPWritePath, func(w http.ResponseWriter, r *http.Request) { 66 | compressed, err := ioutil.ReadAll(r.Body) 67 | if err != nil { 68 | http.Error(w, err.Error(), http.StatusInternalServerError) 69 | return 70 | } 71 | 72 | reqBuf, err := snappy.Decode(nil, compressed) 73 | if err != nil { 74 | http.Error(w, err.Error(), http.StatusBadRequest) 75 | return 76 | } 77 | 78 | var req prompb.WriteRequest 79 | if err := proto.Unmarshal(reqBuf, &req); err != nil { 80 | http.Error(w, err.Error(), http.StatusBadRequest) 81 | return 82 | } 83 | 84 | c.process(req) 85 | }) 86 | 87 | c.mux.HandleFunc("/read", func(w http.ResponseWriter, r *http.Request) { 88 | compressed, err := ioutil.ReadAll(r.Body) 89 | if err != nil { 90 | http.Error(w, err.Error(), http.StatusInternalServerError) 91 | return 92 | } 93 | 94 | reqBuf, err := snappy.Decode(nil, compressed) 95 | if err != nil { 96 | http.Error(w, err.Error(), http.StatusBadRequest) 97 | return 98 | } 99 | 100 | var req prompb.ReadRequest 101 | if err := proto.Unmarshal(reqBuf, &req); err != nil { 102 | http.Error(w, err.Error(), http.StatusBadRequest) 103 | return 104 | } 105 | 106 | var resp *prompb.ReadResponse 107 | resp, err = c.reader.Read(&req) 108 | if err != nil { 109 | http.Error(w, err.Error(), http.StatusInternalServerError) 110 | return 111 | } 112 | 113 | data, err := proto.Marshal(resp) 114 | if err != nil { 115 | http.Error(w, err.Error(), http.StatusInternalServerError) 116 | return 117 | } 118 | 119 | w.Header().Set("Content-Type", "application/x-protobuf") 120 | w.Header().Set("Content-Encoding", "snappy") 121 | 122 | compressed = snappy.Encode(nil, data) 123 | if _, err := w.Write(compressed); err != nil { 124 | http.Error(w, err.Error(), http.StatusInternalServerError) 125 | return 126 | } 127 | }) 128 | 129 | // c.mux.Handle(c.conf.HTTPMetricsPath, prometheus.InstrumentHandler( 130 | // c.conf.HTTPMetricsPath, prometheus.UninstrumentedHandler(), 131 | // )) 132 | 133 | return c, nil 134 | } 135 | 136 | func (c *p2cServer) process(req prompb.WriteRequest) { 137 | for _, series := range req.Timeseries { 138 | c.rx.Add(float64(len(series.Samples))) 139 | var ( 140 | name string 141 | tags []string 142 | ) 143 | 144 | for _, label := range series.Labels { 145 | if model.LabelName(label.Name) == model.MetricNameLabel { 146 | name = label.Value 147 | } 148 | // store tags in = format 149 | // allows for has(tags, "key=val") searches 150 | // probably impossible/difficult to do regex searches on tags 151 | t := fmt.Sprintf("%s=%s", label.Name, label.Value) 152 | tags = append(tags, t) 153 | } 154 | 155 | for _, sample := range series.Samples { 156 | p2c := new(p2cRequest) 157 | p2c.name = name 158 | p2c.ts = time.Unix(sample.Timestamp/1000, 0) 159 | p2c.val = sample.Value 160 | p2c.tags = tags 161 | c.requests <- p2c 162 | } 163 | 164 | } 165 | } 166 | 167 | func (c *p2cServer) Start() error { 168 | c.logger.Info("HTTP server starting...") 169 | c.writer.Start() 170 | return graceful.RunWithErr(c.conf.HTTPAddr, c.conf.HTTPTimeout, c.mux) 171 | } 172 | 173 | func (c *p2cServer) Shutdown() { 174 | close(c.requests) 175 | c.writer.Wait() 176 | 177 | wchan := make(chan struct{}) 178 | go func() { 179 | c.writer.Wait() 180 | close(wchan) 181 | }() 182 | 183 | select { 184 | case <-wchan: 185 | c.logger.Info("Writer shutdown cleanly..") 186 | // All done! 187 | case <-time.After(10 * time.Second): 188 | c.logger.Info("Writer shutdown timed out, samples will be lost..") 189 | } 190 | 191 | } 192 | -------------------------------------------------------------------------------- /version.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // The git commit that was compiled. This will be filled in by the compiler. 4 | var GitCommit string 5 | 6 | // The main version number that is being run at the moment. 7 | const Version = "0.1.0" 8 | 9 | // A pre-release marker for the version. If this is "" (empty string) 10 | // then it means that it is a final release. Otherwise, this is a pre-release 11 | // such as "dev" (in development) 12 | var VersionPrerelease = "" 13 | -------------------------------------------------------------------------------- /writer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "sort" 7 | "sync" 8 | "time" 9 | 10 | clickhouse "github.com/ClickHouse/clickhouse-go/v2" 11 | "github.com/prometheus/client_golang/prometheus" 12 | "go.uber.org/zap" 13 | ) 14 | 15 | var insertSQL = `INSERT INTO %s.%s 16 | (date, name, tags, val, ts) 17 | VALUES (?, ?, ?, ?, ?)` 18 | 19 | var writerContent = []interface{}{"component", "writer"} 20 | 21 | type p2cWriter struct { 22 | conf *config 23 | requests chan *p2cRequest 24 | wg sync.WaitGroup 25 | db *sql.DB 26 | tx prometheus.Counter 27 | ko prometheus.Counter 28 | test prometheus.Counter 29 | timings prometheus.Histogram 30 | 31 | logger *zap.SugaredLogger 32 | } 33 | 34 | func NewP2CWriter(conf *config, reqs chan *p2cRequest, sugar *zap.SugaredLogger) (*p2cWriter, error) { 35 | var err error 36 | w := new(p2cWriter) 37 | w.conf = conf 38 | w.requests = reqs 39 | w.logger = sugar 40 | w.db, err = sql.Open("clickhouse", w.conf.ChDSN) 41 | if err != nil { 42 | w.logger.With(writerContent...).Errorf("connecting to clickhouse: %s", err.Error()) 43 | return w, err 44 | } 45 | 46 | if err := w.db.Ping(); err != nil { 47 | if exception, ok := err.(*clickhouse.Exception); ok { 48 | w.logger.With(writerContent...).Errorf("[%d] %s \n%s\n", exception.Code, exception.Message, exception.StackTrace) 49 | } else { 50 | w.logger.With(writerContent...).Error(err.Error()) 51 | } 52 | return w, err 53 | } 54 | 55 | w.tx = prometheus.NewCounter( 56 | prometheus.CounterOpts{ 57 | Name: "sent_samples_total", 58 | Help: "Total number of processed samples sent to remote storage.", 59 | }, 60 | ) 61 | 62 | w.ko = prometheus.NewCounter( 63 | prometheus.CounterOpts{ 64 | Name: "failed_samples_total", 65 | Help: "Total number of processed samples which failed on send to remote storage.", 66 | }, 67 | ) 68 | 69 | w.test = prometheus.NewCounter( 70 | prometheus.CounterOpts{ 71 | Name: "prometheus_remote_storage_sent_batch_duration_seconds_bucket_test", 72 | Help: "Test metric to ensure backfilled metrics are readable via prometheus.", 73 | }, 74 | ) 75 | 76 | w.timings = prometheus.NewHistogram( 77 | prometheus.HistogramOpts{ 78 | Name: "sent_batch_duration_seconds", 79 | Help: "Duration of sample batch send calls to the remote storage.", 80 | Buckets: prometheus.DefBuckets, 81 | }, 82 | ) 83 | prometheus.MustRegister(w.tx) 84 | prometheus.MustRegister(w.ko) 85 | prometheus.MustRegister(w.test) 86 | prometheus.MustRegister(w.timings) 87 | 88 | return w, nil 89 | } 90 | 91 | func (w *p2cWriter) Start() { 92 | 93 | go func() { 94 | w.wg.Add(1) 95 | w.logger.With(writerContent...).Info("Writer starting..") 96 | sql := fmt.Sprintf(insertSQL, w.conf.ChDB, w.conf.ChTable) 97 | ok := true 98 | for ok { 99 | w.test.Add(1) 100 | // get next batch of requests 101 | var reqs []*p2cRequest 102 | 103 | tstart := time.Now() 104 | for i := 0; i < w.conf.ChBatch; i++ { 105 | var req *p2cRequest 106 | // get requet and also check if channel is closed 107 | req, ok = <-w.requests 108 | if !ok { 109 | w.logger.With(writerContent...).Info("Writer stopping..") 110 | break 111 | } 112 | reqs = append(reqs, req) 113 | } 114 | 115 | // ensure we have something to send.. 116 | nmetrics := len(reqs) 117 | if nmetrics < 1 { 118 | continue 119 | } 120 | 121 | // post them to db all at once 122 | tx, err := w.db.Begin() 123 | if err != nil { 124 | w.logger.With(writerContent...).Errorf("begin transaction: %s", err.Error()) 125 | w.ko.Add(1.0) 126 | continue 127 | } 128 | 129 | // build statements 130 | smt, err := tx.Prepare(sql) 131 | for _, req := range reqs { 132 | if err != nil { 133 | w.logger.With(writerContent...).Errorf("prepare statement: %s", err.Error()) 134 | w.ko.Add(1.0) 135 | continue 136 | } 137 | 138 | // ensure tags are inserted in the same order each time 139 | // possibly/probably impacts indexing? 140 | sort.Strings(req.tags) 141 | _, err = smt.Exec(req.ts, req.name, req.tags, 142 | req.val, req.ts) 143 | 144 | if err != nil { 145 | w.logger.With(writerContent...).Errorf("statement exec: %s", err.Error()) 146 | w.ko.Add(1.0) 147 | } 148 | } 149 | 150 | // commit and record metrics 151 | if err = tx.Commit(); err != nil { 152 | w.logger.With(writerContent...).Errorf("commit failed: %s", err.Error()) 153 | w.ko.Add(1.0) 154 | } else { 155 | w.tx.Add(float64(nmetrics)) 156 | w.timings.Observe(float64(time.Since(tstart))) 157 | } 158 | 159 | } 160 | w.logger.With(writerContent...).Info("Writer stopped..") 161 | w.wg.Done() 162 | }() 163 | } 164 | 165 | func (w *p2cWriter) Wait() { 166 | w.wg.Wait() 167 | } 168 | --------------------------------------------------------------------------------