├── .gitignore ├── LICENSE ├── NOTICE ├── README.md ├── cfg.example.json ├── control ├── g ├── cfg.go ├── g.go └── git.go ├── graph └── graph.go ├── http ├── api.go ├── common.go ├── grafana.go ├── graph_http.go ├── http.go └── proc_http.go ├── main.go ├── proc └── proc.go ├── scripts ├── info ├── info.py ├── last ├── last_raw ├── query └── query.py └── test └── debug /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | 26 | *.swp 27 | *.swo 28 | *.log 29 | .idea 30 | .DS_Store 31 | /var 32 | /query 33 | 34 | *.txt 35 | /falcon-query* 36 | /release 37 | g/git.go 38 | cfg.json -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Open-Falcon 2 | 3 | Copyright (c) 2014-2015 Xiaomi, Inc. All Rights Reserved. 4 | 5 | This product is licensed to you under the Apache License, Version 2.0 (the "License"). 6 | You may not use this product except in compliance with the License. 7 | 8 | This product may include a number of subcomponents with separate copyright notices 9 | and license terms. Your use of these subcomponents is subject to the terms and 10 | conditions of the subcomponent's license, as noted in the LICENSE file. 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | 3 | Query面向终端用户，收到查询请求后，根据一致性哈希算法，会去相应的Graph里面，查询不同metric的数据，汇总后统一返回给用户。 4 | 5 | ## 查询历史数据 6 | 查询过去一段时间内的历史数据，使用接口 `HTTP POST /graph/history`。该接口不能查询最新上报的两个数据点。一个python例子，如下 7 | 8 | ```python 9 | #!-*- coding:utf8 -*- 10 | 11 | import requests 12 | import time 13 | import json 14 | 15 | end = int(time.time()) 16 | start = end - 3600 #查询过去一小时的数据 17 | 18 | d = { 19 | "start": start, 20 | "end": end, 21 | "cf": "AVERAGE", 22 | "endpoint_counters": [ 23 | { 24 | "endpoint": "host1", 25 | "counter": "cpu.idle", 26 | }, 27 | { 28 | "endpoint": "host1", 29 | "counter": "load.1min", 30 | }, 31 | ], 32 | } 33 | 34 | url = "http://127.0.0.1:9966/graph/history" 35 | r = requests.post(url, data=json.dumps(d)) 36 | print r.text 37 | 38 | ``` 39 | 40 | 其中cf的值可以为：AVERAGE、MAX、MIN ，具体可以参考RRDtool的相关概念 41 | 42 | ## 查询最新上报的数据 43 | 查询最新上报的一个数据点，使用接口`HTTP POST /graph/last`。一个bash的例子，如下 44 | 45 | ```bash 46 | #!/bin/bash 47 | if [ $# != 2 ];then 48 | printf "format:./last \"endpoint\" \"counter\"\n" 49 | exit 1 50 | fi 51 | 52 | # args 53 | endpoint=$1 54 | counter=$2 55 | 56 | # form request body 57 | req="[{\"endpoint\":\"$endpoint\", \"counter\":\"$counter\"}]" 58 | 59 | # request 60 | url="http://127.0.0.1:9966/graph/last" 61 | curl -s -X POST -d "$req" "$url" | python -m json.tool 62 | 63 | ``` 64 | 65 | ## 源码编译 66 | 注意: 请首先更新common模块 67 | 68 | ```bash 69 | # download source 70 | cd $GOPATH/src/github.com/open-falcon 71 | git clone https://github.com/open-falcon/query.git # or use git pull to update query 72 | 73 | # update dependencies: open-falcon/common, toolkits/consistent, toolkits/pool 74 | cd query 75 | go get ./... 76 | 77 | # compile and pack 78 | ./control build 79 | ./control pack 80 | ``` 81 | 最后一步会pack出一个tar.gz的安装包，拿着这个包去部署服务即可。你也可以在[这里](https://github.com/open-falcon/query/releases)，下载最新发布的代码。 82 | 83 | ## 服务部署 84 | 服务部署，包括配置修改、启动服务、检验服务、停止服务等。这之前，需要将安装包解压到服务的部署目录下。 85 | 86 | ```bash 87 | # 修改配置, 配置项含义见下文, 注意graph集群的配置 88 | mv cfg.example.json cfg.json 89 | vim cfg.json 90 | # 注意: 从v1.4.0开始, 我们把graph列表的配置信息，移动到了cfg.json中，不再需要graph_backends.txt 91 | 92 | # 启动服务 93 | ./control start 94 | 95 | # 校验服务,这里假定服务开启了9966的http监听端口。检验结果为ok表明服务正常启动。 96 | curl -s "127.0.0.1:9966/health" 97 | 98 | ... 99 | # 停止服务 100 | ./control stop 101 | 102 | ``` 103 | 服务启动后，可以通过日志查看服务的运行状态，日志文件地址为./var/app.log。可以通过`./test/debug`，查看服务的内部状态数据。可以通过scripts下的`query last`等脚本，进行数据查询。 104 | 105 | ## 配置文件格式说明 106 | 注意: 配置文件格式有更新; 请确保 `graph.replicas`和`graph.cluster` 的内容与transfer的配置**完全一致** 107 | 108 | ```bash 109 | { 110 | "debug": "false", // 是否开启debug日志 111 | "http": { 112 | "enabled": true, // 是否开启http.server 113 | "listen": "0.0.0.0:9966" // http.server监听地址&端口 114 | }, 115 | "graph": { 116 | "connTimeout": 1000, // 单位是毫秒，与后端graph建立连接的超时时间，可以根据网络质量微调，建议保持默认 117 | "callTimeout": 5000, // 单位是毫秒，从后端graph读取数据的超时时间，可以根据网络质量微调，建议保持默认 118 | "maxConns": 32, // 连接池相关配置，最大连接数，建议保持默认 119 | "maxIdle": 32, // 连接池相关配置，最大空闲连接数，建议保持默认 120 | "replicas": 500, // 这是一致性hash算法需要的节点副本数量，应该与transfer配置保持一致 121 | "cluster": { // 后端的graph列表，应该与transfer配置保持一致；不支持一条记录中配置两个地址 122 | "graph-00": "test.hostname01:6070", 123 | "graph-01": "test.hostname02:6070" 124 | }, 125 | "api": { // 适配grafana需要的API配置 126 | "query": "http://127.0.0.1:9966", // query的http地址 127 | "dashboard": "http://127.0.0.1:8081", // dashboard的http地址 128 | "max": 500 //API返回结果的最大数量 129 | } 130 | } 131 | } 132 | ``` 133 | 134 | ## 补充说明 135 | 部署完成query组件后，请修改dashboard组件的配置、使其能够正确寻址到query组件。请确保query组件的graph列表 与 transfer的配置 一致。 136 | 137 | -------------------------------------------------------------------------------- /cfg.example.json: -------------------------------------------------------------------------------- 1 | { 2 | "debug": "false", 3 | "http": { 4 | "enabled": true, 5 | "listen": "0.0.0.0:9966" 6 | }, 7 | "graph": { 8 | "connTimeout": 1000, 9 | "callTimeout": 5000, 10 | "maxConns": 32, 11 | "maxIdle": 32, 12 | "replicas": 500, 13 | "cluster": { 14 | "graph-00": "127.0.0.1:6070" 15 | } 16 | }, 17 | "api": { 18 | "query": "http://127.0.0.1:9966", 19 | "dashboard": "http://127.0.0.1:8081", 20 | "max": 500 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /control: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | workspace=$(cd $(dirname $0) && pwd) 3 | cd $workspace 4 | 5 | module=query 6 | app=falcon-$module 7 | conf=cfg.json 8 | pidfile=var/app.pid 9 | logfile=var/app.log 10 | 11 | mkdir -p var &>/dev/null 12 | 13 | 14 | ## build & pack 15 | function build() { 16 | commit=$(git log -1 --pretty=%h) 17 | cat < ./g/git.go 18 | package g 19 | const ( 20 | COMMIT = "$commit" 21 | ) 22 | EOF 23 | go build -o $app main.go 24 | sc=$? 25 | if [ $sc -ne 0 ];then 26 | echo "build error" 27 | exit $sc 28 | else 29 | echo -n "build ok, vsn=" 30 | ./$app -v 31 | fi 32 | } 33 | 34 | function pack() { 35 | build 36 | git log -1 --pretty=%h > gitversion 37 | version=`./$app -v` 38 | tar zcvf $app-$version.tar.gz control $app cfg.example.json gitversion ./test/debug ./scripts 39 | rm -f gitversion &>/dev/null 40 | } 41 | 42 | function packbin() { 43 | build 44 | git log -1 --pretty=%h > gitversion 45 | version=`./$app -v` 46 | tar zcvf $app-bin-$version.tar.gz $app gitversion 47 | rm -f gitversion &>/dev/null 48 | } 49 | 50 | ## opt 51 | function start() { 52 | check_pid 53 | running=$? 54 | if [ $running -gt 0 ];then 55 | echo -n "started, pid=" 56 | cat $pidfile 57 | return 1 58 | fi 59 | 60 | nohup ./$app -c $conf &> $logfile & 61 | echo $! > $pidfile 62 | echo "start ok, pid=$!" 63 | } 64 | 65 | function stop() { 66 | pid=`cat $pidfile` 67 | kill -9 $pid 68 | echo "stoped" 69 | } 70 | 71 | function restart() { 72 | stop 73 | sleep 1 74 | start 75 | } 76 | 77 | ## other 78 | function status() { 79 | check_pid 80 | running=$? 81 | if [ $running -gt 0 ];then 82 | echo -n "running, pid=" 83 | cat $pidfile 84 | else 85 | echo "stoped" 86 | fi 87 | } 88 | 89 | function version() { 90 | ./$app -vg 91 | } 92 | 93 | function tailf() { 94 | tail -f $logfile 95 | } 96 | 97 | ## internal 98 | function check_pid() { 99 | if [ -f $pidfile ];then 100 | pid=`cat $pidfile` 101 | if [ -n $pid ]; then 102 | running=`ps -p $pid|grep -v "PID TTY" |wc -l` 103 | return $running 104 | fi 105 | fi 106 | return 0 107 | } 108 | 109 | ## usage 110 | function usage() { 111 | echo "$0 build|pack|packbin|start|stop|restart|status|tail|version" 112 | } 113 | 114 | ## main 115 | action=$1 116 | case $action in 117 | ## build 118 | "build" ) 119 | build 120 | ;; 121 | "pack" ) 122 | pack 123 | ;; 124 | "packbin" ) 125 | packbin 126 | ;; 127 | ## opt 128 | "start" ) 129 | start 130 | ;; 131 | "stop" ) 132 | stop 133 | ;; 134 | "restart" ) 135 | restart 136 | ;; 137 | ## other 138 | "status" ) 139 | status 140 | ;; 141 | "version" ) 142 | version 143 | ;; 144 | "tail" ) 145 | tailf 146 | ;; 147 | * ) 148 | usage 149 | ;; 150 | esac 151 | -------------------------------------------------------------------------------- /g/cfg.go: -------------------------------------------------------------------------------- 1 | package g 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "sync" 7 | 8 | "github.com/toolkits/file" 9 | ) 10 | 11 | type HttpConfig struct { 12 | Enabled bool `json:"enabled"` 13 | Listen string `json:"listen"` 14 | } 15 | 16 | type GraphConfig struct { 17 | ConnTimeout int32 `json:"connTimeout"` 18 | CallTimeout int32 `json:"callTimeout"` 19 | MaxConns int32 `json:"maxConns"` 20 | MaxIdle int32 `json:"maxIdle"` 21 | Replicas int32 `json:"replicas"` 22 | Cluster map[string]string `json:"cluster"` 23 | } 24 | 25 | type ApiConfig struct { 26 | Query string `json:"query"` 27 | Dashboard string `json:"dashboard"` 28 | Max int `json:"max"` 29 | } 30 | 31 | type GlobalConfig struct { 32 | Debug string `json:"debug"` 33 | Http *HttpConfig `json:"http"` 34 | Graph *GraphConfig `json:"graph"` 35 | Api *ApiConfig `json:"api"` 36 | } 37 | 38 | var ( 39 | ConfigFile string 40 | config *GlobalConfig 41 | configLock = new(sync.RWMutex) 42 | ) 43 | 44 | func Config() *GlobalConfig { 45 | configLock.RLock() 46 | defer configLock.RUnlock() 47 | return config 48 | } 49 | 50 | func ParseConfig(cfg string) { 51 | if cfg == "" { 52 | log.Fatalln("config file not specified: use -c $filename") 53 | } 54 | 55 | if !file.IsExist(cfg) { 56 | log.Fatalln("config file specified not found:", cfg) 57 | } 58 | 59 | ConfigFile = cfg 60 | 61 | configContent, err := file.ToTrimString(cfg) 62 | if err != nil { 63 | log.Fatalln("read config file", cfg, "error:", err.Error()) 64 | } 65 | 66 | var c GlobalConfig 67 | err = json.Unmarshal([]byte(configContent), &c) 68 | if err != nil { 69 | log.Fatalln("parse config file", cfg, "error:", err.Error()) 70 | } 71 | 72 | // set config 73 | configLock.Lock() 74 | defer configLock.Unlock() 75 | config = &c 76 | 77 | log.Println("g.ParseConfig ok, file", cfg) 78 | } 79 | -------------------------------------------------------------------------------- /g/g.go: -------------------------------------------------------------------------------- 1 | package g 2 | 3 | import ( 4 | "runtime" 5 | ) 6 | 7 | // change log 8 | // 1.3.2 add last api for querying last item 9 | // 1.3.3 rm debug log in http.graph 10 | // 1.3.4 add http-api /graph/last/raw 11 | // 1.3.5 fill response with endpoint & counter when rpc Graph.Query getting errors 12 | // 1.4.0 restruct query: use simple rpc conn pool 13 | // 1.4.1 add last item counter, add proc for connpool 14 | // 1.4.2 rm nil items in http.responses 15 | // 1.4.3 spell check, make config consistent with previous 16 | 17 | const ( 18 | VERSION = "1.4.3" 19 | ) 20 | 21 | func init() { 22 | runtime.GOMAXPROCS(runtime.NumCPU()) 23 | } 24 | -------------------------------------------------------------------------------- /g/git.go: -------------------------------------------------------------------------------- 1 | package g 2 | 3 | const ( 4 | COMMIT = "a5ad45c" 5 | ) 6 | -------------------------------------------------------------------------------- /graph/graph.go: -------------------------------------------------------------------------------- 1 | package graph 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "log" 7 | "math" 8 | "time" 9 | 10 | cmodel "github.com/open-falcon/common/model" 11 | cutils "github.com/open-falcon/common/utils" 12 | rings "github.com/toolkits/consistent/rings" 13 | nset "github.com/toolkits/container/set" 14 | spool "github.com/toolkits/pool/simple_conn_pool" 15 | 16 | "github.com/open-falcon/query/g" 17 | ) 18 | 19 | // 连接池 20 | // node_address -> connection_pool 21 | var ( 22 | GraphConnPools *spool.SafeRpcConnPools 23 | ) 24 | 25 | // 服务节点的一致性哈希环 26 | // pk -> node 27 | var ( 28 | GraphNodeRing *rings.ConsistentHashNodeRing 29 | ) 30 | 31 | func Start() { 32 | initNodeRings() 33 | initConnPools() 34 | log.Println("graph.Start ok") 35 | } 36 | 37 | func QueryOne(para cmodel.GraphQueryParam) (resp *cmodel.GraphQueryResponse, err error) { 38 | start, end := para.Start, para.End 39 | endpoint, counter := para.Endpoint, para.Counter 40 | 41 | pool, addr, err := selectPool(endpoint, counter) 42 | if err != nil { 43 | return nil, err 44 | } 45 | 46 | conn, err := pool.Fetch() 47 | if err != nil { 48 | return nil, err 49 | } 50 | 51 | rpcConn := conn.(spool.RpcClient) 52 | if rpcConn.Closed() { 53 | pool.ForceClose(conn) 54 | return nil, errors.New("conn closed") 55 | } 56 | 57 | type ChResult struct { 58 | Err error 59 | Resp *cmodel.GraphQueryResponse 60 | } 61 | 62 | ch := make(chan *ChResult, 1) 63 | go func() { 64 | resp := &cmodel.GraphQueryResponse{} 65 | err := rpcConn.Call("Graph.Query", para, resp) 66 | ch <- &ChResult{Err: err, Resp: resp} 67 | }() 68 | 69 | select { 70 | case <-time.After(time.Duration(g.Config().Graph.CallTimeout) * time.Millisecond): 71 | pool.ForceClose(conn) 72 | return nil, fmt.Errorf("%s, call timeout. proc: %s", addr, pool.Proc()) 73 | case r := <-ch: 74 | if r.Err != nil { 75 | pool.ForceClose(conn) 76 | return r.Resp, fmt.Errorf("%s, call failed, err %v. proc: %s", addr, r.Err, pool.Proc()) 77 | } else { 78 | pool.Release(conn) 79 | 80 | if len(r.Resp.Values) < 1 { 81 | return r.Resp, nil 82 | } 83 | 84 | // TODO query不该做这些事情, 说明graph没做好 85 | fixed := []*cmodel.RRDData{} 86 | for _, v := range r.Resp.Values { 87 | if v == nil || !(v.Timestamp >= start && v.Timestamp <= end) { 88 | continue 89 | } 90 | //FIXME: 查询数据的时候，把所有的负值都过滤掉，因为transfer之前在设置最小值的时候为U 91 | if (r.Resp.DsType == "DERIVE" || r.Resp.DsType == "COUNTER") && v.Value < 0 { 92 | fixed = append(fixed, &cmodel.RRDData{Timestamp: v.Timestamp, Value: cmodel.JsonFloat(math.NaN())}) 93 | } else { 94 | fixed = append(fixed, v) 95 | } 96 | } 97 | r.Resp.Values = fixed 98 | } 99 | return r.Resp, nil 100 | } 101 | } 102 | 103 | func Info(para cmodel.GraphInfoParam) (resp *cmodel.GraphFullyInfo, err error) { 104 | endpoint, counter := para.Endpoint, para.Counter 105 | 106 | pool, addr, err := selectPool(endpoint, counter) 107 | if err != nil { 108 | return nil, err 109 | } 110 | 111 | conn, err := pool.Fetch() 112 | if err != nil { 113 | return nil, err 114 | } 115 | 116 | rpcConn := conn.(spool.RpcClient) 117 | if rpcConn.Closed() { 118 | pool.ForceClose(conn) 119 | return nil, errors.New("conn closed") 120 | } 121 | 122 | type ChResult struct { 123 | Err error 124 | Resp *cmodel.GraphInfoResp 125 | } 126 | ch := make(chan *ChResult, 1) 127 | go func() { 128 | resp := &cmodel.GraphInfoResp{} 129 | err := rpcConn.Call("Graph.Info", para, resp) 130 | ch <- &ChResult{Err: err, Resp: resp} 131 | }() 132 | 133 | select { 134 | case <-time.After(time.Duration(g.Config().Graph.CallTimeout) * time.Millisecond): 135 | pool.ForceClose(conn) 136 | return nil, fmt.Errorf("%s, call timeout. proc: %s", addr, pool.Proc()) 137 | case r := <-ch: 138 | if r.Err != nil { 139 | pool.ForceClose(conn) 140 | return nil, fmt.Errorf("%s, call failed, err %v. proc: %s", addr, r.Err, pool.Proc()) 141 | } else { 142 | pool.Release(conn) 143 | fullyInfo := cmodel.GraphFullyInfo{ 144 | Endpoint: endpoint, 145 | Counter: counter, 146 | ConsolFun: r.Resp.ConsolFun, 147 | Step: r.Resp.Step, 148 | Filename: r.Resp.Filename, 149 | Addr: addr, 150 | } 151 | return &fullyInfo, nil 152 | } 153 | } 154 | } 155 | 156 | func Last(para cmodel.GraphLastParam) (r *cmodel.GraphLastResp, err error) { 157 | endpoint, counter := para.Endpoint, para.Counter 158 | 159 | pool, addr, err := selectPool(endpoint, counter) 160 | if err != nil { 161 | return nil, err 162 | } 163 | 164 | conn, err := pool.Fetch() 165 | if err != nil { 166 | return nil, err 167 | } 168 | 169 | rpcConn := conn.(spool.RpcClient) 170 | if rpcConn.Closed() { 171 | pool.ForceClose(conn) 172 | return nil, errors.New("conn closed") 173 | } 174 | 175 | type ChResult struct { 176 | Err error 177 | Resp *cmodel.GraphLastResp 178 | } 179 | ch := make(chan *ChResult, 1) 180 | go func() { 181 | resp := &cmodel.GraphLastResp{} 182 | err := rpcConn.Call("Graph.Last", para, resp) 183 | ch <- &ChResult{Err: err, Resp: resp} 184 | }() 185 | 186 | select { 187 | case <-time.After(time.Duration(g.Config().Graph.CallTimeout) * time.Millisecond): 188 | pool.ForceClose(conn) 189 | return nil, fmt.Errorf("%s, call timeout. proc: %s", addr, pool.Proc()) 190 | case r := <-ch: 191 | if r.Err != nil { 192 | pool.ForceClose(conn) 193 | return r.Resp, fmt.Errorf("%s, call failed, err %v. proc: %s", addr, r.Err, pool.Proc()) 194 | } else { 195 | pool.Release(conn) 196 | return r.Resp, nil 197 | } 198 | } 199 | } 200 | 201 | func LastRaw(para cmodel.GraphLastParam) (r *cmodel.GraphLastResp, err error) { 202 | endpoint, counter := para.Endpoint, para.Counter 203 | 204 | pool, addr, err := selectPool(endpoint, counter) 205 | if err != nil { 206 | return nil, err 207 | } 208 | 209 | conn, err := pool.Fetch() 210 | if err != nil { 211 | return nil, err 212 | } 213 | 214 | rpcConn := conn.(spool.RpcClient) 215 | if rpcConn.Closed() { 216 | pool.ForceClose(conn) 217 | return nil, errors.New("conn closed") 218 | } 219 | 220 | type ChResult struct { 221 | Err error 222 | Resp *cmodel.GraphLastResp 223 | } 224 | ch := make(chan *ChResult, 1) 225 | go func() { 226 | resp := &cmodel.GraphLastResp{} 227 | err := rpcConn.Call("Graph.LastRaw", para, resp) 228 | ch <- &ChResult{Err: err, Resp: resp} 229 | }() 230 | 231 | select { 232 | case <-time.After(time.Duration(g.Config().Graph.CallTimeout) * time.Millisecond): 233 | pool.ForceClose(conn) 234 | return nil, fmt.Errorf("%s, call timeout. proc: %s", addr, pool.Proc()) 235 | case r := <-ch: 236 | if r.Err != nil { 237 | pool.ForceClose(conn) 238 | return r.Resp, fmt.Errorf("%s, call failed, err %v. proc: %s", addr, r.Err, pool.Proc()) 239 | } else { 240 | pool.Release(conn) 241 | return r.Resp, nil 242 | } 243 | } 244 | } 245 | 246 | func selectPool(endpoint, counter string) (rpool *spool.ConnPool, raddr string, rerr error) { 247 | pkey := cutils.PK2(endpoint, counter) 248 | node, err := GraphNodeRing.GetNode(pkey) 249 | if err != nil { 250 | return nil, "", err 251 | } 252 | 253 | addr, found := g.Config().Graph.Cluster[node] 254 | if !found { 255 | return nil, "", errors.New("node not found") 256 | } 257 | 258 | pool, found := GraphConnPools.Get(addr) 259 | if !found { 260 | return nil, addr, errors.New("addr not found") 261 | } 262 | 263 | return pool, addr, nil 264 | } 265 | 266 | // internal functions 267 | func initConnPools() { 268 | cfg := g.Config() 269 | 270 | // TODO 为了得到Slice,这里做的太复杂了 271 | graphInstances := nset.NewSafeSet() 272 | for _, address := range cfg.Graph.Cluster { 273 | graphInstances.Add(address) 274 | } 275 | GraphConnPools = spool.CreateSafeRpcConnPools(cfg.Graph.MaxConns, cfg.Graph.MaxIdle, 276 | cfg.Graph.ConnTimeout, cfg.Graph.CallTimeout, graphInstances.ToSlice()) 277 | } 278 | 279 | func initNodeRings() { 280 | cfg := g.Config() 281 | GraphNodeRing = rings.NewConsistentHashNodesRing(cfg.Graph.Replicas, cutils.KeysOfMap(cfg.Graph.Cluster)) 282 | } 283 | -------------------------------------------------------------------------------- /http/api.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "bytes" 5 | "github.com/open-falcon/query/g" 6 | "io/ioutil" 7 | "log" 8 | "net/http" 9 | ) 10 | 11 | func postByJson(rw http.ResponseWriter, req *http.Request, url string) { 12 | buf := new(bytes.Buffer) 13 | buf.ReadFrom(req.Body) 14 | s := buf.String() 15 | reqPost, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte(s))) 16 | if err != nil { 17 | log.Println("Error =", err.Error()) 18 | } 19 | reqPost.Header.Set("Content-Type", "application/json") 20 | 21 | client := &http.Client{} 22 | resp, err := client.Do(reqPost) 23 | if err != nil { 24 | log.Println("Error =", err.Error()) 25 | } 26 | defer resp.Body.Close() 27 | 28 | body, _ := ioutil.ReadAll(resp.Body) 29 | rw.Header().Set("Content-Type", "application/json; charset=UTF-8") 30 | rw.Write(body) 31 | } 32 | 33 | func queryInfo(rw http.ResponseWriter, req *http.Request) { 34 | url := g.Config().Api.Query + "/graph/info" 35 | postByJson(rw, req, url) 36 | } 37 | 38 | func queryHistory(rw http.ResponseWriter, req *http.Request) { 39 | url := g.Config().Api.Query + "/graph/history" 40 | postByJson(rw, req, url) 41 | } 42 | 43 | func getRequest(rw http.ResponseWriter, url string) { 44 | req, err := http.NewRequest("GET", url, nil) 45 | if err != nil { 46 | log.Println("Error =", err.Error()) 47 | } 48 | 49 | client := &http.Client{} 50 | resp, err := client.Do(req) 51 | if err != nil { 52 | log.Println("Error =", err.Error()) 53 | } 54 | defer resp.Body.Close() 55 | 56 | body, _ := ioutil.ReadAll(resp.Body) 57 | rw.Header().Set("Content-Type", "application/json; charset=UTF-8") 58 | rw.Write(body) 59 | } 60 | 61 | func dashboardEndpoints(rw http.ResponseWriter, req *http.Request) { 62 | url := g.Config().Api.Dashboard + req.URL.RequestURI() 63 | getRequest(rw, url) 64 | } 65 | 66 | func postByForm(rw http.ResponseWriter, req *http.Request, url string) { 67 | req.ParseForm() 68 | client := &http.Client{} 69 | resp, err := client.PostForm(url, req.PostForm) 70 | if err != nil { 71 | log.Println("Error =", err.Error()) 72 | } 73 | defer resp.Body.Close() 74 | 75 | body, _ := ioutil.ReadAll(resp.Body) 76 | rw.Header().Set("Content-Type", "application/json; charset=UTF-8") 77 | rw.Write(body) 78 | } 79 | 80 | func dashboardCounters(rw http.ResponseWriter, req *http.Request) { 81 | url := g.Config().Api.Dashboard + "/api/counters" 82 | postByForm(rw, req, url) 83 | } 84 | 85 | func dashboardChart(rw http.ResponseWriter, req *http.Request) { 86 | url := g.Config().Api.Dashboard + "/chart" 87 | postByForm(rw, req, url) 88 | } 89 | 90 | func configApiRoutes() { 91 | http.HandleFunc("/api/info", queryInfo) 92 | http.HandleFunc("/api/history", queryHistory) 93 | http.HandleFunc("/api/endpoints", dashboardEndpoints) 94 | http.HandleFunc("/api/counters", dashboardCounters) 95 | http.HandleFunc("/api/chart", dashboardChart) 96 | } 97 | -------------------------------------------------------------------------------- /http/common.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | 7 | "github.com/toolkits/file" 8 | 9 | "github.com/open-falcon/query/g" 10 | ) 11 | 12 | func configCommonRoutes() { 13 | http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { 14 | w.Write([]byte("ok\n")) 15 | }) 16 | 17 | http.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) { 18 | w.Write([]byte(fmt.Sprintf("%s\n", g.VERSION))) 19 | }) 20 | 21 | http.HandleFunc("/workdir", func(w http.ResponseWriter, r *http.Request) { 22 | w.Write([]byte(fmt.Sprintf("%s\n", file.SelfDir()))) 23 | }) 24 | 25 | http.HandleFunc("/config", func(w http.ResponseWriter, r *http.Request) { 26 | RenderDataJson(w, g.Config()) 27 | }) 28 | 29 | } 30 | -------------------------------------------------------------------------------- /http/grafana.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "github.com/open-falcon/query/g" 7 | "io/ioutil" 8 | "log" 9 | "math/rand" 10 | "net/http" 11 | "net/url" 12 | "regexp" 13 | "strconv" 14 | "strings" 15 | "time" 16 | ) 17 | 18 | type Idc struct { 19 | Id int 20 | Pop_id int 21 | Name string 22 | Count int 23 | Area string 24 | Province string 25 | City string 26 | Updated_at string 27 | } 28 | 29 | type Province struct { 30 | Id int 31 | Province string 32 | Count int 33 | Updated_at string 34 | } 35 | 36 | type City struct { 37 | Id int 38 | City string 39 | Province string 40 | Count int 41 | Updated_at string 42 | } 43 | 44 | func getHosts(w http.ResponseWriter, req *http.Request, hostKeyword string) { 45 | if len(hostKeyword) == 1 { 46 | hostKeyword = ".+" 47 | } 48 | rand.Seed(time.Now().UTC().UnixNano()) 49 | random64 := rand.Float64() 50 | _r := strconv.FormatFloat(random64, 'f', -1, 32) 51 | maxQuery := strconv.Itoa(g.Config().Api.Max) 52 | url := "/api/endpoints" + "?q=" + hostKeyword + "&tags&limit=" + maxQuery + "&_r=" + _r + "®ex_query=1" 53 | if strings.Index(g.Config().Api.Query, req.Host) >= 0 { 54 | url = "http://localhost:9966" + url 55 | } else { 56 | url = g.Config().Api.Query + url 57 | } 58 | 59 | reqGet, err := http.NewRequest("GET", url, nil) 60 | if err != nil { 61 | StdRender(w, "", err) 62 | } 63 | 64 | client := &http.Client{} 65 | resp, err := client.Do(reqGet) 66 | if err != nil { 67 | StdRender(w, "", err) 68 | } 69 | 70 | defer resp.Body.Close() 71 | 72 | result := []interface{}{} 73 | if resp.Status == "200 OK" { 74 | body, _ := ioutil.ReadAll(resp.Body) 75 | var nodes = make(map[string]interface{}) 76 | if err := json.Unmarshal(body, &nodes); err != nil { 77 | StdRender(w, "", err) 78 | } 79 | for _, host := range nodes["data"].([]interface{}) { 80 | item := map[string]interface{}{ 81 | "text": host, 82 | "expandable": true, 83 | } 84 | result = append(result, item) 85 | } 86 | RenderJson(w, result) 87 | } else { 88 | RenderJson(w, result) 89 | } 90 | } 91 | 92 | func getNextCounterSegment(metric string, counter string) string { 93 | if len(metric) > 0 { 94 | metric += "." 95 | } 96 | counter = strings.Replace(counter, metric, "", 1) 97 | segment := strings.Split(counter, ".")[0] 98 | return segment 99 | } 100 | 101 | func checkSegmentExpandable(segment string, counter string) bool { 102 | segments := strings.Split(counter, ".") 103 | expandable := !(segment == segments[len(segments)-1]) 104 | return expandable 105 | } 106 | 107 | func getMetrics(w http.ResponseWriter, req *http.Request, query string) { 108 | result := []interface{}{} 109 | 110 | query = strings.Replace(query, "#.*", "", -1) 111 | arrQuery := strings.Split(query, "#") 112 | host, arrMetric := arrQuery[0], arrQuery[1:] 113 | maxQuery := strconv.Itoa(g.Config().Api.Max) 114 | metric := strings.Join(arrMetric, ".") 115 | reg, _ := regexp.Compile("(^{|}$)") 116 | host = reg.ReplaceAllString(host, "") 117 | host = strings.Replace(host, ",", "\",\"", -1) 118 | 119 | endpoints := "[\"" + host + "\"]" 120 | 121 | rand.Seed(time.Now().UTC().UnixNano()) 122 | random64 := rand.Float64() 123 | _r := strconv.FormatFloat(random64, 'f', -1, 32) 124 | 125 | form := url.Values{} 126 | form.Set("endpoints", endpoints) 127 | form.Add("q", metric) 128 | form.Add("limit", maxQuery) 129 | form.Add("_r", _r) 130 | 131 | target := "/api/counters" 132 | if strings.Index(g.Config().Api.Query, req.Host) >= 0 { 133 | target = "http://localhost:9966" + target 134 | } else { 135 | target = g.Config().Api.Query + target 136 | } 137 | 138 | reqPost, err := http.NewRequest("POST", target, strings.NewReader(form.Encode())) 139 | if err != nil { 140 | log.Println("Error =", err.Error()) 141 | } 142 | reqPost.Header.Set("Content-Type", "application/x-www-form-urlencoded") 143 | 144 | client := &http.Client{} 145 | resp, err := client.Do(reqPost) 146 | if err != nil { 147 | log.Println("Error =", err.Error()) 148 | } 149 | defer resp.Body.Close() 150 | 151 | if resp.Status == "200 OK" { 152 | body, _ := ioutil.ReadAll(resp.Body) 153 | var nodes = make(map[string]interface{}) 154 | if err := json.Unmarshal(body, &nodes); err != nil { 155 | log.Println(err.Error()) 156 | } 157 | var segmentPool = make(map[string]int) 158 | for _, data := range nodes["data"].([]interface{}) { 159 | counter := data.([]interface{})[0].(string) 160 | segment := getNextCounterSegment(metric, counter) 161 | expandable := checkSegmentExpandable(segment, counter) 162 | if _, ok := segmentPool[segment]; !ok { 163 | item := map[string]interface{}{ 164 | "text": segment, 165 | "expandable": expandable, 166 | } 167 | result = append(result, item) 168 | segmentPool[segment] = 1 169 | } 170 | } 171 | RenderJson(w, result) 172 | } else { 173 | RenderJson(w, result) 174 | } 175 | } 176 | 177 | func setQueryEditor(w http.ResponseWriter, req *http.Request) { 178 | query := req.URL.Query().Get("query") 179 | query = strings.Replace(query, ".%", "", -1) 180 | query = strings.Replace(query, ".undefined", "", -1) 181 | query = strings.Replace(query, ".select metric", "", -1) 182 | if !strings.Contains(query, "#") { 183 | getHosts(w, req, query) 184 | } else { 185 | getMetrics(w, req, query) 186 | } 187 | } 188 | 189 | func getMetricValues(req *http.Request, host string, targets []string, result []interface{}) []interface{} { 190 | endpoint_counters := []interface{}{} 191 | metric := strings.Join(targets, ".") 192 | if strings.Contains(host, "{") { // Templating metrics request 193 | host = strings.Replace(host, "{", "", -1) 194 | host = strings.Replace(host, "}", "", -1) 195 | hosts := strings.Split(host, ",") 196 | for _, host := range hosts { 197 | item := map[string]string{ 198 | "endpoint": host, 199 | "counter": metric, 200 | } 201 | endpoint_counters = append(endpoint_counters, item) 202 | } 203 | } else { 204 | item := map[string]string{ 205 | "endpoint": host, 206 | "counter": metric, 207 | } 208 | endpoint_counters = append(endpoint_counters, item) 209 | } 210 | 211 | if len(endpoint_counters) > 0 { 212 | from, err := strconv.ParseInt(req.PostForm["from"][0], 10, 64) 213 | until, err := strconv.ParseInt(req.PostForm["until"][0], 10, 64) 214 | url := "/graph/history" 215 | if strings.Index(g.Config().Api.Query, req.Host) >= 0 { 216 | url = "http://localhost:9966" + url 217 | } else { 218 | url = g.Config().Api.Query + url 219 | } 220 | 221 | args := map[string]interface{}{ 222 | "start": from, 223 | "end": until, 224 | "cf": "AVERAGE", 225 | "endpoint_counters": endpoint_counters, 226 | } 227 | bs, err := json.Marshal(args) 228 | if err != nil { 229 | log.Println("Error =", err.Error()) 230 | } 231 | 232 | reqPost, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte(bs))) 233 | if err != nil { 234 | log.Println("Error =", err.Error()) 235 | } 236 | reqPost.Header.Set("Content-Type", "application/json") 237 | 238 | client := &http.Client{} 239 | resp, err := client.Do(reqPost) 240 | if err != nil { 241 | log.Println("Error =", err.Error()) 242 | } 243 | defer resp.Body.Close() 244 | 245 | if resp.Status == "200 OK" { 246 | body, _ := ioutil.ReadAll(resp.Body) 247 | nodes := []interface{}{} 248 | if err := json.Unmarshal(body, &nodes); err != nil { 249 | log.Println(err.Error()) 250 | } 251 | 252 | for _, node := range nodes { 253 | if _, ok := node.(map[string]interface{})["Values"]; ok { 254 | result = append(result, node) 255 | } 256 | } 257 | } 258 | } 259 | return result 260 | } 261 | 262 | func getValues(w http.ResponseWriter, req *http.Request) { 263 | result := []interface{}{} 264 | req.ParseForm() 265 | for _, target := range req.PostForm["target"] { 266 | if !strings.Contains(target, ".select metric") { 267 | targets := strings.Split(target, "#") 268 | host, targets := targets[0], targets[1:] 269 | result = getMetricValues(req, host, targets, result) 270 | } 271 | } 272 | RenderJson(w, result) 273 | } 274 | 275 | func GrafanaApiParser(w http.ResponseWriter, req *http.Request) { 276 | if req.Method == "GET" { 277 | setQueryEditor(w, req) 278 | } else if req.Method == "POST" { 279 | getValues(w, req) 280 | } 281 | } 282 | 283 | func configGrafanaRoutes() { 284 | http.HandleFunc("/api/grafana/", GrafanaApiParser) 285 | } 286 | -------------------------------------------------------------------------------- /http/graph_http.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "log" 7 | "net/http" 8 | 9 | cmodel "github.com/open-falcon/common/model" 10 | "github.com/open-falcon/query/graph" 11 | "github.com/open-falcon/query/proc" 12 | ) 13 | 14 | type GraphHistoryParam struct { 15 | Start int `json:"start"` 16 | End int `json:"end"` 17 | CF string `json:"cf"` 18 | EndpointCounters []cmodel.GraphInfoParam `json:"endpoint_counters"` 19 | } 20 | 21 | func configGraphRoutes() { 22 | 23 | // method:post 24 | http.HandleFunc("/graph/history", func(w http.ResponseWriter, r *http.Request) { 25 | // statistics 26 | proc.HistoryRequestCnt.Incr() 27 | 28 | var body GraphHistoryParam 29 | decoder := json.NewDecoder(r.Body) 30 | err := decoder.Decode(&body) 31 | if err != nil { 32 | StdRender(w, "", err) 33 | return 34 | } 35 | 36 | if len(body.EndpointCounters) == 0 { 37 | StdRender(w, "", errors.New("empty_payload")) 38 | return 39 | } 40 | 41 | data := []*cmodel.GraphQueryResponse{} 42 | for _, ec := range body.EndpointCounters { 43 | request := cmodel.GraphQueryParam{ 44 | Start: int64(body.Start), 45 | End: int64(body.End), 46 | ConsolFun: body.CF, 47 | Endpoint: ec.Endpoint, 48 | Counter: ec.Counter, 49 | } 50 | result, err := graph.QueryOne(request) 51 | if err != nil { 52 | log.Printf("graph.queryOne fail, %v", err) 53 | } 54 | if result == nil { 55 | continue 56 | } 57 | data = append(data, result) 58 | } 59 | 60 | // statistics 61 | proc.HistoryResponseCounterCnt.IncrBy(int64(len(data))) 62 | for _, item := range data { 63 | proc.HistoryResponseItemCnt.IncrBy(int64(len(item.Values))) 64 | } 65 | 66 | StdRender(w, data, nil) 67 | }) 68 | 69 | // post, info 70 | http.HandleFunc("/graph/info", func(w http.ResponseWriter, r *http.Request) { 71 | // statistics 72 | proc.InfoRequestCnt.Incr() 73 | 74 | var body []*cmodel.GraphInfoParam 75 | decoder := json.NewDecoder(r.Body) 76 | err := decoder.Decode(&body) 77 | if err != nil { 78 | StdRender(w, "", err) 79 | return 80 | } 81 | 82 | if len(body) == 0 { 83 | StdRender(w, "", errors.New("empty")) 84 | return 85 | } 86 | 87 | data := []*cmodel.GraphFullyInfo{} 88 | for _, param := range body { 89 | if param == nil { 90 | continue 91 | } 92 | info, err := graph.Info(*param) 93 | if err != nil { 94 | log.Printf("graph.info fail, resp: %v, err: %v", info, err) 95 | } 96 | if info == nil { 97 | continue 98 | } 99 | data = append(data, info) 100 | } 101 | 102 | StdRender(w, data, nil) 103 | }) 104 | 105 | // post, last 106 | http.HandleFunc("/graph/last", func(w http.ResponseWriter, r *http.Request) { 107 | // statistics 108 | proc.LastRequestCnt.Incr() 109 | 110 | var body []*cmodel.GraphLastParam 111 | decoder := json.NewDecoder(r.Body) 112 | err := decoder.Decode(&body) 113 | if err != nil { 114 | StdRender(w, "", err) 115 | return 116 | } 117 | 118 | if len(body) == 0 { 119 | StdRender(w, "", errors.New("empty")) 120 | return 121 | } 122 | 123 | data := []*cmodel.GraphLastResp{} 124 | for _, param := range body { 125 | if param == nil { 126 | continue 127 | } 128 | last, err := graph.Last(*param) 129 | if err != nil { 130 | log.Printf("graph.last fail, resp: %v, err: %v", last, err) 131 | } 132 | if last == nil { 133 | continue 134 | } 135 | data = append(data, last) 136 | } 137 | 138 | // statistics 139 | proc.LastRequestItemCnt.IncrBy(int64(len(data))) 140 | 141 | StdRender(w, data, nil) 142 | }) 143 | 144 | // post, last/raw 145 | http.HandleFunc("/graph/last/raw", func(w http.ResponseWriter, r *http.Request) { 146 | // statistics 147 | proc.LastRawRequestCnt.Incr() 148 | 149 | var body []*cmodel.GraphLastParam 150 | decoder := json.NewDecoder(r.Body) 151 | err := decoder.Decode(&body) 152 | if err != nil { 153 | StdRender(w, "", err) 154 | return 155 | } 156 | 157 | if len(body) == 0 { 158 | StdRender(w, "", errors.New("empty")) 159 | return 160 | } 161 | 162 | data := []*cmodel.GraphLastResp{} 163 | for _, param := range body { 164 | if param == nil { 165 | continue 166 | } 167 | last, err := graph.LastRaw(*param) 168 | if err != nil { 169 | log.Printf("graph.last.raw fail, resp: %v, err: %v", last, err) 170 | } 171 | if last == nil { 172 | continue 173 | } 174 | data = append(data, last) 175 | } 176 | // statistics 177 | proc.LastRawRequestItemCnt.IncrBy(int64(len(data))) 178 | StdRender(w, data, nil) 179 | }) 180 | 181 | } 182 | -------------------------------------------------------------------------------- /http/http.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "net/http" 7 | _ "net/http/pprof" 8 | 9 | "github.com/open-falcon/query/g" 10 | ) 11 | 12 | type Dto struct { 13 | Msg string `json:"msg"` 14 | Data interface{} `json:"data"` 15 | } 16 | 17 | func Start() { 18 | if !g.Config().Http.Enabled { 19 | log.Println("http.Start warning, not enabled") 20 | return 21 | } 22 | 23 | // config http routes 24 | configCommonRoutes() 25 | configProcHttpRoutes() 26 | configGraphRoutes() 27 | configApiRoutes() 28 | configGrafanaRoutes() 29 | 30 | // start http server 31 | addr := g.Config().Http.Listen 32 | s := &http.Server{ 33 | Addr: addr, 34 | MaxHeaderBytes: 1 << 30, 35 | } 36 | 37 | log.Println("http.Start ok, listening on", addr) 38 | log.Fatalln(s.ListenAndServe()) 39 | } 40 | 41 | func RenderJson(w http.ResponseWriter, v interface{}) { 42 | bs, err := json.Marshal(v) 43 | if err != nil { 44 | http.Error(w, err.Error(), http.StatusInternalServerError) 45 | return 46 | } 47 | w.Header().Set("Content-Type", "application/json; charset=UTF-8") 48 | w.Write(bs) 49 | } 50 | 51 | func RenderDataJson(w http.ResponseWriter, data interface{}) { 52 | RenderJson(w, Dto{Msg: "success", Data: data}) 53 | } 54 | 55 | func RenderMsgJson(w http.ResponseWriter, msg string) { 56 | RenderJson(w, map[string]string{"msg": msg}) 57 | } 58 | 59 | func AutoRender(w http.ResponseWriter, data interface{}, err error) { 60 | if err != nil { 61 | RenderMsgJson(w, err.Error()) 62 | return 63 | } 64 | RenderDataJson(w, data) 65 | } 66 | 67 | func StdRender(w http.ResponseWriter, data interface{}, err error) { 68 | if err != nil { 69 | w.WriteHeader(400) 70 | RenderMsgJson(w, err.Error()) 71 | return 72 | } 73 | RenderJson(w, data) 74 | } 75 | -------------------------------------------------------------------------------- /http/proc_http.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "net/http" 5 | "strings" 6 | 7 | "github.com/open-falcon/query/graph" 8 | "github.com/open-falcon/query/proc" 9 | ) 10 | 11 | func configProcHttpRoutes() { 12 | // TO BE DISCARDed 13 | http.HandleFunc("/statistics/all", func(w http.ResponseWriter, r *http.Request) { 14 | RenderDataJson(w, proc.GetAll()) 15 | }) 16 | 17 | // counter 18 | http.HandleFunc("/counter/all", func(w http.ResponseWriter, r *http.Request) { 19 | RenderDataJson(w, proc.GetAll()) 20 | }) 21 | 22 | // conn pools 23 | http.HandleFunc("/proc/connpool", func(w http.ResponseWriter, r *http.Request) { 24 | result := strings.Join(graph.GraphConnPools.Proc(), "\n") 25 | w.Write([]byte(result)) 26 | }) 27 | } 28 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/open-falcon/query/g" 9 | "github.com/open-falcon/query/graph" 10 | "github.com/open-falcon/query/http" 11 | "github.com/open-falcon/query/proc" 12 | ) 13 | 14 | func main() { 15 | cfg := flag.String("c", "cfg.json", "specify config file") 16 | version := flag.Bool("v", false, "show version") 17 | versionGit := flag.Bool("vg", false, "show version and git commit log") 18 | flag.Parse() 19 | 20 | if *version { 21 | fmt.Println(g.VERSION) 22 | os.Exit(0) 23 | } 24 | if *versionGit { 25 | fmt.Println(g.VERSION, g.COMMIT) 26 | os.Exit(0) 27 | } 28 | 29 | // config 30 | g.ParseConfig(*cfg) 31 | // proc 32 | proc.Start() 33 | 34 | // graph 35 | graph.Start() 36 | 37 | // http 38 | http.Start() 39 | 40 | select {} 41 | } 42 | -------------------------------------------------------------------------------- /proc/proc.go: -------------------------------------------------------------------------------- 1 | package proc 2 | 3 | import ( 4 | nproc "github.com/toolkits/proc" 5 | "log" 6 | ) 7 | 8 | // 统计指标的整体数据 9 | var ( 10 | // http请求次数 11 | HistoryRequestCnt = nproc.NewSCounterQps("HistoryRequestCnt") 12 | InfoRequestCnt = nproc.NewSCounterQps("InfoRequestCnt") 13 | LastRequestCnt = nproc.NewSCounterQps("LastRequestCnt") 14 | LastRawRequestCnt = nproc.NewSCounterQps("LastRawRequestCnt") 15 | 16 | // http回执的监控数据条数 17 | HistoryResponseCounterCnt = nproc.NewSCounterQps("HistoryResponseCounterCnt") 18 | HistoryResponseItemCnt = nproc.NewSCounterQps("HistoryResponseItemCnt") 19 | LastRequestItemCnt = nproc.NewSCounterQps("LastRequestItemCnt") 20 | LastRawRequestItemCnt = nproc.NewSCounterQps("LastRawRequestItemCnt") 21 | 22 | // TODO http request delay 23 | ) 24 | 25 | func Start() { 26 | log.Println("proc.Start, ok") 27 | } 28 | 29 | func GetAll() []interface{} { 30 | ret := make([]interface{}, 0) 31 | 32 | // http request 33 | ret = append(ret, HistoryRequestCnt.Get()) 34 | ret = append(ret, InfoRequestCnt.Get()) 35 | ret = append(ret, LastRequestCnt.Get()) 36 | ret = append(ret, LastRawRequestCnt.Get()) 37 | 38 | // http response 39 | ret = append(ret, HistoryResponseCounterCnt.Get()) 40 | ret = append(ret, HistoryResponseItemCnt.Get()) 41 | ret = append(ret, LastRequestItemCnt.Get()) 42 | ret = append(ret, LastRawRequestItemCnt.Get()) 43 | 44 | return ret 45 | } 46 | -------------------------------------------------------------------------------- /scripts/info: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ $# != 2 ];then 3 | printf "format:./query \"endpoint\" \"counter\"\n" 4 | exit 1 5 | fi 6 | 7 | # args 8 | endpoint=$1 9 | counter=$2 10 | 11 | # form request body 12 | req="[{\"endpoint\":\"$endpoint\", \"counter\":\"$counter\"}]" 13 | 14 | # request 15 | url="http://127.0.0.1:9966/graph/info" 16 | curl -s -X POST -d "$req" "$url" | python -m json.tool 17 | -------------------------------------------------------------------------------- /scripts/info.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | 4 | d = [ 5 | { 6 | "endpoint": "hh-op-mon-tran01.bj", 7 | "counter": "load.15min", 8 | }, 9 | { 10 | "endpoint": "hh-op-mon-tran01.bj", 11 | "counter": "net.if.in.bytes/iface=eth0", 12 | }, 13 | { 14 | "endpoint": "10.202.31.14:7934", 15 | "counter": "p2-com.xiaomi.miui.mibi.service.MibiService-method-createTradeV1", 16 | }, 17 | ] 18 | 19 | url = "http://query.falcon.miliao.srv:9966/graph/info" 20 | r = requests.post(url, data=json.dumps(d)) 21 | print r.text 22 | 23 | #curl "localhost:9966/graph/info/one?endpoint=`hostname`&counter=load.1min" |python -m json.tool 24 | -------------------------------------------------------------------------------- /scripts/last: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ $# != 2 ];then 3 | printf "format:./last \"endpoint\" \"counter\"\n" 4 | exit 1 5 | fi 6 | 7 | # args 8 | endpoint=$1 9 | counter=$2 10 | 11 | # form request body 12 | req="[{\"endpoint\":\"$endpoint\", \"counter\":\"$counter\"}]" 13 | 14 | # request 15 | url="http://127.0.0.1:9966/graph/last" 16 | curl -s -X POST -d "$req" "$url" | python -m json.tool 17 | -------------------------------------------------------------------------------- /scripts/last_raw: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ $# != 2 ];then 3 | printf "format:./last_raw \"endpoint\" \"counter\"\n" 4 | exit 1 5 | fi 6 | 7 | # args 8 | endpoint=$1 9 | counter=$2 10 | 11 | # form request body 12 | req="[{\"endpoint\":\"$endpoint\", \"counter\":\"$counter\"}]" 13 | 14 | # request 15 | url="http://127.0.0.1:9966/graph/last/raw" 16 | curl -s -X POST -d "$req" "$url" | python -m json.tool 17 | -------------------------------------------------------------------------------- /scripts/query: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ $# != 2 ];then 3 | printf "format:./query \"endpoint\" \"counter\"\n" 4 | exit 1 5 | fi 6 | 7 | # args 8 | endpoint=$1 9 | counter=$2 10 | cf="AVERAGE" 11 | 12 | # form request body 13 | end_ts=`date +%s` 14 | start_ts=`expr $end_ts - 180` 15 | req="{\"start\":$start_ts, \"end\":$end_ts, \"cf\":\"$cf\", \"endpoint_counters\": [{\"endpoint\":\"$endpoint\", \"counter\":\"$counter\"}]}" 16 | 17 | # request 18 | url="http://127.0.0.1:9966/graph/history" 19 | curl -s -X POST -d "$req" "$url" | python -m json.tool 20 | -------------------------------------------------------------------------------- /scripts/query.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import time 3 | import json 4 | 5 | end = int(time.time()) 6 | start = end - 3600 7 | 8 | d = { 9 | "start": start, 10 | "end": end, 11 | "cf": "AVERAGE", 12 | "endpoint_counters": [ 13 | { 14 | "endpoint": "lg-op-mon-onebox01.bj", 15 | "counter": "load.1min", 16 | }, 17 | { 18 | "endpoint": "lg-op-mon-onebox01.bj", 19 | "counter": "load.5min", 20 | }, 21 | { 22 | "endpoint": "lg-op-mon-onebox01.bj", 23 | "counter": "load.15min", 24 | }, 25 | ], 26 | } 27 | 28 | url = "http://localhost:19966/graph/history" 29 | r = requests.post(url, data=json.dumps(d)) 30 | print r.text 31 | 32 | #curl "http://query.falcon.miliao.srv:9966/graph/history/one?cf=AVERAGE&endpoint=`hostname`&start=`date -d '1 hours ago' +%s`&counter=load.1min" |python -m json.tool 33 | -------------------------------------------------------------------------------- /test/debug: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ## test home 3 | testdir=$(cd $(dirname $0)/; pwd) 4 | ## word home 5 | workdir=$(dirname $testdir) 6 | cd $workdir 7 | 8 | module=query 9 | app=falcon-$module 10 | pidfile=var/app.pid 11 | logfile=var/app.log 12 | control=./control 13 | httpprex="127.0.0.1:9966" 14 | 15 | ## statistics 16 | function counter(){ 17 | curl -s "$httpprex/counter/all" | python -m json.tool 18 | } 19 | 20 | function conn_pool(){ 21 | curl -s "$httpprex/proc/connpool" 22 | } 23 | 24 | 25 | ## tail 26 | function tail_log(){ 27 | $control tail 28 | } 29 | 30 | ## build 31 | function build(){ 32 | $control build 33 | [ $? -eq 0 ] && echo -e "build ok" || { echo -e "build error"; exit 1; } 34 | } 35 | function start(){ 36 | $control start 37 | } 38 | function stop(){ 39 | $control stop 40 | } 41 | 42 | 43 | action=$1 44 | case $action in 45 | "build") 46 | build 47 | ;; 48 | "start") 49 | start 50 | ;; 51 | "stop") 52 | stop 53 | ;; 54 | "restart") 55 | stop && build && sleep 1 && start 56 | ;; 57 | "tail") 58 | tail_log 59 | ;; 60 | "connpool") 61 | conn_pool 62 | ;; 63 | *) 64 | counter 65 | ;; 66 | esac 67 | 68 | --------------------------------------------------------------------------------