├── LICENSE ├── README.md ├── README_CN.md ├── build.sh ├── config └── config.go ├── go.mod ├── go.sum ├── images └── arch.png ├── main.go ├── message └── message.go ├── probe ├── assembly.go ├── const.go ├── packet.go ├── packet_test.go ├── probe.go ├── sql.go ├── sql_test.go └── worker.go ├── server ├── client.go ├── cluster.go ├── collector.go ├── dispatcher.go ├── hub.go ├── pusher.go └── server.go └── util ├── encode.go ├── mysql.go ├── net.go └── statistics.go /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 yanyu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | #### README: [中文](README_CN.md) 2 | 3 | # Mysql Probe 4 | A distributed MySQL packets capture and report system inspired by [vividcortex](https://www.vividcortex.com/) and [linkedin blog](https://engineering.linkedin.com/blog/2017/09/query-analyzer--a-tool-for-analyzing-mysql-queries-without-overh) 5 | 6 | ## Modules 7 | There is only one component which could run as three mode: 8 | * slave 9 | * master 10 | * standby 11 | 12 | ### Slave 13 | Slave run at the same machine with MySQL. A probe will be started to capture the MySQL query infos, such as SQL, error and execution latency. This is the base component which could run without the support of the others. 14 | 15 | ### Master 16 | Master is responsible for collecting infos from slaves. Aggregated data will be reported by websocket. 17 | 18 | ### Standby 19 | Standby is a special master that runs as the backup of the master. It is only available in gossip cluster mode. 20 | 21 | ## Cluster 22 | There are two cluster modes, **gossip** and **static**. 23 | 24 | ### Gossip Cluster 25 | In gossip mode, nodes are aware of each other, auto failover could be taken by the system. 26 | 27 | #### Interface 28 | * collector("/collector"): A websocket interface for caller to get assembled data from master or slave. 29 | * join("/cluster/join?addr="): A http interface to join current node to a cluster, 'addr' is one of the cluster node's gossip address. 30 | * leave("/cluster/leave"): A http interface to make current node left from its cluster. 31 | * listnodes("/cluster/listnodes") : A http interface to list the topology of current node. 32 | * config-update("/config/update?{key}={value}"): A http interface to config the node dynamiclly. Only 'report\_period\_ms', the sampling freuency of this node, supported currently. 33 | 34 | ### Static Cluster 35 | There are only masters and slaves in static mode. Manual intervention is needed when nodes down. 36 | 37 | #### Interface 38 | 39 | Interfaces both availiable on master and slave: 40 | 41 | * collector("/collector"): A websocket interface for caller to get assembled data from master or slave. 42 | * config-update("/config/update?{key}={value}"): A http interface to config the node dynamiclly. Only 'report\_period\_ms', the sampling freuency of this node, supported currently. 43 | 44 | Interfaces only availiable on master: 45 | 46 | * join("/cluster/join?addr="): A http interface to add a slave to current node. 47 | * leave("/cluster/leave"): A http interface to make the node left from its cluster. All the slave of the node would be removed. 48 | * remove("/cluster/remove?addr="): A http interface to remove a slave from a master. 'addr' is the server address of the slave. 49 | * listnodes("/cluster/listnodes"): A http interface to list the topology of the node. 50 | 51 | ### Build your own cluster 52 | 53 | In most production enviroment, it usually needs a flat and stateless aggregation layer for data processing. The tree structure of the two buildin clusters are more like test modes to some extent. 54 | Users could build their own production enviroment base on slave. With the help of **Pusher**, slave could report data to one member of a group of servers which could be the data processing layer. 55 | ![image](https://github.com/deatheyes/MysqlProbe/blob/master/images/arch.png) 56 | 57 | ## Configuration 58 | The configuration is a yaml file: 59 | 60 | slave: true # true if run as slave. In gossip mode, those nodes not slave are initialized as master. 61 | serverport: 8667 # websocket address the node listen 62 | interval: 10 # report interval, slaves and master(s) will report assembled data periodically by websocket 63 | slowthresholdms: 100 # threshold to record slow query 64 | cluster: 65 | gossip: true # true if run as gossip mode 66 | group: test # cluster name 67 | port: 0 # gossip bind port 68 | probe: 69 | device: lo0,en0 # devices to probe, splited by ',', slave only 70 | port: 3306 # port to probe, slave only 71 | snappylength: 0 # snappy buffer length of the probe, slave only 72 | workers: 2 # number of workers to process probe data, slave only 73 | pusher: 74 | servers: 127.0.0.1:8668,127.0.0.1:8669 # server list splited by ','. pusher will select one server to push data 75 | path: websocket # websocket path 76 | preconnect: true # create connection to all servers 77 | watcher: # watcher is responsible for cache and refresh map of dbname and connection 78 | uname: test # user name for login MySQL 79 | passward: test # passward to login MySQL 80 | websocket: # webscoket config for client and server 81 | writetimeoutms: 1000 # websocket write timeout(ms) 82 | pingtimeouts: 30 # webscoket ping period(s) 83 | reconnectperiods: 10 # websocket reconnect period(s) 84 | maxmessagesize: 16384 # websocket max message size(k) 85 | 86 | ### Global 87 | 88 | * slave: The node's role. 89 | * serverport: Webserver port. Data will be pushed into any clients connected to this server with path '/collector'. 90 | * interval: Data pushing interval. 91 | * slowthresholdms: Threshold for data collector to record detial query infomation(**message.Message**). 92 | 93 | ### Cluster 94 | 95 | This is an optional configuration. By default, gossip will be utilized. If you don't associated the nodes with each other, you can build your own cluster above those standalone slaves. 96 | 97 | * gossip: Cluster mode gossip|static 98 | * group: The lable distinguishs nodes belong to different clusters. 99 | * port: Gossip binding port. Specially, '0' indicates a ramdom port which could be found in the log. 100 | 101 | ### Probe 102 | 103 | Most of configurations of this section ralate to **libpcap**. Only slave node creates probes. Obviously, slave must be deployed at the same machine with MySQL. 104 | 105 | * device: One or multiple interfaces to probe, splited by ','. 106 | * port: MySQL port to probe. Single port supported currently. 107 | * snappylength: Snappy buffer length of the probe. It is suggested to be set to 0 or left aside if you don't know how your system supports this argument. See **Note** for more information. 108 | * workers: Number of workers to process probe data. Probe dispatchs tcp packets to workers by connections. 109 | 110 | ### Pusher 111 | 112 | Compared with the websocket server in **Global Configuation**, **Pusher** is a optional module to push data to one of the servers actively. Pusher is usefull in building your own cluster, For example, targets of the pusher could be your proxy cluster to prepare the data for your storage, dashboard or ML system. 113 | 114 | * servers: Server list to push data. 115 | * path: Url path for websocket. 116 | * preconnect: Create connection to all the servers ahead or not. 117 | 118 | ### Watcher 119 | 120 | Watcher is the module responsible for building map from connection to db. It needs MySQL authority to run 'show processlist'. 121 | 122 | ## Output 123 | Data collected from slave or master will be reported in form of json compressed by snappy. The report contains statistical items: 124 | 125 | * SQL template: A SQL template is a SQL like text without constant condition value. eg. "select * from user where name=:v1". **SQL template is reported periodically as it is a wast of io and storage to transport and store these long strings. Aggregation layer should cache the Key and SQL template mapping, for example, in Redis or Memcached.** 126 | * latency: The execution latency in microsecond. 127 | * timestamp: Request and response timestamps. 128 | * status: Wether sueecssed or not. 129 | 130 | Detail data structure can be found in **message.go** 131 | 132 | ## Note 133 | * On Linux, users may come up with an error 'Activate: can't mmap rx ring: Invalid argument', please refer [here](https://stackoverflow.com/questions/11397367/issue-in-pcap-set-buffer-size) for more detail. 134 | * It must set the device to the logic device if the machine has virtual network adapter with load balancing, or the packets captured from any physical device would be incomplete. 135 | -------------------------------------------------------------------------------- /README_CN.md: -------------------------------------------------------------------------------- 1 | #### README: [English](README.md) 2 | 3 | # Mysql Probe 4 | 受 [vividcortex](https://www.vividcortex.com/) 及 [linkedin blog](https://engineering.linkedin.com/blog/2017/09/query-analyzer--a-tool-for-analyzing-mysql-queries-without-overh) 启发而开发的 MySQL 抓包监控系统。 5 | 6 | ## 模块 7 | 只有一个二进制文件，允许以三种模式运行: 8 | * slave 9 | * master 10 | * standby 11 | 12 | ### Slave 13 | 与 MySQL 同机部署的探针，主要用于抓取 MySQL 请求，并获取 SQL、错误、响应行等基准数据，同时基于这些数据做一些初级计算，例如请求耗时、平均响应时间、99分位数等等。它是整个系统最基础的模块，可以提供除集群外的所有功能，能够独立于其它两种模块运行。 14 | 15 | ### Master 16 | 负责从下层 slave 收集信息，并通过 websocket 汇报聚合后的数据。 17 | 18 | ### Standby 19 | 用于容灾的 master，仅用于 gossip 集群模式。 20 | 21 | ## 集群 22 | 目前集成了两种集群模式：**gossip**、**static** 23 | 24 | ### Gossip 集群 25 | 节点间通过 gossip 互联，从而实现自动容灾。 26 | 27 | #### 接口 28 | * collector("/collector"): 从 master 或者 slave 获取聚合或采集到的数据，websocket 协议。 29 | * join("/cluster/join?addr="): 向集群中加入节点, 'addr' 为已在集群中的其中一个节点的地址，http 协议。 30 | * leave("/cluster/leave"): 从集群中移除当前节点，http 协议。 31 | * listnodes("/cluster/listnodes") : 列出集群拓扑中的所有节点， http 协议。 32 | * config-update("/config/update?{key}={value}"): 动态配置接口，http 协议。目前只有采样频率 'report\_period\_ms' 能够配置。 33 | 34 | ### Static 集群 35 | 此集群模式中只会有 master 和 slave 节点。如果出现节点宕机，需要人工介入。 36 | 37 | #### 接口 38 | 39 | master 及 slave 上均存在的接口: 40 | * collector("/collector"): 从 master 或者 slave 获取聚合或采集到的数据，websocket 协议。 41 | * config-update("/config/update?{key}={value}"): 动态配置接口，http 协议。目前只有采样频率 'report\_period\_ms' 能够配置。 42 | 43 | 仅存在于 master 的接口: 44 | * join("/cluster/join?addr="): 将某一从节点加入集群，http 协议。 45 | * leave("/cluster/leave"): 从集群中移除当前节点，其所有从节点均会被移除，http 协议。 46 | * remove("/cluster/remove?addr="): 删除某一从节点，地址以 'addr' 指示，http 协议。 47 | * listnodes("/cluster/listnodes"): 列出当前集群拓扑，http 协议。 48 | 49 | ### 制定自己的集群 50 | 51 | 上边两种树形结构的集群更多的是实验性质，而大多数生产环境中都需要平滑且无状态的数据处理层。可以基于 slave 制定生产环境，其携带的 **pusher** 模块可以将数据推送至一组用于数据处理服务器中的某一节点。 52 | ![image](https://github.com/deatheyes/MysqlProbe/blob/master/images/arch.png) 53 | 54 | ## 配置 55 | yaml 格式的配置文件: 56 | 57 | slave: true # 是否以 slave 模式运行。在 gossip 集群中所有非 slave 节点最初都会以 master 模式运行。 58 | serverport: 8667 # websocket 端口。 59 | interval: 1 # 数据汇报周期(s)，slave 和 master 会按该周期通过 '/collector' 及 pusher 以 websocket 协议汇报数据。 60 | slowthresholdms: 100 # 慢查询阈值，响应时间超过该值的请求都会标记为慢查询。 61 | cluster: 62 | gossip: true # 是否以 gossip 模式启动。 63 | group: test # 集群名。 64 | port: 0 # gossip 端口，0 为自动选择。 65 | probe: 66 | device: lo0,en0 # 需要监控的设备以 ',' 分隔，仅对 slave 生效。 67 | port: 3306 # 抓包端口， 仅对 slave 生效。 68 | snappylength: 0 # 快照缓冲区长度，仅对 slave 生效。 69 | workers: 2 # 处理抓包数据的协程数，仅对 slave 生效。 70 | pusher: 71 | servers: 127.0.0.1:8668,127.0.0.1:8669 # 以 ',' 分隔的服务节点，pusher 会从中选择一个推送数据，websocket 协议。 72 | path: websocket # 服务节点的 websocket 路径。 73 | preconnect: true # 是否提前创建连接。 74 | watcher: # watcher 用于缓存、更新数据库名与连接信息，通过 127.0.0.1 连接本地 MySQL。 75 | uname: test # MySQL 用户名。 76 | passward: test # MySQL 用户密码。 77 | websocket: # webscoket 配置。 78 | writetimeoutms: 1000 # 写超时(ms)。 79 | pingtimeouts: 30 # ping 超时(s)。 80 | reconnectperiods: 10 # 连接重建等待时间(s)。 81 | maxmessagesize: 16384 # 最大消息体(k)。 82 | 83 | ### 全局配置 84 | 85 | * slave: 节点角色。 86 | * serverport: 服务端口。数据会通过 '/collector' 接口推送给所有连接的客户端。 87 | * interval: 数据推送周期(s)。 88 | * slowthresholdms: 慢查询阈值(ms)。 89 | 90 | ### 集群配置(cluster) 91 | 92 | 此部分配置为可选项，默认会初始化为 gossip 集群。如果需要制定集群，不要通过相关接口添加 gossip 节点即可。 93 | 94 | * gossip: 集群模式。 95 | * group: 集群名，用于避免不同集群节点的错误引入。 96 | * port: gossip 端口，设置为 0 时会自动选择，日志中会打印出实际使用端口。 97 | 98 | ### 采集配置(probe) 99 | 100 | 此部分的大多数配置都与 **libpcap** 相关。只有 slave 会创建 probe，需要与 MySQL 同机部署。 101 | 102 | * device: 需要监控的设备，多个设备以 ',' 分隔。 103 | * port: MySQL 端口，目前仅支持配置一个。 104 | * snappylength: libpcap 的快照缓冲区大小，如果不确定操作系统对 libpcap 的支持程度，建议设置为 0。参考 **注意事项** 获取更多相关信息。 105 | * workers: probe 中用于处理采集数据的协程数目。 106 | 107 | ### 推送器(pusher) 108 | 109 | **pusher** 是一个可选模块，相较于全局配置中的 '/collector' 接口，它会将数据推送给其中一个下游节点。利用它可以制定集群，例如将其下游设置为数据处理的 proxy 集群，proxy 集群完成数据清洗后再导入存储层、数据挖掘、机器学习等系统。 110 | 111 | * servers: 下游服务组，多个节点间以 ',' 分隔。 112 | * path: websocket 路径。 113 | * preconnect: 是否预先创建对所有下游节点的连接。 114 | 115 | ### 监视器(watcher) 116 | 117 | 用于创建、缓存库与连接的映射表，以 127.0.0.1 方式连接 MySQL，须拥有中执行 'show processlist' 的权限。 118 | 119 | ## 输出 120 | slave 或 master 的汇报数据为经过 snappy 压缩过的 json 数据，主要包含以下信息: 121 | 122 | * SQL 模板: 经过格式化的 SQL，如 "select * from user where name=:v1"。**为了节省IO以及存储，SQL 模板是周期性返回的，大部分情况下只会返回 Key(MD5值)，下游聚合逻辑需要缓存 SQL 模板以及其 Key 的对应关系，可以通过 Redis 或者 Memcached 实现这部分逻辑。** 123 | * 延迟: 毫秒维度的请求响应时间。 124 | * 时间戳: 请求、响应包被抓取的时间戳。 125 | * 状态: 成功、失败、响应行等。 126 | 127 | 更具体的信息可以参考 **message.go** 128 | 129 | ## 注意事项 130 | * 在 Linux 环境中，可能会遇到 'Activate: can't mmap rx ring: Invalid argument' 错误， 可以参考 [这里](https://stackoverflow.com/questions/11397367/issue-in-pcap-set-buffer-size) 获取相关信息。 131 | * 如果存在多物理网卡负载均衡模式的逻辑网卡，须将抓包设备设置为逻辑网卡，负载均衡会导致从单一物理网卡上抓取的数据包不全。 132 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # build 3 | DATE=$(date +%FT%T%z) 4 | BRANCH=$(git symbolic-ref --short -q HEAD) 5 | SHA1=$(git rev-parse HEAD) 6 | go build -ldflags "-X main.branch=${BRANCH} -X main.date=${DATE} -X main.sha1=${SHA1}" 7 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "path" 7 | 8 | "gopkg.in/yaml.v2" 9 | ) 10 | 11 | // Config holds the parameters starting probe, server and cluster 12 | type Config struct { 13 | Slave bool `yaml:"slave"` // run as slave in the cluster 14 | Port uint16 `yaml:"serverport"` // server port 15 | Interval uint16 `yaml:"interval"` // report interval 16 | SlowThresholdMs int64 `yaml:"slowthresholdms"` // threshold to record slow query 17 | Cluster Cluster `yaml:"cluster"` // cluster config 18 | Probe Probe `yaml:"probe"` // probe conifg, only slave will start a probe 19 | Pusher Pusher `yaml:"pusher"` // pusher config 20 | Watcher ConnectionWatcher `yaml:"watcher"` // connection watcher config 21 | Websocket Websocket `yaml:"websocket"` // websocket config 22 | Role string `yaml:"-"` // role of this node 23 | Path string `yaml:"-"` // config file path 24 | } 25 | 26 | // Cluster specify the arguments to run cluster 27 | type Cluster struct { 28 | Gossip bool `yaml:"gossip"` // if run as gossip cluster mode 29 | Group string `yaml:"group"` // cluster name 30 | Port uint16 `yaml:"port"` // gossip binding port, random if not set 31 | } 32 | 33 | // Probe specify the arguments to initialize pcap 34 | type Probe struct { 35 | Device string `yaml:"device"` // local devices probe monitor, splited by ',' 36 | Port uint16 `yaml:"port"` // port for bpf filter 37 | SnapLen int32 `yaml:"snappylength"` // snappy buffer length 38 | Workers int `yaml:"workers"` // worker number 39 | } 40 | 41 | // Pusher specify the arguments to create static receiver pool 42 | type Pusher struct { 43 | Servers string `yaml:"servers"` // server list splited by ',' 44 | Path string `yaml:"path"` // websocket path 45 | Preconnect bool `yaml:"preconnect"` // preconnect to all servers 46 | } 47 | 48 | // ConnectionWatcher is the config of util.ConnectionWatcher 49 | type ConnectionWatcher struct { 50 | Uname string `yaml:"uname"` 51 | Password string `yaml:"password"` 52 | } 53 | 54 | // Websocket is the config of websocket client and server 55 | type Websocket struct { 56 | ConnectTimeoutMs int `yaml:"connecttimeoutms"` 57 | WriteTimeoutMs int `yaml:"writetimeoutms"` 58 | PingTimeoutS int `yaml:"pingtimeouts"` 59 | ReconnectPeriodS int `yaml:"reconnectperiods"` 60 | MaxMessageSize int64 `yaml:"maxmessagesize"` 61 | } 62 | 63 | // ReadFile load config from file 64 | func ReadFile(file string) (*Config, error) { 65 | data, err := ioutil.ReadFile(file) 66 | if err != nil { 67 | return nil, err 68 | } 69 | c := &Config{} 70 | if err := yaml.Unmarshal(data, c); err != nil { 71 | return nil, err 72 | } 73 | return c, nil 74 | } 75 | 76 | // ToBytes marshal the config 77 | func ToBytes(config *Config) []byte { 78 | data, _ := yaml.Marshal(config) 79 | return data 80 | } 81 | 82 | // Seeds holds the basic cluster infomation 83 | type Seeds struct { 84 | Epic uint64 `yaml:"epic"` 85 | Name string `yaml:"name"` 86 | Addrs []string `yaml:"seeds"` 87 | Role string `yaml:"role"` 88 | } 89 | 90 | // SeedsFromFile read seed from file 91 | func SeedsFromFile(file string) (*Seeds, error) { 92 | data, err := ioutil.ReadFile(file) 93 | if err != nil { 94 | return nil, err 95 | } 96 | s := &Seeds{} 97 | if err := yaml.Unmarshal(data, s); err != nil { 98 | return nil, err 99 | } 100 | return s, nil 101 | } 102 | 103 | // SeedsToBytes marshal the seeds 104 | func SeedsToBytes(s *Seeds) []byte { 105 | data, _ := yaml.Marshal(s) 106 | return data 107 | } 108 | 109 | // SeedsToFile write the seeds to file 110 | func SeedsToFile(seeds *Seeds, file string) error { 111 | dir := path.Dir(file) 112 | tmpfile, err := ioutil.TempFile(dir, "seeds") 113 | if err != nil { 114 | return err 115 | } 116 | defer os.Remove(tmpfile.Name()) 117 | 118 | content, err := yaml.Marshal(seeds) 119 | if err != nil { 120 | return err 121 | } 122 | 123 | if _, err := tmpfile.Write(content); err != nil { 124 | return err 125 | } 126 | 127 | if err := tmpfile.Close(); err != nil { 128 | return err 129 | } 130 | 131 | if err := os.Rename(tmpfile.Name(), file); err != nil { 132 | return err 133 | } 134 | return nil 135 | } 136 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/deatheyes/MysqlProbe 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/deatheyes/sqlparser v0.0.0-20190910094615-6f1dfd045c7e 7 | github.com/go-sql-driver/mysql v1.7.1 8 | github.com/golang/glog v1.1.2 9 | github.com/golang/snappy v0.0.4 10 | github.com/google/gopacket v1.1.19 11 | github.com/gorilla/websocket v1.5.0 12 | github.com/hashicorp/golang-lru v1.0.2 13 | github.com/hashicorp/memberlist v0.5.0 14 | github.com/pborman/uuid v1.2.1 15 | gopkg.in/yaml.v2 v2.4.0 16 | ) 17 | 18 | require ( 19 | github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da // indirect 20 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c // indirect 21 | github.com/google/uuid v1.0.0 // indirect 22 | github.com/hashicorp/errwrap v1.0.0 // indirect 23 | github.com/hashicorp/go-immutable-radix v1.0.0 // indirect 24 | github.com/hashicorp/go-msgpack v0.5.3 // indirect 25 | github.com/hashicorp/go-multierror v1.0.0 // indirect 26 | github.com/hashicorp/go-sockaddr v1.0.0 // indirect 27 | github.com/miekg/dns v1.1.26 // indirect 28 | github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect 29 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 // indirect 30 | golang.org/x/net v0.0.0-20190923162816-aa69164e4478 // indirect 31 | golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect 32 | ) 33 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= 2 | github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= 3 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 4 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/deatheyes/sqlparser v0.0.0-20190910094615-6f1dfd045c7e h1:+BrCD4Zsz2hYdW6+KaBr0A1NpflRdvkgfI+MzPLNezU= 6 | github.com/deatheyes/sqlparser v0.0.0-20190910094615-6f1dfd045c7e/go.mod h1:wih6+0hrMvhnSzKGm/LUL1kYXTn34mdR/L3eolgXDnw= 7 | github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= 8 | github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= 9 | github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= 10 | github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= 11 | github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= 12 | github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 13 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw= 14 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 15 | github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= 16 | github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= 17 | github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA= 18 | github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 19 | github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= 20 | github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= 21 | github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= 22 | github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 23 | github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= 24 | github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= 25 | github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= 26 | github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= 27 | github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= 28 | github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= 29 | github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= 30 | github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= 31 | github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= 32 | github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= 33 | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 34 | github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= 35 | github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= 36 | github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR/prTM= 37 | github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4mHgHUZ8lrOI0= 38 | github.com/miekg/dns v1.1.26 h1:gPxPSwALAeHJSjarOs00QjVdV9QoBvc1D2ujQUr5BzU= 39 | github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= 40 | github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= 41 | github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= 42 | github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= 43 | github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= 44 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 45 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 46 | github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= 47 | github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= 48 | github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= 49 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 50 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 51 | golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= 52 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= 53 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 54 | golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= 55 | golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 56 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 57 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 58 | golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g= 59 | golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 60 | golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= 61 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 62 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 63 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 64 | golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 65 | golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 66 | golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg= 67 | golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 68 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 69 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 70 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 71 | golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 72 | golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 73 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 74 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 75 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 76 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 77 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 78 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 79 | -------------------------------------------------------------------------------- /images/arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deatheyes/MysqlProbe/721af93d32c2991d9a3eb9677e79c8c8f6cc51e4/images/arch.png -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | "os/signal" 8 | "strings" 9 | 10 | _ "net/http/pprof" 11 | 12 | "github.com/golang/glog" 13 | 14 | "github.com/deatheyes/MysqlProbe/config" 15 | "github.com/deatheyes/MysqlProbe/probe" 16 | "github.com/deatheyes/MysqlProbe/server" 17 | "github.com/deatheyes/MysqlProbe/util" 18 | ) 19 | 20 | var ( 21 | configfile string 22 | version bool 23 | 24 | // version info 25 | sha1 string 26 | date string 27 | branch string 28 | ) 29 | 30 | func showVersion() { 31 | fmt.Printf("mysql probe - author yanyu - https://github.com/deatheyes/MysqlProbe - %s - %s - %s", branch, date, sha1) 32 | } 33 | 34 | func init() { 35 | flag.StringVar(&configfile, "c", "./conf/config.yaml", "yaml `config` file path") 36 | flag.BoolVar(&version, "version", false, "show version") 37 | glog.MaxSize = 1 << 28 38 | } 39 | 40 | func main() { 41 | flag.Parse() 42 | 43 | if version { 44 | showVersion() 45 | return 46 | } 47 | 48 | conf, err := config.ReadFile(configfile) 49 | if err != nil { 50 | glog.Fatalf("load config failed: %v", err) 51 | return 52 | } 53 | 54 | glog.Infof("MySQL Probe - branch: %s, date: %s, sha1: %s", branch, date, sha1) 55 | 56 | glog.Infof("load config done: %s", string(config.ToBytes(conf))) 57 | conf.Path = configfile 58 | // set default report interval if necessary 59 | if conf.Interval == 0 { 60 | conf.Interval = 5 61 | } 62 | 63 | conf.Role = "master" 64 | if conf.Slave { 65 | conf.Role = "slave" 66 | } 67 | 68 | // initilize websocket config 69 | server.InitWebsocketEnv(conf) 70 | 71 | // start server 72 | glog.Infof("run server, role: %v port: %v report period: %v s gossip: %v group: %v", 73 | conf.Role, conf.Port, conf.Interval, conf.Cluster.Gossip, conf.Cluster.Group) 74 | s := server.NewServer(conf) 75 | go s.Run() 76 | 77 | // check if need to start probe 78 | if conf.Slave { 79 | glog.Info("start probe...") 80 | if len(conf.Probe.Device) == 0 { 81 | glog.Fatal("start probe failed, no device specified") 82 | return 83 | } 84 | 85 | // default watcher config 86 | if len(conf.Watcher.Uname) == 0 { 87 | conf.Watcher.Uname = "test" 88 | } 89 | if len(conf.Watcher.Password) == 0 { 90 | conf.Watcher.Password = "test" 91 | } 92 | 93 | glog.Infof("run watcher, uname:%v port: %v", conf.Watcher.Uname, conf.Probe.Port) 94 | w := util.NewConnectionWatcher(conf.Watcher.Uname, conf.Watcher.Password, conf.Probe.Port) 95 | // probe all ports is prohibited 96 | if conf.Probe.Port == 0 { 97 | glog.Fatal("start probe failed, no probe port specified") 98 | } 99 | // set default snappy buffer length if needed 100 | if conf.Probe.SnapLen <= 0 { 101 | conf.Probe.SnapLen = int32(65535) 102 | } 103 | // set default woker number 1 if needed 104 | if conf.Probe.Workers <= 0 { 105 | conf.Probe.Workers = 1 106 | } 107 | 108 | // multi devices support 109 | devices := strings.Split(conf.Probe.Device, ",") 110 | for _, device := range devices { 111 | if len(device) != 0 { 112 | // start probe 113 | glog.Infof("run probe, device: %v snappylength: %v port: %v workers: %v", device, conf.Probe.SnapLen, conf.Probe.Port, conf.Probe.Workers) 114 | p := probe.NewProbe(device, conf.Probe.SnapLen, conf.Probe.Port, conf.Probe.Workers, s.Collector().MessageIn(), w) 115 | if err := p.Init(); err != nil { 116 | glog.Fatalf("init probe failed: %v", err) 117 | return 118 | } 119 | go p.Run() 120 | } 121 | } 122 | } 123 | 124 | interrupt := make(chan os.Signal, 1) 125 | signal.Notify(interrupt, os.Interrupt) 126 | <-interrupt 127 | } 128 | -------------------------------------------------------------------------------- /message/message.go: -------------------------------------------------------------------------------- 1 | package message 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "sync" 7 | "time" 8 | 9 | "github.com/deatheyes/MysqlProbe/util" 10 | ) 11 | 12 | // reponse status 13 | const ( 14 | ServerStatusInTrans uint16 = 0x0001 15 | ServerStatusAutocommit uint16 = 0x0002 16 | ServerStatusResultsExists uint16 = 0x0008 17 | ServerStatusNoGoodIndexUsed uint16 = 0x0010 18 | ServerStatusNoIndexUsed uint16 = 0x0020 19 | ServerStatusCursorExists uint16 = 0x0040 20 | ServerStatusLastRowSnet uint16 = 0x0080 21 | ServerStatusDbDropped uint16 = 0x0100 22 | ServerStatusNoBackslashEscapes uint16 = 0x0200 23 | ServerStatusMetadataChanged uint16 = 0x0400 24 | ServerQueryWasSlow uint16 = 0x0800 25 | ServerPsOutParams uint16 = 0x1000 26 | ServerStatusInTransReadonly uint16 = 0x2000 27 | ServerSessionStateChanged uint16 = 0x4000 28 | ) 29 | 30 | const sampleInterval = 10 * time.Second 31 | 32 | // Message is the info of a sql query 33 | type Message struct { 34 | DB string `json:"-"` // db name 35 | SQL string `json:"-"` // templated sql 36 | Raw string `json:"c,omitempty"` // raw sql 37 | Err bool `json:"d"` // sql process error 38 | ErrMsg string `json:"e,omitempty"` // sql process error message 39 | Errno uint16 `json:"f,omitempty"` // sql process error number 40 | ServerStatus uint16 `json:"g"` // server response status code 41 | AffectRows uint64 `json:"h"` // affect rows 42 | TimestampReq int64 `json:"i"` // timestamp for request package 43 | TimestampRsp int64 `json:"j"` // timestamp for response package 44 | Latency float32 `json:"k"` // latency in millisecond 45 | Vars map[string]string `json:"l,omitempty"` // Vars retrieved form Raw 46 | ServerIP string `json:"-"` // server ip 47 | ServerPort uint16 `json:"-"` // server port 48 | ClientIP string `json:"n"` // client ip 49 | ClientPort uint16 `json:"-"` // client port 50 | AssemblyKey string `json:"-"` // hash key for assembly 51 | UnknownExec bool `json:"-"` // execute command without crossponding sql 52 | } 53 | 54 | // HashKey hash(sql) for map 55 | func (m *Message) HashKey() (hash string, lastreport time.Time, lastsample time.Time) { 56 | var ok bool 57 | if hash, lastreport, lastsample, ok = GlobalSQLHashCache.get(m.SQL); ok { 58 | return 59 | } 60 | return util.Hash(m.SQL), lastreport, lastsample 61 | } 62 | 63 | // AssemblyHashKey hash(db+ip+sql) for map, same as AssembleHashKey() 64 | func (m *Message) AssemblyHashKey() (hash string, lastreport time.Time, lastsample time.Time) { 65 | var ok bool 66 | key := m.DB + m.ServerIP + m.SQL 67 | if hash, lastreport, lastsample, ok = GlobalSQLHashCache.get(key); ok { 68 | return 69 | } 70 | return util.Hash(key), lastreport, lastsample 71 | } 72 | 73 | // SummaryHashKey is the hash key of the corresponding AssemblySummary 74 | func (m *Message) SummaryHashKey() string { 75 | return m.DB + "|" + m.ServerIP 76 | } 77 | 78 | // Summary is a collection of counters and recoreds 79 | type Summary struct { 80 | Key string `json:"a"` // hash key of SQL 81 | SQL string `json:"b,omitempty"` // SQL template 82 | SuccessCount int `json:"c"` // success query number 83 | FailedCount int `json:"d"` // failed query number 84 | LastSeen int64 `json:"e"` // the latest timestamp 85 | SuccCostTotal float32 `json:"f"` // total cost of success query in millisecond 86 | FailedCostTotal float32 `json:"g"` // total cost of failed query in millisecond 87 | NoGoodIndexUsed int64 `json:"h"` // count of SERVER_STATUS_NO_GOOD_INDEX_USED 88 | NoIndexUsed int64 `json:"i"` // count of SERVER_STATUS_NO_INDEX_USED 89 | QueryWasSlow int64 `json:"j"` // count of SERVER_QUERY_WAS_SLOW 90 | QPS *int `json:"k,omitempty"` // qps 91 | AverageLatency *float32 `json:"l,omitempty"` // average latency in millisecond 92 | MinLatency *float32 `json:"m,omitempty"` // min latency in millisecond 93 | MaxLatency *float32 `json:"n,omitempty"` // max latency in millisecond 94 | Latency99 *float32 `json:"o,omitempty"` // latency of 99 quantile in milliseconds 95 | Slow []*Message `json:"p,omitempty"` // slow queries 96 | AssemblyKey string `json:"-"` // hash key for assembly 97 | Sample *Message `json:"r,omitempty"` // one normal query sample 98 | } 99 | 100 | // AddMessage assemble a Message to this summary 101 | func (s *Summary) AddMessage(m *Message, slow bool, client bool) bool { 102 | if m == nil { 103 | return false 104 | } 105 | 106 | // init hash key for speed up 107 | if len(s.Key) == 0 { 108 | now := time.Now() 109 | key, lastreport, lastsample := m.HashKey() 110 | s.Key = key 111 | if !client { 112 | if now.Sub(lastreport) > cacheExpiration { 113 | // report SQL as less as possible 114 | s.SQL = m.SQL 115 | lastreport = now 116 | } 117 | 118 | // take the first message as a sample 119 | s.Sample = m 120 | if now.Sub(lastsample) < sampleInterval { 121 | // Caution: Here modify the 'message' 122 | s.Sample.Raw = "" 123 | } else { 124 | lastsample = now 125 | } 126 | // Caution: s.SQL maybe empty here, use m.SQL instead. 127 | GlobalSQLHashCache.set(m.SQL, key, lastreport, lastsample) 128 | } 129 | s.AssemblyKey = m.AssemblyKey 130 | } 131 | 132 | if m.Err { 133 | s.FailedCount++ 134 | s.FailedCostTotal += m.Latency 135 | } else { 136 | s.SuccessCount++ 137 | s.SuccCostTotal += m.Latency 138 | } 139 | 140 | if s.LastSeen < m.TimestampReq { 141 | s.LastSeen = m.TimestampReq 142 | } 143 | // status flags 144 | if m.ServerStatus&ServerStatusNoIndexUsed != 0 { 145 | s.NoIndexUsed++ 146 | } 147 | if m.ServerStatus&ServerStatusNoGoodIndexUsed != 0 { 148 | s.NoGoodIndexUsed++ 149 | } 150 | if m.ServerStatus&ServerQueryWasSlow != 0 { 151 | s.QueryWasSlow++ 152 | } 153 | // slow query 154 | if slow { 155 | s.Slow = append(s.Slow, m) 156 | } 157 | return true 158 | } 159 | 160 | // SummaryGroup groups summary by sql 161 | type SummaryGroup struct { 162 | Summary map[string]*Summary 163 | } 164 | 165 | func newSummaryGroup() *SummaryGroup { 166 | return &SummaryGroup{Summary: make(map[string]*Summary)} 167 | } 168 | 169 | // MarshalJSON interface 170 | func (s *SummaryGroup) MarshalJSON() ([]byte, error) { 171 | var list []*Summary 172 | for _, v := range s.Summary { 173 | list = append(list, v) 174 | } 175 | 176 | bf := bytes.NewBuffer([]byte{}) 177 | jsonEncoder := json.NewEncoder(bf) 178 | jsonEncoder.SetEscapeHTML(false) 179 | jsonEncoder.Encode(list) 180 | return bf.Bytes(), nil 181 | } 182 | 183 | // UnmarshalJSON interface 184 | func (s *SummaryGroup) UnmarshalJSON(b []byte) error { 185 | var list []*Summary 186 | if err := json.Unmarshal(b, &list); err != nil { 187 | return err 188 | } 189 | 190 | s.Summary = make(map[string]*Summary) 191 | for _, v := range list { 192 | s.Summary[v.Key] = v 193 | } 194 | return nil 195 | } 196 | 197 | // AddMessage asseble a Message to this summary 198 | func (s *SummaryGroup) AddMessage(m *Message, slow bool, client bool) bool { 199 | if m == nil { 200 | return false 201 | } 202 | 203 | key, _, _ := m.HashKey() 204 | v := s.Summary[key] 205 | if v == nil { 206 | v = &Summary{} 207 | s.Summary[key] = v 208 | } 209 | return v.AddMessage(m, slow, client) 210 | } 211 | 212 | // ClientSummaryUnit flattens the ClientSummaryGroup 213 | type ClientSummaryUnit struct { 214 | IP string `json:"a"` 215 | Group *SummaryGroup `json:"b"` 216 | } 217 | 218 | // ClientSummaryGroup is a wrapper of map[string]*SummaryGroup 219 | type ClientSummaryGroup struct { 220 | ClientGroup map[string]*SummaryGroup 221 | } 222 | 223 | func newClientSummaryGroup() *ClientSummaryGroup { 224 | return &ClientSummaryGroup{ 225 | ClientGroup: make(map[string]*SummaryGroup), 226 | } 227 | } 228 | 229 | // MarshalJSON interface 230 | func (g *ClientSummaryGroup) MarshalJSON() ([]byte, error) { 231 | var list []*ClientSummaryUnit 232 | for k, v := range g.ClientGroup { 233 | list = append(list, &ClientSummaryUnit{IP: k, Group: v}) 234 | } 235 | 236 | bf := bytes.NewBuffer([]byte{}) 237 | jsonEncoder := json.NewEncoder(bf) 238 | jsonEncoder.SetEscapeHTML(false) 239 | jsonEncoder.Encode(list) 240 | return bf.Bytes(), nil 241 | } 242 | 243 | // UnmarshalJSON interface 244 | func (g *ClientSummaryGroup) UnmarshalJSON(b []byte) error { 245 | var list []*ClientSummaryUnit 246 | if err := json.Unmarshal(b, &list); err != nil { 247 | return err 248 | } 249 | 250 | g.ClientGroup = make(map[string]*SummaryGroup) 251 | for _, v := range list { 252 | if len(v.Group.Summary) == 0 { 253 | continue 254 | } 255 | g.ClientGroup[v.IP] = v.Group 256 | } 257 | return nil 258 | } 259 | 260 | // AddMessage asseble a Message to this summary 261 | func (g *ClientSummaryGroup) AddMessage(m *Message, slow bool) bool { 262 | if m == nil { 263 | return false 264 | } 265 | 266 | key := m.ClientIP 267 | v := g.ClientGroup[key] 268 | if v == nil { 269 | v = newSummaryGroup() 270 | g.ClientGroup[key] = v 271 | } 272 | return v.AddMessage(m, slow, true) 273 | } 274 | 275 | // AssemblySummary group Summary by db and server ip 276 | type AssemblySummary struct { 277 | DB string `json:"a"` // DB name 278 | IP string `json:"b"` // server ip 279 | LastSeen int64 `json:"c"` // timestamp 280 | Group *SummaryGroup `json:"d"` // summary group by query 281 | Client *ClientSummaryGroup `json:"e"` // summary group by client 282 | UnknownExec int64 `json:"f"` // counter of execute commands without sql 283 | } 284 | 285 | func newAssemblySummary() *AssemblySummary { 286 | return &AssemblySummary{ 287 | Group: newSummaryGroup(), 288 | Client: newClientSummaryGroup(), 289 | UnknownExec: 0, 290 | } 291 | } 292 | 293 | // HashKey for mapping 294 | func (s *AssemblySummary) HashKey() string { 295 | return s.DB + "|" + s.IP 296 | } 297 | 298 | // AddMessage asseble a Message to this summary 299 | func (s *AssemblySummary) AddMessage(m *Message, slow bool) bool { 300 | if m == nil { 301 | return false 302 | } 303 | 304 | if len(s.IP) == 0 { 305 | s.DB = m.DB 306 | s.IP = m.ServerIP 307 | } 308 | 309 | if s.LastSeen < m.TimestampRsp { 310 | s.LastSeen = m.TimestampRsp 311 | } 312 | 313 | if m.UnknownExec { 314 | s.UnknownExec++ 315 | return true 316 | } 317 | 318 | s.Group.AddMessage(m, false, false) 319 | s.Client.AddMessage(m, slow) 320 | 321 | return true 322 | } 323 | 324 | // Report group captured info by DB 325 | type Report struct { 326 | DB map[string]*AssemblySummary 327 | } 328 | 329 | // NewReport create a Report object 330 | func NewReport() *Report { 331 | return &Report{ 332 | DB: make(map[string]*AssemblySummary), 333 | } 334 | } 335 | 336 | // MarshalJSON interface 337 | func (r *Report) MarshalJSON() ([]byte, error) { 338 | bf := bytes.NewBuffer([]byte{}) 339 | jsonEncoder := json.NewEncoder(bf) 340 | jsonEncoder.SetEscapeHTML(false) 341 | jsonEncoder.Encode(r.DB) 342 | return bf.Bytes(), nil 343 | } 344 | 345 | // UnmarshalJSON interface 346 | func (r *Report) UnmarshalJSON(b []byte) error { 347 | return json.Unmarshal(b, &r.DB) 348 | } 349 | 350 | // Merge assemble another Report to this one 351 | func (r *Report) Merge(ar *Report) { 352 | if r == nil { 353 | return 354 | } 355 | 356 | // data error if there is a override 357 | for k, v := range ar.DB { 358 | r.DB[k] = v 359 | } 360 | } 361 | 362 | // AddMessage asseble a Message to this Report 363 | func (r *Report) AddMessage(m *Message, slow bool) bool { 364 | if m == nil { 365 | return false 366 | } 367 | 368 | d := r.DB[m.DB] 369 | if d == nil { 370 | d = newAssemblySummary() 371 | r.DB[m.DB] = d 372 | } 373 | return d.AddMessage(m, slow) 374 | } 375 | 376 | // Reset collect Messages to reuse 377 | func (r *Report) Reset() { 378 | r.DB = make(map[string]*AssemblySummary) 379 | } 380 | 381 | // DecodeReportFromBytes unmarshal bytes to a Report 382 | func DecodeReportFromBytes(data []byte) (*Report, error) { 383 | r := &Report{} 384 | if err := json.Unmarshal(data, r); err != nil { 385 | return nil, err 386 | } 387 | return r, nil 388 | } 389 | 390 | // EncodeReportToBytes marshal a Report to bytes 391 | func EncodeReportToBytes(r *Report) ([]byte, error) { 392 | return r.MarshalJSON() 393 | } 394 | 395 | const cacheExpiration = time.Minute * 5 396 | const cacheCleanInterval = time.Minute * 30 397 | 398 | // GlobalSQLHashCache is cache the SQL=>Hash(SQL) pairs 399 | var GlobalSQLHashCache = newSQLHashCache() 400 | 401 | // SQLHashCache cache the SQL=>MD5(SQL) pairs 402 | type SQLHashCache struct { 403 | cache map[string]*cacheItem 404 | sync.RWMutex 405 | } 406 | 407 | type cacheItem struct { 408 | value string 409 | lastreport time.Time 410 | lastsample time.Time 411 | } 412 | 413 | func newSQLHashCache() *SQLHashCache { 414 | c := &SQLHashCache{ 415 | cache: make(map[string]*cacheItem), 416 | } 417 | go c.run() 418 | return c 419 | } 420 | 421 | func (c *SQLHashCache) get(key string) (value string, lastreport time.Time, lastsample time.Time, ok bool) { 422 | c.RLock() 423 | defer c.RUnlock() 424 | 425 | item, ok := c.cache[key] 426 | if ok { 427 | return item.value, item.lastreport, item.lastsample, ok 428 | } 429 | return 430 | } 431 | 432 | func (c *SQLHashCache) set(key string, value string, report time.Time, sample time.Time) { 433 | c.Lock() 434 | defer c.Unlock() 435 | 436 | if _, ok := c.cache[key]; !ok { 437 | c.cache[key] = &cacheItem{value, report, sample} 438 | } else { 439 | c.cache[key].value = value 440 | c.cache[key].lastreport = report 441 | c.cache[key].lastsample = sample 442 | } 443 | } 444 | 445 | func (c *SQLHashCache) setValue(key string, value string) { 446 | c.Lock() 447 | defer c.Unlock() 448 | 449 | if _, ok := c.cache[key]; !ok { 450 | c.cache[key] = &cacheItem{value: value} 451 | } else { 452 | c.cache[key].value = value 453 | } 454 | } 455 | 456 | func (c *SQLHashCache) setLastReport(key string, lastreport time.Time) { 457 | c.Lock() 458 | defer c.Unlock() 459 | 460 | if _, ok := c.cache[key]; !ok { 461 | c.cache[key] = &cacheItem{lastreport: lastreport} 462 | } else { 463 | c.cache[key].lastreport = lastreport 464 | } 465 | } 466 | 467 | func (c *SQLHashCache) setLastSample(key string, lastsample time.Time) { 468 | c.Lock() 469 | defer c.Unlock() 470 | 471 | if _, ok := c.cache[key]; !ok { 472 | c.cache[key] = &cacheItem{lastsample: lastsample} 473 | } else { 474 | c.cache[key].lastsample = lastsample 475 | } 476 | } 477 | 478 | func (c *SQLHashCache) run() { 479 | ticker := time.NewTicker(cacheCleanInterval) 480 | for { 481 | <-ticker.C 482 | 483 | c.Lock() 484 | now := time.Now() 485 | for k, v := range c.cache { 486 | if now.Sub(v.lastreport) > cacheCleanInterval && now.Sub(v.lastsample) > cacheCleanInterval { 487 | delete(c.cache, k) 488 | } 489 | } 490 | c.Unlock() 491 | } 492 | } 493 | -------------------------------------------------------------------------------- /probe/assembly.go: -------------------------------------------------------------------------------- 1 | package probe 2 | 3 | import ( 4 | "encoding/binary" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/golang/glog" 9 | "github.com/google/gopacket" 10 | "github.com/google/gopacket/layers" 11 | LRUCache "github.com/hashicorp/golang-lru" 12 | 13 | "github.com/deatheyes/MysqlProbe/message" 14 | "github.com/deatheyes/MysqlProbe/util" 15 | ) 16 | 17 | // Key is the pair of networker and transport Flow 18 | type Key struct { 19 | net, transport gopacket.Flow 20 | } 21 | 22 | func (k Key) String() string { 23 | return fmt.Sprintf("%v:%v", k.net, k.transport) 24 | } 25 | 26 | // IsRequest is a callback set by user to distinguish flow direction. 27 | type IsRequest func(netFlow, tcpFlow gopacket.Flow) bool 28 | 29 | // MysqlStream is a tcp assembly stream wrapper of ReaderStream 30 | type MysqlStream struct { 31 | assembly *Assembly // owner 32 | key Key // hash key 33 | localIP string // server ip 34 | localPort uint16 // server port 35 | clientIP string // client ip 36 | clientPort uint16 // client port 37 | name string // stream name for log 38 | lastSeen time.Time // timestamp of the lastpacket processed 39 | closed bool // close flag 40 | stop chan struct{} // notify close 41 | in chan gopacket.Packet // input channel 42 | dbname string // dbname get from handshake response 43 | uname string // uname get from handshake response 44 | cache *LRUCache.Cache // lru cache of prepare statment 45 | } 46 | 47 | func newMysqlStream(assembly *Assembly, localIP string, localPort uint16, clientIP string, clientPort uint16, key Key) *MysqlStream { 48 | c, _ := LRUCache.New(lruCacheSize) 49 | s := &MysqlStream{ 50 | assembly: assembly, 51 | key: key, 52 | localIP: localIP, 53 | localPort: localPort, 54 | clientIP: clientIP, 55 | clientPort: clientPort, 56 | name: fmt.Sprintf("%v-%v", assembly.wname, key), 57 | closed: false, 58 | stop: make(chan struct{}), 59 | in: make(chan gopacket.Packet, inputQueueLength), 60 | cache: c, 61 | } 62 | go s.run() 63 | return s 64 | } 65 | 66 | func (s *MysqlStream) close() { 67 | if !s.closed { 68 | s.closed = true 69 | close(s.stop) 70 | } 71 | } 72 | 73 | func (s *MysqlStream) run() { 74 | basePacket := &MysqlBasePacket{} 75 | reqPacket := &MysqlRequestPacket{} 76 | rspPacket := &MysqlResponsePacket{} 77 | waitting := false // if there is a request packet parsed 78 | var msg *message.Message 79 | var err error 80 | handshake := false 81 | for { 82 | select { 83 | case packet := <-s.in: 84 | tcp := packet.TransportLayer().(*layers.TCP) 85 | // Ignore empty TCP packets 86 | if !tcp.SYN && !tcp.FIN && !tcp.RST && len(tcp.Payload) == 0 { 87 | continue 88 | } 89 | 90 | key := Key{packet.NetworkLayer().NetworkFlow(), packet.TransportLayer().TransportFlow()} 91 | if s.assembly.isRequest(key.net, key.transport) { 92 | // parse client packet 93 | // Note: there may be many mysql packets in one tcp packet. 94 | // we only care about the first mysql packet, 95 | // which should only be the first part of tcp payload regardless of what the tcp packet seq is. 96 | if _, err = basePacket.DecodeFromBytes(tcp.Payload); err != nil { 97 | glog.V(6).Infof("[%v] parse request base packet failed: %v", s.name, err) 98 | continue 99 | } 100 | 101 | // parse handshake response 102 | if handshake && basePacket.Seq() == mysqlRspSeq { 103 | // this packet should be a handshake response 104 | uname, dbname, err := basePacket.parseHandShakeResponse() 105 | if err != nil { 106 | // maybe not a handshake response 107 | glog.Warningf("[%v] parse handshake response failed: %v", s.name, err) 108 | } else { 109 | glog.V(6).Infof("[%v] parse handshake response done, uname: %v, dbname: %v", s.name, uname, dbname) 110 | s.uname = uname 111 | s.dbname = dbname 112 | waitting = false 113 | if s.cache.Len() > 0 { 114 | s.cache.Purge() 115 | } 116 | handshake = false 117 | } 118 | continue 119 | } 120 | 121 | // filter 122 | if basePacket.Seq() != mysqlReqSeq { 123 | glog.V(8).Infof("[%v] skip unconcerned packet %v", s.name, tcp.Payload) 124 | continue 125 | } 126 | 127 | if err = basePacket.ParseRequestPacket(reqPacket); err != nil { 128 | glog.V(6).Infof("[%v] parse request packet failed: %v", s.name, err) 129 | continue 130 | } 131 | 132 | // reuse message not sent 133 | if msg == nil { 134 | msg = &message.Message{} 135 | } 136 | // parse request and build message 137 | msg.TimestampReq = packet.Metadata().Timestamp.UnixNano() 138 | msg.UnknownExec = false 139 | switch reqPacket.cmd { 140 | case comQuery: 141 | switch reqPacket.queryType { 142 | case queryNormal: 143 | // this is a raw sql query 144 | msg.SQL, msg.Vars = generateQuery(reqPacket.Stmt(), true) 145 | msg.Raw = reqPacket.SQL() 146 | glog.V(6).Infof("[%v] [query][normal] sql: %v", s.name, reqPacket.SQL()) 147 | case queryPrepare: 148 | msg.SQL = reqPacket.SQL() 149 | msg.Raw = "" 150 | glog.V(6).Infof("[%v] [query][prepare] name: %v, sql: %v", s.name, reqPacket.queryName, msg.SQL) 151 | case queryExecute: 152 | stmtName := reqPacket.queryName 153 | if v, ok := s.cache.Get(stmtName); !ok { 154 | msg.UnknownExec = true 155 | glog.V(5).Infof("[%v] [query][execute] no corresponding local statement found, stmtName: %v", s.name, stmtName) 156 | } else { 157 | msg.SQL = v.(string) 158 | msg.Raw = "" 159 | glog.V(6).Infof("[%v] [query][execute] stmtName: %v, sql: %v", s.name, stmtName, msg.SQL) 160 | } 161 | } 162 | case comStmtPrepare: 163 | // the statement will be registered if processed OK 164 | glog.V(6).Infof("[%v] [prepare] sql: %v", s.name, reqPacket.SQL()) 165 | case comStmtExecute: 166 | stmtID := reqPacket.stmtID 167 | if v, ok := s.cache.Get(stmtID); !ok { 168 | // no statement, the corresponding prepare request has not been captured. 169 | msg.UnknownExec = true 170 | glog.V(5).Infof("[%v] [execute] no corresponding local statement found, stmtID: %v", s.name, stmtID) 171 | } else { 172 | msg.SQL = v.(string) 173 | msg.Raw = "" 174 | glog.V(6).Infof("[%v] [execute] stmtID: %v, sql: %v", s.name, stmtID, msg.SQL) 175 | } 176 | case comInitDB: 177 | glog.V(6).Infof("[%v] [init db] dbname: %v", s.name, reqPacket.dbname) 178 | default: 179 | // not the packet concerned, continue 180 | glog.V(8).Infof("[%v] receive unconcerned request packet", s.name) 181 | waitting = false 182 | continue 183 | } 184 | // request ready 185 | waitting = true 186 | } else { 187 | // parse server packet 188 | // Note: there may be many mysql packets in one tcp packet. 189 | // we only care about the first mysql packet, 190 | // which should only be the first part of tcp payload regardless of what the tcp packet seq is. 191 | if _, err = basePacket.DecodeFromBytes(tcp.Payload); err != nil { 192 | glog.V(6).Infof("[%v] parse response base packet failed: %v", s.name, err) 193 | continue 194 | } 195 | 196 | if basePacket.Seq() == mysqlReqSeq { 197 | handshake = true 198 | continue 199 | } 200 | 201 | if !waitting { 202 | // if there is no request, skip this packet ASAP 203 | continue 204 | } 205 | 206 | // filter 207 | if basePacket.Seq() != mysqlRspSeq { 208 | glog.V(8).Infof("[%v] skip unconcerned packet %v", s.name, tcp.Payload) 209 | continue 210 | } 211 | 212 | if err = basePacket.ParseResponsePacket(reqPacket.cmd, rspPacket); err != nil { 213 | glog.V(6).Infof("[%v] parse request packet failed: %v", s.name, err) 214 | continue 215 | } 216 | msg.TimestampRsp = packet.Metadata().Timestamp.UnixNano() 217 | msg.Latency = float32(msg.TimestampRsp-msg.TimestampReq) / 1000000 218 | 219 | // parse reponse and fill message 220 | switch rspPacket.flag { 221 | case iOK: 222 | msg.Err = false 223 | msg.AffectRows = rspPacket.affectedRows 224 | msg.ServerStatus = rspPacket.status 225 | switch reqPacket.CMD() { 226 | case comQuery: 227 | // if is a prepare query, register the SQL. 228 | if reqPacket.queryType == queryPrepare { 229 | s.cache.Remove(reqPacket.queryName) 230 | s.cache.Add(reqPacket.queryName, reqPacket.SQL()) 231 | glog.V(6).Infof("[%v] [query][prepare] response OK, stmtName: %v, sql: %v", s.name, reqPacket.queryName, reqPacket.SQL()) 232 | } 233 | case comStmtPrepare: 234 | // register the prepare statement. 235 | s.cache.Remove(rspPacket.stmtID) 236 | s.cache.Add(rspPacket.stmtID, reqPacket.SQL()) 237 | glog.V(6).Infof("[%v] [prepare] response OK, stmtID: %v, sql: %v", s.name, rspPacket.stmtID, reqPacket.SQL()) 238 | case comInitDB: 239 | s.dbname = reqPacket.dbname 240 | glog.V(6).Infof("[%v] [init db] response OK, dbname: %v", s.name, reqPacket.dbname) 241 | } 242 | case iERR: 243 | msg.Err = true 244 | msg.ErrMsg = rspPacket.message 245 | msg.Errno = rspPacket.errno 246 | default: 247 | // response for SELECT 248 | msg.Err = false 249 | } 250 | 251 | // report 252 | // don't report those message without SQL. 253 | // there is no SQL in prepare message. 254 | // need more precise filter about control command such as START, END. 255 | if (len(msg.SQL) > 5 || msg.UnknownExec) && msg.Latency < maxSpan { 256 | // fill client and server info 257 | msg.ServerIP = s.localIP 258 | msg.ServerPort = s.localPort 259 | msg.ClientIP = s.clientIP 260 | msg.ClientPort = s.clientPort 261 | // set db name 262 | if len(s.dbname) != 0 { 263 | msg.DB = s.dbname 264 | } else { 265 | // find db name 266 | clientAddr := fmt.Sprintf("%s:%v", s.clientIP, s.clientPort) 267 | if info := s.assembly.watcher.Get(clientAddr); info != nil { 268 | msg.DB = string(info.DB) 269 | } else { 270 | msg.DB = unknowDbName 271 | } 272 | } 273 | msg.AssemblyKey, _, _ = msg.AssemblyHashKey() 274 | 275 | glog.V(6).Infof("[%v] mysql query parsed done: %v", s.name, msg.SQL) 276 | 277 | s.assembly.out <- msg 278 | msg = nil 279 | } 280 | waitting = false 281 | } 282 | case <-s.stop: 283 | glog.V(6).Infof("[%v] close stream", s.name) 284 | return 285 | } 286 | } 287 | } 288 | 289 | // Assembly dispatchs packet according to net flow and tcp flow 290 | type Assembly struct { 291 | streamMap map[Key]*MysqlStream // allocated stream 292 | out chan<- *message.Message // channle to report message. 293 | isRequest IsRequest // check if it is a request stream. 294 | wname string // worker name for log. 295 | watcher *util.ConnectionWatcher // wathcer to get connection info 296 | } 297 | 298 | // Assemble send the packet to specify stream 299 | func (a *Assembly) Assemble(packet gopacket.Packet) { 300 | key := Key{packet.NetworkLayer().NetworkFlow(), packet.TransportLayer().TransportFlow()} 301 | reverse := Key{key.net.Reverse(), key.transport.Reverse()} 302 | var s *MysqlStream 303 | if a.streamMap[key] != nil { 304 | s = a.streamMap[key] 305 | } else { 306 | s = a.streamMap[reverse] 307 | } 308 | 309 | if s == nil { 310 | var serverIP, clientIP string 311 | var serverPort, clientPort uint16 312 | if a.isRequest(key.net, key.transport) { 313 | serverIP = key.net.Dst().String() 314 | serverPort = binary.BigEndian.Uint16(key.transport.Dst().Raw()) 315 | clientIP = key.net.Src().String() 316 | clientPort = binary.BigEndian.Uint16(key.transport.Src().Raw()) 317 | } else { 318 | serverIP = key.net.Src().String() 319 | serverPort = binary.BigEndian.Uint16(key.transport.Src().Raw()) 320 | clientIP = key.net.Dst().String() 321 | clientPort = binary.BigEndian.Uint16(key.transport.Dst().Raw()) 322 | } 323 | 324 | s = newMysqlStream(a, serverIP, serverPort, clientIP, clientPort, key) 325 | a.streamMap[key] = s 326 | } 327 | s.lastSeen = packet.Metadata().Timestamp 328 | s.in <- packet 329 | } 330 | 331 | // CloseOlderThan remove those streams expired and return the number of them 332 | func (a *Assembly) CloseOlderThan(t time.Time) int { 333 | count := 0 334 | packetNum := 0 335 | cacheItemNum := 0 336 | for k, v := range a.streamMap { 337 | if v.lastSeen.Before(t) { 338 | count++ 339 | v.close() 340 | delete(a.streamMap, k) 341 | packetNum += len(v.in) 342 | cacheItemNum += v.cache.Len() 343 | } 344 | } 345 | glog.V(3).Infof("[%v] packets: %d, cache items: %d", a.wname, packetNum, cacheItemNum) 346 | return count 347 | } 348 | -------------------------------------------------------------------------------- /probe/const.go: -------------------------------------------------------------------------------- 1 | package probe 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // mysql 8 | const ( 9 | iOK byte = 0x00 10 | iLocalInFile byte = 0xfb 11 | iEOF byte = 0xfe 12 | iERR byte = 0xff 13 | ) 14 | 15 | const ( 16 | comQuit byte = iota + 1 17 | comInitDB 18 | comQuery 19 | comFieldList 20 | comCreateDB 21 | comDropDB 22 | comRefresh 23 | comShutdown 24 | comStatistics 25 | comProcessInfo 26 | comConnect 27 | comProcessKill 28 | comDebug 29 | comPing 30 | comTime 31 | comDelayedInsert 32 | comChangeUser 33 | comBinlogDump 34 | comTableDump 35 | comConnectOut 36 | comRegisterSlave 37 | comStmtPrepare 38 | comStmtExecute 39 | comStmtSendLongData 40 | comStmtClose 41 | comStmtReset 42 | comSetOption 43 | comStmtFetch 44 | ) 45 | 46 | // query type 47 | const ( 48 | queryNormal byte = iota + 1 49 | queryPrepare 50 | queryExecute 51 | ) 52 | 53 | // capability flags 54 | const ( 55 | clientLongPassword uint32 = 1 << iota 56 | clientFoundRows 57 | clientLongFlag 58 | clientConnectWithDB 59 | clientNoSchema 60 | clientCompress 61 | clientODBC 62 | clientLocalFiles 63 | clientIgnoreSpace 64 | clientProtocol41 65 | clientInteractive 66 | clientSSL 67 | clientIgnoreSIGPIPE 68 | clientTransactions 69 | clientReserved 70 | clientSecureConn 71 | clientMultiStatements 72 | clientMultiResults 73 | clientPSMultiResults 74 | clientPluginAuth 75 | clientConnectAttrs 76 | clientPluginAuthLenEncClientData 77 | clientCanHandleExpiredPasswords 78 | clientSessionTrack 79 | clientDeprecateEOF 80 | ) 81 | 82 | // probe 83 | const ( 84 | inputQueueLength = 2000 // stream input queue length 85 | streamExpiration = 150 * time.Second // empty stream expiration 86 | cleanDeviation = 10 // clean deviation in case of IO congestion 87 | unknowDbName = "unknown" // unkonwn db name 88 | ) 89 | 90 | // assembly 91 | const ( 92 | mysqlReqSeq = 0 // mysql request sequence 93 | mysqlRspSeq = 1 // mysql response sequence 94 | maxSpan float32 = 60000 // max response latency（ms）in case of package jam 95 | ) 96 | 97 | // cache 98 | const lruCacheSize = 500 // lru cache size of prepare command 99 | -------------------------------------------------------------------------------- /probe/packet.go: -------------------------------------------------------------------------------- 1 | package probe 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "errors" 7 | "fmt" 8 | 9 | "github.com/deatheyes/sqlparser" 10 | "github.com/golang/glog" 11 | 12 | "github.com/deatheyes/MysqlProbe/util" 13 | ) 14 | 15 | var errNotEnouthData = errors.New("not enough data") 16 | var errParsedFailed = errors.New("parsed failed") 17 | 18 | // MysqlBasePacket is the complete packet with header and payload 19 | type MysqlBasePacket struct { 20 | Header []byte // header 21 | Data []byte // body 22 | } 23 | 24 | // DecodeFromBytes try to decode the first packet from bytes 25 | func (p *MysqlBasePacket) DecodeFromBytes(data []byte) (int, error) { 26 | if len(data) < 4 { 27 | return 0, errNotEnouthData 28 | } 29 | 30 | p.Header = data[0:4] 31 | length := int(uint32(data[0]) | uint32(data[1])<<8 | uint32(data[2])<<16) 32 | 33 | if length+4 > len(data) { 34 | // this packet may not be the first packet of the request or response 35 | glog.V(8).Infof("unexpected data length: %v, required: %v, data: %s", len(data), length+4, data) 36 | return 0, errNotEnouthData 37 | } 38 | p.Data = data[4 : length+4] 39 | return length + 4, nil 40 | } 41 | 42 | // Seq return the Sequence id 43 | func (p *MysqlBasePacket) Seq() byte { 44 | return p.Header[3] 45 | } 46 | 47 | // Length retrun the body length 48 | func (p *MysqlBasePacket) Length() int { 49 | return int(uint32(p.Header[0]) | uint32(p.Header[1])<<8 | uint32(p.Header[2])<<16) 50 | } 51 | 52 | // MysqlRequestPacket retains the infomation of query packet 53 | type MysqlRequestPacket struct { 54 | seq byte 55 | cmd byte 56 | sql []byte 57 | stmtID uint32 // statement id of execute 58 | stmt sqlparser.Statement 59 | dbname string 60 | queryType byte // normal | prepare | execute 61 | queryName string // name of prepare or execute 62 | } 63 | 64 | // Seq return the sequence id in head 65 | func (p *MysqlRequestPacket) Seq() uint8 { 66 | return uint8(p.seq) 67 | } 68 | 69 | // SQL return the sql in query packet 70 | func (p *MysqlRequestPacket) SQL() string { 71 | return string(p.sql) 72 | } 73 | 74 | // Stmt return the AST of the sql in query packet 75 | func (p *MysqlRequestPacket) Stmt() sqlparser.Statement { 76 | return p.stmt 77 | } 78 | 79 | // StmtID return the statement id of a execution request 80 | func (p *MysqlRequestPacket) StmtID() uint32 { 81 | return p.stmtID 82 | } 83 | 84 | // CMD return the request command flag 85 | func (p *MysqlRequestPacket) CMD() byte { 86 | return p.cmd 87 | } 88 | 89 | // MysqlResponsePacket retains the infomation about the response packet of query 90 | type MysqlResponsePacket struct { 91 | seq byte 92 | flag byte 93 | affectedRows uint64 94 | insertID uint64 95 | status uint16 96 | errno uint16 97 | message string 98 | stmtID uint32 99 | } 100 | 101 | // ParseRequestPacket filter out the query packet 102 | func (p *MysqlBasePacket) ParseRequestPacket(packet *MysqlRequestPacket) error { 103 | if len(p.Data) < 2 { 104 | return errNotEnouthData 105 | } 106 | // clean flag 107 | packet.queryType = queryNormal 108 | switch p.Data[0] { 109 | case comQuery: 110 | stmt, err := sqlparser.Parse(string(p.Data[1:])) 111 | if err != nil || stmt == nil { 112 | glog.V(6).Infof("possible not a request packet, prase statement failed: %v", err) 113 | return errParsedFailed 114 | } 115 | packet.seq = p.Seq() 116 | if v, ok := stmt.(*sqlparser.Prepare); ok { 117 | // prepare query 118 | sql, _ := GenerateSourceQuery(v.Stmt) 119 | packet.sql = []byte(sql) 120 | packet.queryType = queryPrepare 121 | packet.queryName, _ = GenerateSourceQuery(v.Name) 122 | packet.stmt = stmt 123 | packet.cmd = comQuery 124 | return nil 125 | } 126 | if v, ok := stmt.(*sqlparser.Execute); ok { 127 | // execute query 128 | packet.queryType = queryExecute 129 | packet.queryName, _ = GenerateSourceQuery(v.Name) 130 | packet.stmt = stmt 131 | packet.cmd = comQuery 132 | return nil 133 | } 134 | if v, ok := stmt.(*sqlparser.Use); ok { 135 | // use dbname 136 | packet.cmd = comInitDB 137 | packet.dbname = v.DBName.String() 138 | return nil 139 | } 140 | // normal query 141 | packet.cmd = comQuery 142 | packet.sql = p.Data[1:] 143 | packet.stmt = stmt 144 | return nil 145 | case comStmtPrepare: 146 | packet.seq = p.Seq() 147 | packet.cmd = comStmtPrepare 148 | packet.sql = p.Data[1:] 149 | return nil 150 | case comStmtExecute: 151 | // we only care about the statement id currently 152 | if len(p.Data) < 5 { 153 | return errNotEnouthData 154 | } 155 | packet.seq = p.Seq() 156 | packet.cmd = comStmtExecute 157 | packet.stmtID = uint32(p.Data[1]) | uint32(p.Data[2])<<8 | uint32(p.Data[3])<<16 | uint32(p.Data[4])<<24 158 | return nil 159 | case comInitDB: 160 | packet.seq = p.Seq() 161 | packet.cmd = comInitDB 162 | packet.dbname = string(p.Data[1:]) 163 | return nil 164 | default: 165 | return errParsedFailed 166 | } 167 | } 168 | 169 | // ParseResponsePacket distinguish OK packet, Err packet and Result set Packet 170 | func (p *MysqlBasePacket) ParseResponsePacket(reqType byte, packet *MysqlResponsePacket) (err error) { 171 | // possible panic while processing length encoding, reover 172 | defer func() { 173 | if r := recover(); r != nil { 174 | glog.Warningf("[recover] parse response failed: %v", r) 175 | err = r.(error) 176 | } 177 | }() 178 | 179 | if len(p.Data) < 1 { 180 | return errNotEnouthData 181 | } 182 | switch reqType { 183 | case comQuery: 184 | return p.parseResultSetHeader(packet) 185 | case comStmtPrepare: 186 | return p.parsePrepare(packet) 187 | case comStmtExecute: 188 | return p.parseResultSetHeader(packet) 189 | case comInitDB: 190 | return p.parseResultSetHeader(packet) 191 | default: 192 | return errParsedFailed 193 | } 194 | } 195 | 196 | func (p *MysqlBasePacket) parsePrepareOK(packet *MysqlResponsePacket) error { 197 | packet.flag = p.Data[0] 198 | if len(p.Data) != 12 { 199 | return errParsedFailed 200 | } 201 | packet.stmtID = binary.LittleEndian.Uint32(p.Data[1:5]) 202 | return nil 203 | } 204 | 205 | func (p *MysqlBasePacket) parseOK(packet *MysqlResponsePacket) error { 206 | var n, m int 207 | packet.flag = p.Data[0] 208 | // OK packet with extend info 209 | packet.affectedRows, _, n = util.ReadLengthEncodedInteger(p.Data[1:]) 210 | packet.insertID, _, m = util.ReadLengthEncodedInteger(p.Data[1+n:]) 211 | packet.status = util.ReadStatus(p.Data[1+n+m : 1+n+m+2]) 212 | return nil 213 | } 214 | 215 | func (p *MysqlBasePacket) parseErr(packet *MysqlResponsePacket) error { 216 | packet.flag = p.Data[0] 217 | packet.errno = binary.LittleEndian.Uint16(p.Data[1:3]) 218 | pos := 3 219 | // SQL State [optional: # + 5bytes string] 220 | if p.Data[3] == 0x23 { 221 | //sqlstate := string(data[4 : 4+5]) 222 | pos = 9 223 | } 224 | packet.message = string(p.Data[pos:]) 225 | return nil 226 | } 227 | 228 | func (p *MysqlBasePacket) parseLocalInFile(packet *MysqlResponsePacket) error { 229 | packet.seq = p.Seq() 230 | packet.flag = p.Data[0] 231 | return nil 232 | } 233 | 234 | func (p *MysqlBasePacket) parseResultSetHeader(packet *MysqlResponsePacket) error { 235 | switch p.Data[0] { 236 | case iOK: 237 | return p.parseOK(packet) 238 | case iERR: 239 | return p.parseErr(packet) 240 | case iLocalInFile: 241 | return p.parseLocalInFile(packet) 242 | } 243 | 244 | // column count 245 | _, _, n := util.ReadLengthEncodedInteger(p.Data) 246 | if n-len(p.Data) == 0 { 247 | packet.seq = p.Seq() 248 | packet.flag = p.Data[0] 249 | return nil 250 | } 251 | return errParsedFailed 252 | } 253 | 254 | func (p *MysqlBasePacket) parsePrepare(packet *MysqlResponsePacket) error { 255 | switch p.Data[0] { 256 | case iOK: 257 | return p.parsePrepareOK(packet) 258 | case iERR: 259 | return p.parseErr(packet) 260 | default: 261 | return errParsedFailed 262 | } 263 | } 264 | 265 | // https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::HandshakeResponse 266 | func (p *MysqlBasePacket) parseHandShakeResponse320(capabilities uint32) (uname string, dbname string, err error) { 267 | pos := 2 + 3 268 | // uname string[0x00] 269 | unameEndIndex := bytes.IndexByte(p.Data[pos:], 0x00) 270 | if unameEndIndex < 0 { 271 | err = errors.New("[handshake response320] failed in parsing uname") 272 | return 273 | } 274 | uname = string(p.Data[pos : pos+unameEndIndex]) 275 | pos += unameEndIndex + 1 276 | 277 | if capabilities&clientConnectWithDB != 0 { 278 | // auth response string[0x00] 279 | authEndIndex := bytes.IndexByte(p.Data[pos:], 0x00) 280 | if authEndIndex < 0 { 281 | err = errors.New("[handshake response320] failed in parsing auth response") 282 | return 283 | } 284 | pos += authEndIndex + 1 285 | 286 | // dbname string[0x00] 287 | dbnameEndIndex := bytes.IndexByte(p.Data[pos:], 0x00) 288 | if dbnameEndIndex < 0 { 289 | err = errors.New("[handshake response320] failed in parsing dbname") 290 | return 291 | } 292 | dbname = string(p.Data[pos : pos+dbnameEndIndex]) 293 | pos += dbnameEndIndex + 1 294 | } 295 | // TODO: parse auth-response[EOF], which we don't need currently 296 | return 297 | } 298 | 299 | // https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::HandshakeResponse 300 | func (p *MysqlBasePacket) parseHandShakeResponse41(capabilities uint32) (uname string, dbname string, err error) { 301 | pos := 4 + 4 + 1 + 23 302 | if len(p.Data) < pos { 303 | err = fmt.Errorf("[handshake response41] unexpected data length: %v", len(p.Data)) 304 | return 305 | } 306 | 307 | // uname string[0x00] 308 | unameEndIndex := bytes.IndexByte(p.Data[pos:], 0x00) 309 | if unameEndIndex < 0 { 310 | err = errors.New("[handshake response41] failed in parsing uname") 311 | return 312 | } 313 | uname = string(p.Data[pos : pos+unameEndIndex]) 314 | pos += unameEndIndex + 1 315 | 316 | if capabilities&clientPluginAuthLenEncClientData != 0 { 317 | // plugin auth length encode client data 318 | pluginInfoLength, _, m := util.ReadLengthEncodedInteger(p.Data[pos:]) 319 | pos += m + int(pluginInfoLength) 320 | if pos > len(p.Data) { 321 | err = errors.New("[handshake response41] failed in parsing clientPluginAuth") 322 | return 323 | } 324 | } else if capabilities&clientSecureConn != 0 { 325 | // client secure connection 326 | secureInfoLength, _, m := util.ReadLengthEncodedInteger(p.Data[pos:]) 327 | pos += m + int(secureInfoLength) 328 | if pos > len(p.Data) { 329 | err = errors.New("[handshake response41] failed in parsing clientSecureConn") 330 | return 331 | } 332 | } else { 333 | // auth response string[0x00] 334 | stringEndIndex := bytes.IndexByte(p.Data[pos:], 0x00) 335 | if stringEndIndex < 0 { 336 | err = errors.New("[handshake response41] failed in parsing auth-response") 337 | return 338 | } 339 | pos += stringEndIndex + 1 340 | } 341 | 342 | if capabilities&clientConnectWithDB != 0 { 343 | // dbname string[0x00] 344 | dbnameEndIndex := bytes.IndexByte(p.Data[pos:], 0x00) 345 | if dbnameEndIndex < 0 { 346 | err = errors.New("[handshake response41] failed in parsing dbname") 347 | return 348 | } 349 | dbname = string(p.Data[pos : pos+dbnameEndIndex]) 350 | pos += dbnameEndIndex + 1 351 | } 352 | 353 | if capabilities&clientPluginAuth != 0 { 354 | // client plugin auth string[0x00] 355 | pluginAuthEndIndex := bytes.IndexByte(p.Data[pos:], 0x00) 356 | if pluginAuthEndIndex < 0 { 357 | err = errors.New("[handshake response41] failed in parsing auth plugin name") 358 | return 359 | } 360 | pos += pluginAuthEndIndex + 1 361 | } 362 | 363 | // TODO: parse client connect attributes, which we don't need currently 364 | /*if capabilities&clientConnectAttrs != 0 { 365 | }*/ 366 | return 367 | } 368 | 369 | func (p *MysqlBasePacket) parseHandShakeResponse() (uname string, dbname string, err error) { 370 | if len(p.Data) < 5 { 371 | err = errNotEnouthData 372 | return 373 | } 374 | 375 | capabilities := uint32(p.Data[0]) | uint32(p.Data[1])<<8 376 | if capabilities&clientProtocol41 == 0 { 377 | return p.parseHandShakeResponse320(capabilities) 378 | } 379 | capabilities = capabilities | uint32(p.Data[2])<<16 | uint32(p.Data[3])<<24 380 | return p.parseHandShakeResponse41(capabilities) 381 | } 382 | -------------------------------------------------------------------------------- /probe/packet_test.go: -------------------------------------------------------------------------------- 1 | package probe 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestParseBasePacket(t *testing.T) { 8 | data := []byte{0x01, 0x00, 0x00, 0x01, 0x00} 9 | packet := &MysqlBasePacket{} 10 | len, err := packet.DecodeFromBytes(data) 11 | if err != nil { 12 | t.Error(err) 13 | } 14 | 15 | want := 5 16 | if len != want { 17 | t.Errorf("unexpected length: %v, want: %v", len, want) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /probe/probe.go: -------------------------------------------------------------------------------- 1 | // Package probe classifies the packets into stream according to network flow and transport flow. 2 | // stream(network1:transport1)\ 3 | // / \ 4 | // / hash(transport flow)- stream(network2:transport1)- \ 5 | // packet-> hash(network flow) messages 6 | // \ hash(transport flow)- stream(network3:transport2)- / 7 | // \ / 8 | // stream(network4:transport2)/ 9 | // | tcp assembly | 10 | package probe 11 | 12 | import ( 13 | "fmt" 14 | "strings" 15 | 16 | "github.com/golang/glog" 17 | "github.com/google/gopacket" 18 | "github.com/google/gopacket/layers" 19 | "github.com/google/gopacket/pcap" 20 | 21 | "github.com/deatheyes/MysqlProbe/message" 22 | "github.com/deatheyes/MysqlProbe/util" 23 | ) 24 | 25 | // Probe need to deloyed at server side. 26 | type Probe struct { 27 | device string 28 | snapLen int32 29 | localIPs []string 30 | port uint16 // probe port. 31 | filter string // bpf filter. 32 | inited bool // flag if could be run. 33 | workers []*Worker // probe worker group processing packet. 34 | workerNum int // worker number. 35 | out chan<- *message.Message // data collect channel. 36 | watcher *util.ConnectionWatcher // db connection watcher. 37 | } 38 | 39 | // NewProbe create a probe to collect and parse packets 40 | func NewProbe(device string, snapLen int32, port uint16, workerNum int, out chan<- *message.Message, watcher *util.ConnectionWatcher) *Probe { 41 | p := &Probe{ 42 | device: device, 43 | snapLen: snapLen, 44 | port: port, 45 | inited: false, 46 | workerNum: workerNum, 47 | out: out, 48 | watcher: watcher, 49 | } 50 | return p 51 | } 52 | 53 | // Init is the preprocess before the probe starts 54 | func (p *Probe) Init() error { 55 | IPs, err := util.GetLocalIPs() 56 | if err != nil { 57 | return err 58 | } 59 | p.localIPs = IPs 60 | slice := []string{} 61 | for _, h := range p.localIPs { 62 | item := fmt.Sprintf("(src host %v and src port %v) or (dst host %v and dst port %v)", h, p.port, h, p.port) 63 | slice = append(slice, item) 64 | } 65 | p.filter = fmt.Sprintf("tcp and host not 127.0.0.1 and (%v)", strings.Join(slice, " or ")) 66 | if p.workerNum <= 0 { 67 | p.workerNum = 1 68 | } 69 | p.inited = true 70 | return nil 71 | } 72 | 73 | // IsRequest distinguish if is a inbound request 74 | func (p *Probe) IsRequest(dstIP string, dstPort uint16) bool { 75 | if dstPort != p.port { 76 | return false 77 | } 78 | for _, ip := range p.localIPs { 79 | if ip == dstIP { 80 | return true 81 | } 82 | } 83 | return false 84 | } 85 | 86 | func (p *Probe) String() string { 87 | return fmt.Sprintf("device: %v, snapshot length: %v, probe port: %v, bpf filter: %v, local IPs: %v, inited: %v, workers: %v", 88 | p.device, p.snapLen, p.port, p.filter, p.localIPs, p.inited, p.workerNum) 89 | } 90 | 91 | // Run starts the probe after inited 92 | func (p *Probe) Run() { 93 | if !p.inited { 94 | glog.Fatal("probe not inited") 95 | return 96 | } 97 | 98 | glog.Infof("probe run - %s", p) 99 | for id := 0; id < p.workerNum; id++ { 100 | p.workers = append(p.workers, NewProbeWorker(p, id)) 101 | } 102 | 103 | // run probe. 104 | handle, err := pcap.OpenLive(p.device, p.snapLen, true, pcap.BlockForever) 105 | if err != nil { 106 | glog.Fatalf("pcap open live failed: %v", err) 107 | return 108 | } 109 | if err := handle.SetBPFFilter(p.filter); err != nil { 110 | glog.Fatalf("set bpf filter failed: %v", err) 111 | return 112 | } 113 | defer handle.Close() 114 | 115 | packetSource := gopacket.NewPacketSource(handle, handle.LinkType()) 116 | packetSource.NoCopy = true 117 | for packet := range packetSource.Packets() { 118 | if packet.NetworkLayer() == nil || packet.TransportLayer() == nil || packet.TransportLayer().LayerType() != layers.LayerTypeTCP { 119 | glog.Warning("unexpected packet") 120 | continue 121 | } 122 | // dispatch packet by stream transport flow. 123 | id := int(packet.TransportLayer().TransportFlow().FastHash() % uint64(p.workerNum)) 124 | p.workers[id].in <- packet 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /probe/sql.go: -------------------------------------------------------------------------------- 1 | package probe 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/deatheyes/sqlparser" 7 | ) 8 | 9 | // templateFormatter replace all the const values to '?' 10 | func templateFormatter(buf *sqlparser.TrackedBuffer, node sqlparser.SQLNode) { 11 | if f, ok := node.(*sqlparser.FuncExpr); ok { 12 | var distinct string 13 | if f.Distinct { 14 | distinct = "distinct " 15 | } 16 | 17 | val := fmt.Sprintf("%s%s", distinct, sqlparser.String(f.Exprs)) 18 | arg := buf.FuncArg() 19 | buf.Vars[arg] = val 20 | buf.Myprintf("%s(%s)", f.Name.String(), arg) 21 | return 22 | } 23 | 24 | if value, ok := node.(*sqlparser.ComparisonExpr); ok { 25 | if value.Operator == sqlparser.InStr || value.Operator == sqlparser.NotInStr { 26 | val := sqlparser.String(value.Right) 27 | arg := buf.VarArg() 28 | buf.Vars[arg] = val 29 | buf.Myprintf("%v %s %s", value.Left, value.Operator, arg) 30 | return 31 | } 32 | } 33 | 34 | if value, ok := node.(*sqlparser.SQLVal); ok { 35 | switch value.Type { 36 | case sqlparser.ValArg: 37 | buf.WriteArg(string(value.Val)) 38 | default: 39 | val := sqlparser.String(value) 40 | arg := buf.VarArg() 41 | buf.Vars[arg] = val 42 | buf.Myprintf("%s", arg) 43 | } 44 | return 45 | } 46 | 47 | if _, ok := node.(*sqlparser.NullVal); ok { 48 | arg := buf.VarArg() 49 | buf.Vars[arg] = "null" 50 | buf.Myprintf("%s", arg) 51 | return 52 | } 53 | 54 | node.Format(buf) 55 | } 56 | 57 | func generateQuery(node sqlparser.SQLNode, template bool) (string, map[string]string) { 58 | var buff *sqlparser.TrackedBuffer 59 | if template { 60 | buff = sqlparser.NewTrackedBuffer(templateFormatter) 61 | } else { 62 | buff = sqlparser.NewTrackedBuffer(nil) 63 | } 64 | node.Format(buff) 65 | return buff.String(), buff.Vars 66 | } 67 | 68 | // GenerateSourceQuery rebuild the query by AST 69 | func GenerateSourceQuery(node sqlparser.SQLNode) (string, map[string]string) { 70 | return generateQuery(node, false) 71 | } 72 | 73 | // GenerateTemplateQuery generate a template according to the AST 74 | func GenerateTemplateQuery(node sqlparser.SQLNode) (string, map[string]string) { 75 | return generateQuery(node, true) 76 | } 77 | -------------------------------------------------------------------------------- /probe/sql_test.go: -------------------------------------------------------------------------------- 1 | package probe 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/deatheyes/sqlparser" 7 | ) 8 | 9 | func TestTransfrom(t *testing.T) { 10 | query := "select * from t where a = 1 and b in (5, 6, 7)" 11 | stmt, err := sqlparser.Parse(query) 12 | if err != nil { 13 | t.Error(err) 14 | } 15 | 16 | template, _ := generateQuery(stmt, true) 17 | want := "select * from t where a = ? and b in ?" 18 | if template != want { 19 | t.Errorf("unexpected result, template: %v , want: %v", template, want) 20 | } 21 | 22 | want = query 23 | result, _ := generateQuery(stmt, false) 24 | if result != want { 25 | t.Errorf("unexpected result, query: %v , want: %v", result, want) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /probe/worker.go: -------------------------------------------------------------------------------- 1 | package probe 2 | 3 | import ( 4 | "encoding/binary" 5 | "fmt" 6 | "math/rand" 7 | "time" 8 | 9 | "github.com/golang/glog" 10 | "github.com/google/gopacket" 11 | 12 | "github.com/deatheyes/MysqlProbe/message" 13 | ) 14 | 15 | // Worker assembles the data from tcp connection dispatched by Probe 16 | type Worker struct { 17 | owner *Probe // owner. 18 | in chan gopacket.Packet // input channel. 19 | out chan<- *message.Message // output channel. 20 | id int // worker id. 21 | name string // worker name for logging. 22 | } 23 | 24 | // NewProbeWorker create a new woker to assemble tcp packets 25 | func NewProbeWorker(probe *Probe, id int) *Worker { 26 | p := &Worker{ 27 | owner: probe, 28 | in: make(chan gopacket.Packet), 29 | out: probe.out, 30 | id: id, 31 | name: fmt.Sprintf("%v-%v", probe.device, id), 32 | } 33 | go p.Run() 34 | return p 35 | } 36 | 37 | // Run initilizes and starts the assembly 38 | func (w *Worker) Run() { 39 | f := func(netFlow, tcpFlow gopacket.Flow) bool { 40 | ip := netFlow.Dst() 41 | port := tcpFlow.Dst() 42 | return w.owner.IsRequest(ip.String(), binary.BigEndian.Uint16(port.Raw())) 43 | } 44 | 45 | assembly := &Assembly{ 46 | streamMap: make(map[Key]*MysqlStream), 47 | out: w.out, 48 | isRequest: f, 49 | wname: w.name, 50 | watcher: w.owner.watcher, 51 | } 52 | 53 | deviation := time.Duration(rand.Intn(cleanDeviation)) * time.Second 54 | ticker := time.NewTicker(streamExpiration + deviation) 55 | defer ticker.Stop() 56 | 57 | glog.Infof("[%v] initilization done, stream expiration: %v", w.name, streamExpiration) 58 | 59 | for { 60 | select { 61 | case packet := <-w.in: 62 | assembly.Assemble(packet) 63 | case <-ticker.C: 64 | // close expired stream 65 | glog.V(8).Infof("[%v] try to close expired streams", w.name) 66 | assembly.CloseOlderThan(time.Now().Add(-streamExpiration)) 67 | } 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /server/client.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | 7 | "github.com/golang/glog" 8 | "github.com/gorilla/websocket" 9 | ) 10 | 11 | var ( 12 | connectTimeout = time.Second 13 | writeTimeout = time.Second 14 | pingTimeout = 30 * time.Second 15 | retryPeriod = 10 * time.Second 16 | maxMessageSize int64 = 1 << 24 17 | ) 18 | 19 | func pingPeriod() time.Duration { 20 | return pingTimeout * 9 / 10 21 | } 22 | 23 | var upgrader = websocket.Upgrader{ 24 | ReadBufferSize: 1 << 24, 25 | WriteBufferSize: 1 << 24, 26 | CheckOrigin: func(r *http.Request) bool { 27 | return true 28 | }, 29 | } 30 | 31 | // Client is a integration of network data used by hub 32 | type Client struct { 33 | hub Hub // hub to register this client 34 | conn *websocket.Conn // websocket connection 35 | send chan []byte // channel of outbound messages 36 | dead bool // flag if the client is closed, used to detect a retry 37 | retry int // count of retry 38 | ping bool // if need to ping the peer 39 | } 40 | 41 | func (c *Client) writePump() { 42 | ticker := time.NewTicker(pingPeriod()) 43 | defer func() { 44 | ticker.Stop() 45 | c.conn.Close() 46 | c.dead = true 47 | }() 48 | 49 | for { 50 | select { 51 | case message, ok := <-c.send: 52 | if !ok { 53 | // channel has been closed by the hub 54 | c.conn.WriteMessage(websocket.CloseMessage, []byte{}) 55 | glog.V(8).Info("channel has been closed by the dispatcher") 56 | return 57 | } 58 | 59 | start := time.Now() 60 | c.conn.SetWriteDeadline(start.Add(writeTimeout)) 61 | if err := c.conn.WriteMessage(websocket.BinaryMessage, message); err != nil { 62 | glog.Warningf("write message failed: %v, cost: %v", err, time.Now().Sub(start)) 63 | return 64 | } 65 | case <-ticker.C: 66 | if c.ping { 67 | c.conn.SetWriteDeadline(time.Now().Add(writeTimeout)) 68 | if err := c.conn.WriteMessage(websocket.PingMessage, nil); err != nil { 69 | // client closed unexpected 70 | glog.Warningf("ping client failed: %v", err) 71 | return 72 | } 73 | } 74 | } 75 | } 76 | } 77 | 78 | func (c *Client) readPump() { 79 | defer func() { 80 | if c.hub != nil { 81 | c.hub.Unregister() <- c 82 | } 83 | c.conn.Close() 84 | c.dead = true 85 | }() 86 | 87 | c.conn.SetReadLimit(maxMessageSize) 88 | c.conn.SetReadDeadline(time.Now().Add(pingTimeout)) 89 | c.conn.SetPongHandler( 90 | func(string) error { 91 | c.conn.SetReadDeadline(time.Now().Add(pingTimeout)) 92 | return nil 93 | }, 94 | ) 95 | 96 | for { 97 | _, data, err := c.conn.ReadMessage() 98 | if err != nil { 99 | if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseNormalClosure) { 100 | glog.Warningf("connection closed unexpected: %v", err) 101 | } else { 102 | glog.Warningf("read data failed: %v", err) 103 | } 104 | return 105 | } 106 | if c.hub != nil { 107 | c.hub.ProcessData(data) 108 | } 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /server/cluster.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "os" 10 | "path" 11 | "strconv" 12 | "strings" 13 | "sync" 14 | "time" 15 | 16 | "github.com/golang/glog" 17 | 18 | "github.com/hashicorp/memberlist" 19 | "github.com/pborman/uuid" 20 | 21 | localConfig "github.com/deatheyes/MysqlProbe/config" 22 | ) 23 | 24 | // lables of role 25 | const ( 26 | NodeRoleSlave = "slave" 27 | NodeRoleMaster = "master" 28 | NodeRoleStandby = "standby" 29 | ) 30 | 31 | // seeds file to store and reload cluster nodes 32 | const ( 33 | SeedsFileName = "seeds.yaml" 34 | ) 35 | 36 | // Broadcast is the implicment of memberlists' Broadcast 37 | type Broadcast struct { 38 | msg []byte 39 | notify chan<- struct{} 40 | } 41 | 42 | func (b *Broadcast) Invalidates(other memberlist.Broadcast) bool { 43 | return false 44 | } 45 | 46 | func (b *Broadcast) Message() []byte { 47 | return b.msg 48 | } 49 | 50 | func (b *Broadcast) Finished() { 51 | if b.notify != nil { 52 | close(b.notify) 53 | } 54 | } 55 | 56 | // MetaData keeps the base infomation of a node 57 | type MetaData struct { 58 | Role string `json:"role"` // master, standy, probe 59 | Epic uint64 `json:"epic"` // epic for message checking 60 | Group string `json:"group"` // group(cluster) name 61 | ServerPort uint16 `json:"server_port"` // dispatcher port 62 | } 63 | 64 | // Node contains the unit's topology 65 | type Node struct { 66 | Name string `json:"name"` 67 | IP string `json:"ip"` 68 | GossipPort uint16 `json:"gossip_port"` 69 | Meta *MetaData `json:"meta"` 70 | } 71 | 72 | // DistributedSystem decide how the cluster works 73 | type DistributedSystem interface { 74 | Run() 75 | Join(addr string) error 76 | Remove(addr string) error 77 | Leave() error 78 | ListNodes() ([]byte, error) 79 | ConfigUpdate(key string, val string) error 80 | } 81 | 82 | // GossipSystem is an auto failure dectect distributed system 83 | type GossipSystem struct { 84 | server *Server // owner 85 | meta *MetaData // local meta 86 | master string // master of current cluster 87 | seeds []string // seeds to join 88 | list *memberlist.Memberlist 89 | broadcasts *memberlist.TransmitLimitedQueue 90 | localIP string 91 | localPort uint16 92 | port uint16 93 | name string // node name in the cluster 94 | seedsFilePath string // seeds file for boot, usually used when restart 95 | 96 | sync.Mutex 97 | } 98 | 99 | // NewGossipSystem create a new gossip based system 100 | func NewGossipSystem(server *Server, role string, group string, port uint16) *GossipSystem { 101 | dir := path.Dir(server.config.Path) 102 | seedsFilePath := path.Join(dir, SeedsFileName) 103 | return &GossipSystem{ 104 | server: server, 105 | meta: &MetaData{Role: role, Epic: 0, Group: group, ServerPort: server.config.Port}, 106 | port: port, 107 | seedsFilePath: seedsFilePath, 108 | } 109 | } 110 | 111 | func (d *GossipSystem) writeSeeds() { 112 | if d.list == nil || d.list.Members() == nil { 113 | // list is not inited 114 | return 115 | } 116 | 117 | var addrs []string 118 | for _, m := range d.list.Members() { 119 | addrs = append(addrs, fmt.Sprintf("%v:%v", m.Addr.String(), m.Port)) 120 | } 121 | 122 | seeds := &localConfig.Seeds{Addrs: addrs, Epic: d.meta.Epic, Name: d.name, Role: d.meta.Role} 123 | 124 | if err := localConfig.SeedsToFile(seeds, d.seedsFilePath); err != nil { 125 | glog.Warningf("write seeds failed: %v", err) 126 | } 127 | glog.V(7).Infof("write seeds to %v done", d.seedsFilePath) 128 | } 129 | 130 | // Run initiliaze and start the system 131 | func (d *GossipSystem) Run() { 132 | hostname, _ := os.Hostname() 133 | config := memberlist.DefaultWANConfig() 134 | config.Delegate = d 135 | config.Events = d 136 | config.Alive = d 137 | config.BindPort = int(d.port) 138 | config.Name = hostname + "-" + uuid.NewUUID().String() 139 | 140 | if d.meta.Role == NodeRoleMaster { 141 | d.master = config.Name 142 | } else { 143 | d.master = "" 144 | } 145 | 146 | // try load seeds from file 147 | if _, err := os.Stat(d.seedsFilePath); os.IsNotExist(err) { 148 | glog.Info("no seeds to start cluster") 149 | d.name = config.Name 150 | list, err := memberlist.Create(config) 151 | if err != nil { 152 | glog.Fatalf("init distributed system failed: %v", err) 153 | } 154 | d.list = list 155 | n := d.list.LocalNode() 156 | d.localIP = n.Addr.String() 157 | d.localPort = uint16(n.Port) 158 | } else { 159 | // override the config with seeds 160 | seeds, err := localConfig.SeedsFromFile(d.seedsFilePath) 161 | if err != nil { 162 | glog.Fatalf("load seeds failed: %v", err) 163 | return 164 | } 165 | 166 | switch seeds.Role { 167 | case NodeRoleMaster, NodeRoleSlave, NodeRoleStandby: 168 | d.meta.Role = seeds.Role 169 | default: 170 | glog.Warningf("unkonwn role: %v", seeds.Role) 171 | } 172 | 173 | d.meta.Epic = seeds.Epic 174 | 175 | if len(seeds.Name) != 0 { 176 | config.Name = seeds.Name 177 | d.name = seeds.Name 178 | } 179 | 180 | list, err := memberlist.Create(config) 181 | if err != nil { 182 | glog.Fatalf("init distributed system with seeds %v failed: %v", seeds, err) 183 | } 184 | d.list = list 185 | n := d.list.LocalNode() 186 | d.localIP = n.Addr.String() 187 | d.localPort = uint16(n.Port) 188 | 189 | // try to join the cluster recorded by seeds 190 | if len(seeds.Addrs) > 0 { 191 | glog.Infof("init distributed system by addresses: %v", seeds.Addrs) 192 | if _, err := d.list.Join(seeds.Addrs); err != nil { 193 | glog.Fatalf("join seeds failed: %v", err) 194 | return 195 | } 196 | } 197 | } 198 | 199 | d.broadcasts = &memberlist.TransmitLimitedQueue{ 200 | NumNodes: func() int { 201 | return d.list.NumMembers() 202 | }, 203 | RetransmitMult: 3, 204 | } 205 | glog.Infof("init distributed system done, local member %s:%d, name: %v", d.localIP, d.localPort, d.name) 206 | } 207 | 208 | // delegate 209 | 210 | // NodeMeta reutrn the binary meta data 211 | func (d *GossipSystem) NodeMeta(limit int) []byte { 212 | data, err := json.Marshal(d.meta) 213 | if err != nil { 214 | glog.Warningf("marshal meta failed: %v", err) 215 | return nil 216 | } 217 | return data 218 | } 219 | 220 | func (d *GossipSystem) NotifyMsg(b []byte) {} 221 | 222 | func (d *GossipSystem) GetBroadcasts(overhead, limit int) [][]byte { 223 | return d.broadcasts.GetBroadcasts(overhead, limit) 224 | } 225 | 226 | func (d *GossipSystem) LocalState(join bool) []byte { 227 | return nil 228 | } 229 | 230 | func (d *GossipSystem) MergeRemoteState(buf []byte, join bool) { 231 | 232 | } 233 | 234 | // OnRoleChanged is a callback to process nodes' role changed 235 | func (d *GossipSystem) OnRoleChanged(oldRole, newRole string) { 236 | glog.V(5).Infof("my role change form %v to %v", oldRole, newRole) 237 | switch newRole { 238 | case NodeRoleMaster: 239 | // start collect data from nodes 240 | d.server.collector.EnableConnection() 241 | // create connection for all slaves 242 | for _, m := range d.list.Members() { 243 | meta := &MetaData{} 244 | if err := json.Unmarshal(m.Meta, meta); err != nil { 245 | glog.Warningf("unmarshal meta failed: %v", err) 246 | continue 247 | } 248 | if meta.Role == NodeRoleSlave { 249 | d.server.collector.AddNode(fmt.Sprintf("%s:%d", m.Addr.String(), meta.ServerPort)) 250 | } 251 | } 252 | default: 253 | // clean all node in collector 254 | d.server.collector.DisableConnection() 255 | } 256 | } 257 | 258 | func (d *GossipSystem) checkPromotion(meta *MetaData, node *memberlist.Node) { 259 | if meta.Role == NodeRoleMaster { 260 | // check if need to update status 261 | if d.meta.Epic < meta.Epic || (d.meta.Epic == meta.Epic && strings.Compare(d.master, node.Name) < 0) { 262 | d.meta.Epic = meta.Epic 263 | d.master = node.Name 264 | if d.meta.Role == NodeRoleMaster { 265 | // switch to standby 266 | d.meta.Role = NodeRoleStandby 267 | d.OnRoleChanged(NodeRoleMaster, NodeRoleStandby) 268 | } 269 | } 270 | } 271 | } 272 | 273 | // event delegate 274 | 275 | // NotifyJoin is called when a node join the cluster 276 | func (d *GossipSystem) NotifyJoin(node *memberlist.Node) { 277 | if node.Name == d.name { 278 | return 279 | } 280 | 281 | meta := &MetaData{} 282 | if err := json.Unmarshal(node.Meta, meta); err != nil { 283 | glog.Warningf("unmarshal meta failed: %v", err) 284 | return 285 | } 286 | 287 | d.Lock() 288 | defer d.Unlock() 289 | 290 | d.checkPromotion(meta, node) 291 | 292 | // see if need to add node to collector 293 | if d.meta.Role == NodeRoleMaster && meta.Role == NodeRoleSlave { 294 | // we are the master, and found a new slave 295 | d.server.collector.AddNode(fmt.Sprintf("%s:%d", node.Addr.String(), meta.ServerPort)) 296 | } 297 | // update seeds 298 | go d.writeSeeds() 299 | } 300 | 301 | // NotifyUpdate is called when a node change its gossip message 302 | func (d *GossipSystem) NotifyUpdate(node *memberlist.Node) { 303 | meta := &MetaData{} 304 | if err := json.Unmarshal(node.Meta, meta); err != nil { 305 | glog.Warningf("unmarshal meta failed: %v", err) 306 | return 307 | } 308 | 309 | d.Lock() 310 | defer d.Unlock() 311 | 312 | d.checkPromotion(meta, node) 313 | // currently, role could not be switched between slave and master|standby. 314 | } 315 | 316 | // NotifyLeave is called when a node leave the gossip cluster 317 | func (d *GossipSystem) NotifyLeave(node *memberlist.Node) { 318 | meta := &MetaData{} 319 | if err := json.Unmarshal(node.Meta, meta); err != nil { 320 | glog.Warningf("unmarshal meta failed: %v", err) 321 | return 322 | } 323 | 324 | d.Lock() 325 | defer d.Unlock() 326 | 327 | if meta.Role == NodeRoleMaster { 328 | // check if need an election, only standby could start an election. 329 | if d.master == node.Name { 330 | // master left 331 | if d.meta.Role == NodeRoleStandby { 332 | // start an election 333 | d.meta.Epic++ 334 | d.meta.Role = NodeRoleMaster 335 | d.OnRoleChanged(NodeRoleStandby, NodeRoleMaster) 336 | } 337 | } 338 | } 339 | 340 | if d.meta.Role == NodeRoleMaster && meta.Role == NodeRoleSlave { 341 | // remove the left slave node from collector 342 | d.server.collector.RemoveNode(fmt.Sprintf("%s:%d", node.Addr.String(), meta.ServerPort)) 343 | } 344 | go d.writeSeeds() 345 | } 346 | 347 | // NotifyAlive is a interface called by the heart beat scheme of memberlist 348 | func (d *GossipSystem) NotifyAlive(peer *memberlist.Node) error { 349 | meta := &MetaData{} 350 | if err := json.Unmarshal(peer.Meta, meta); err != nil { 351 | glog.Warningf("unmarshal meta failed: %v", err) 352 | return fmt.Errorf("unmarshal meta failed: %v", err) 353 | } 354 | 355 | d.Lock() 356 | defer d.Unlock() 357 | 358 | // reject those who is not the same group as us. 359 | if meta.Group != d.meta.Group { 360 | return fmt.Errorf("unexpected gorup: %v", meta.Group) 361 | } 362 | return nil 363 | } 364 | 365 | // Join add a node specified by 'addr' into cluster 366 | func (d *GossipSystem) Join(addr string) error { 367 | if _, err := d.list.Join([]string{addr}); err != nil { 368 | return err 369 | } 370 | return nil 371 | } 372 | 373 | var leavetimeout = 10 * time.Second 374 | 375 | // Leave removes current node from cluster 376 | func (d *GossipSystem) Leave() error { 377 | if err := d.list.Leave(leavetimeout); err != nil { 378 | return err 379 | } 380 | return nil 381 | } 382 | 383 | // Remove is not supported in gossip system 384 | func (d *GossipSystem) Remove(addr string) error { 385 | return errors.New("gossip system don't support this interface") 386 | } 387 | 388 | // ListNodes shows cluster nodes info 389 | func (d *GossipSystem) ListNodes() ([]byte, error) { 390 | nodes := []*Node{} 391 | for _, m := range d.list.Members() { 392 | meta := &MetaData{} 393 | if err := json.Unmarshal(m.Meta, meta); err != nil { 394 | glog.Warningf("unmarshal meta failed: %v", err) 395 | continue 396 | } 397 | nodes = append(nodes, &Node{Name: m.Name, IP: m.Addr.String(), GossipPort: uint16(m.Port), Meta: meta}) 398 | } 399 | 400 | data, err := json.Marshal(nodes) 401 | if err != nil { 402 | return nil, err 403 | } 404 | return data, nil 405 | } 406 | 407 | // ConfigUpdate updates a key-value configuration 408 | func (d *GossipSystem) ConfigUpdate(key string, val string) error { 409 | switch key { 410 | case "report_period_ms": 411 | period, err := strconv.ParseInt(val, 10, 64) 412 | if err != nil { 413 | return err 414 | } 415 | d.server.collector.UpdateReportPeriod(time.Duration(period) * time.Millisecond) 416 | return nil 417 | default: 418 | glog.Warningf("unsupport key: %v", key) 419 | return errors.New("unsupport key") 420 | } 421 | } 422 | 423 | // StaticSystem is a mannually control distributed system 424 | type StaticSystem struct { 425 | server *Server // owner 426 | role string // role of this node 427 | nodes map[string]*Node // slaves' info 428 | group string // cluster group 429 | seedsFilePath string // cluster nodes file, usually used when restart 430 | 431 | sync.Mutex 432 | } 433 | 434 | // NewStaticSystem ... 435 | // there is no standby static system 436 | // slave can only added by master 437 | func NewStaticSystem(server *Server, role string, group string) *StaticSystem { 438 | dir := path.Dir(server.config.Path) 439 | seedsFilePath := path.Join(dir, SeedsFileName) 440 | return &StaticSystem{ 441 | server: server, 442 | role: role, 443 | group: group, 444 | nodes: make(map[string]*Node), 445 | seedsFilePath: seedsFilePath, 446 | } 447 | } 448 | 449 | func (d *StaticSystem) addNodeByAddr(addr string) error { 450 | if _, ok := d.nodes[addr]; !ok { 451 | ss := strings.Split(addr, ":") 452 | if len(ss) != 2 { 453 | return fmt.Errorf("unexpected address: %v", addr) 454 | } 455 | 456 | port, err := strconv.ParseUint(ss[1], 10, 16) 457 | if err != nil { 458 | return err 459 | } 460 | d.server.collector.AddNode(addr) 461 | d.nodes[addr] = &Node{ 462 | Name: addr, 463 | IP: ss[0], 464 | GossipPort: 0, 465 | Meta: &MetaData{ 466 | Role: NodeRoleSlave, 467 | Epic: 0, 468 | ServerPort: uint16(port), 469 | Group: d.group, 470 | }, 471 | } 472 | } 473 | // update cluster file 474 | d.writeSeeds() 475 | return nil 476 | } 477 | 478 | // Run starts the StaticSystem 479 | func (d *StaticSystem) Run() { 480 | // slaves don't need the seeds 481 | if d.role == NodeRoleSlave { 482 | return 483 | } 484 | 485 | // try load cluster from seeds 486 | if _, err := os.Stat(d.seedsFilePath); os.IsNotExist(err) { 487 | glog.Info("no seeds to start cluster") 488 | } else { 489 | seeds, err := localConfig.SeedsFromFile(d.seedsFilePath) 490 | if err != nil { 491 | glog.Fatalf("load seeds failed: %v", err) 492 | } 493 | 494 | for _, addr := range seeds.Addrs { 495 | if err := d.addNodeByAddr(addr); err != nil { 496 | glog.Warningf("load node %v failed: %v", addr, err) 497 | } 498 | } 499 | } 500 | } 501 | 502 | func (d *StaticSystem) writeSeeds() { 503 | var addrs []string 504 | for k := range d.nodes { 505 | addrs = append(addrs, k) 506 | } 507 | 508 | // we must sure not to miss any nodes, so block and update cluster nodes file 509 | seeds := &localConfig.Seeds{Addrs: addrs} 510 | if err := localConfig.SeedsToFile(seeds, d.seedsFilePath); err != nil { 511 | glog.Warningf("write seed to %v failed: %v", d.seedsFilePath, err) 512 | } 513 | glog.V(7).Infof("write seed to %v done", d.seedsFilePath) 514 | } 515 | 516 | // ErrNotMaster returned when the node is not master 517 | var ErrNotMaster = errors.New("not master") 518 | 519 | // Join adds a node specified by 'addr' to current cluster 520 | func (d *StaticSystem) Join(addr string) error { 521 | // only master could join slave 522 | if d.role != NodeRoleMaster { 523 | return ErrNotMaster 524 | } 525 | 526 | d.Lock() 527 | defer d.Unlock() 528 | 529 | return d.addNodeByAddr(addr) 530 | } 531 | 532 | // Leave remove current node from its cluster 533 | func (d *StaticSystem) Leave() error { 534 | if d.role != NodeRoleMaster { 535 | return ErrNotMaster 536 | } 537 | 538 | d.Lock() 539 | defer d.Unlock() 540 | 541 | d.server.collector.DisableConnection() 542 | // update cluster file 543 | d.writeSeeds() 544 | return nil 545 | } 546 | 547 | // Remove delete a node specified by 'addr' from current cluster 548 | func (d *StaticSystem) Remove(addr string) error { 549 | // only master could remove slave 550 | if d.role != NodeRoleMaster { 551 | return ErrNotMaster 552 | } 553 | 554 | d.Lock() 555 | defer d.Unlock() 556 | 557 | if _, ok := d.nodes[addr]; ok { 558 | d.server.collector.RemoveNode(addr) 559 | delete(d.nodes, addr) 560 | } 561 | // update cluster file 562 | d.writeSeeds() 563 | return nil 564 | } 565 | 566 | // ListNodes show cluster topology 567 | func (d *StaticSystem) ListNodes() ([]byte, error) { 568 | if d.role != NodeRoleMaster { 569 | return nil, ErrNotMaster 570 | } 571 | 572 | d.Lock() 573 | defer d.Unlock() 574 | 575 | nodes := []*Node{} 576 | for _, n := range d.nodes { 577 | nodes = append(nodes, n) 578 | } 579 | 580 | data, err := json.Marshal(nodes) 581 | if err != nil { 582 | return nil, err 583 | } 584 | return data, nil 585 | } 586 | 587 | // ConfigUpdate react to dynamic config change 588 | func (d *StaticSystem) ConfigUpdate(key string, val string) error { 589 | switch key { 590 | case "report_period_ms": 591 | // update myself 592 | period, err := strconv.ParseInt(val, 10, 64) 593 | if err != nil { 594 | return err 595 | } 596 | d.server.collector.UpdateReportPeriod(time.Duration(period) * time.Millisecond) 597 | return nil 598 | default: 599 | glog.Warningf("unsupport key: %v", key) 600 | return errors.New("unsupport key") 601 | } 602 | } 603 | 604 | // http handle function 605 | func serveJoin(d DistributedSystem, w http.ResponseWriter, r *http.Request) { 606 | r.ParseForm() 607 | addr := r.Form.Get("addr") 608 | 609 | if err := d.Join(addr); err != nil { 610 | http.Error(w, err.Error(), 500) 611 | return 612 | } 613 | io.WriteString(w, "OK") 614 | } 615 | 616 | func serveLeave(d DistributedSystem, w http.ResponseWriter, r *http.Request) { 617 | if err := d.Leave(); err != nil { 618 | glog.Warningf("leave cluster failed: %v", err) 619 | http.Error(w, err.Error(), 500) 620 | return 621 | } 622 | io.WriteString(w, "OK") 623 | } 624 | 625 | func serveListNodes(d DistributedSystem, w http.ResponseWriter, r *http.Request) { 626 | data, err := d.ListNodes() 627 | if err != nil { 628 | glog.Warningf("list cluster nodes failed: %v", err) 629 | http.Error(w, err.Error(), 500) 630 | return 631 | } 632 | io.WriteString(w, string(data)) 633 | } 634 | 635 | func serveConfigUpdate(d DistributedSystem, w http.ResponseWriter, r *http.Request) { 636 | r.ParseForm() 637 | 638 | key := "report_period_ms" 639 | val := r.Form.Get(key) 640 | 641 | if err := d.ConfigUpdate(key, val); err != nil { 642 | glog.Warningf("update config failed, key: %v val: %v err: %v", key, val, err) 643 | http.Error(w, err.Error(), 500) 644 | return 645 | } 646 | io.WriteString(w, "OK") 647 | } 648 | -------------------------------------------------------------------------------- /server/collector.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "net/url" 5 | "sync" 6 | "time" 7 | 8 | "github.com/golang/glog" 9 | "github.com/golang/snappy" 10 | "github.com/gorilla/websocket" 11 | 12 | "github.com/deatheyes/MysqlProbe/message" 13 | "github.com/deatheyes/MysqlProbe/util" 14 | ) 15 | 16 | const ( 17 | updatePeriod = 10 * time.Second 18 | ) 19 | 20 | // Collector is responsable for assembling data 21 | // master collector gathers info from slaves 22 | // slave collector gathers info from probes 23 | type Collector struct { 24 | clients map[*Client]bool // connection to slaves 25 | clientAddrs map[string]*Client // connection addr to slaves 26 | report chan<- []byte // channel to report 27 | reportIn chan *message.Report // channel to gather report 28 | messageIn chan *message.Message // channel to gather message 29 | stop chan struct{} // channel to stop collector 30 | register chan *Client // client register channel 31 | unregister chan *Client // client unregister channel 32 | registerAddr chan string // cluster node register channel 33 | unregisterAddr chan string // cluster node unregister channel 34 | rejectConnection chan bool // notify to unregister all the connection 35 | reportPeriod time.Duration // period to report and flush merged message 36 | shutdown bool // ture if already stoppted 37 | disableConnection bool // true if disable to accept connection 38 | configChanged bool // reload flag 39 | qps *util.RollingNumber // qps caculator 40 | latency *util.RollingNumber // latency caculator 41 | latencyRange *util.QuantileGroup // latency range caculator 42 | slowThreshold int64 // threshold to record slow querys 43 | 44 | sync.Mutex 45 | } 46 | 47 | // NewCollector create a collecotr 48 | func NewCollector(report chan<- []byte, reportPeriod time.Duration, slowThreshold int64, disableConnection bool) *Collector { 49 | qps, _ := util.NewRollingNumber(10000, 100) 50 | latency, _ := util.NewRollingNumber(10000, 100) 51 | latencyRange := util.NewQuantileGroup(time.Minute, 1000) 52 | return &Collector{ 53 | clients: make(map[*Client]bool), 54 | clientAddrs: make(map[string]*Client), 55 | report: report, 56 | reportIn: make(chan *message.Report, 1000), 57 | messageIn: make(chan *message.Message, 1000), 58 | register: make(chan *Client), 59 | unregister: make(chan *Client), 60 | registerAddr: make(chan string), 61 | unregisterAddr: make(chan string), 62 | stop: make(chan struct{}), 63 | reportPeriod: reportPeriod, 64 | shutdown: false, 65 | disableConnection: disableConnection, 66 | configChanged: false, 67 | qps: qps, 68 | latency: latency, 69 | latencyRange: latencyRange, 70 | slowThreshold: slowThreshold, 71 | } 72 | } 73 | 74 | // UpdateReportPeriod reload the reportPeriod 75 | func (c *Collector) UpdateReportPeriod(reportPeriod time.Duration) { 76 | if c.reportPeriod != reportPeriod { 77 | c.reportPeriod = reportPeriod 78 | c.configChanged = true 79 | } 80 | } 81 | 82 | // DisableConnection clean all client connections 83 | func (c *Collector) DisableConnection() { 84 | c.rejectConnection <- true 85 | } 86 | 87 | // EnableConnection enable and refresh client connections 88 | func (c *Collector) EnableConnection() { 89 | c.rejectConnection <- false 90 | } 91 | 92 | // AddNode add a cluster node specified by addr 93 | func (c *Collector) AddNode(addr string) { 94 | c.registerAddr <- addr 95 | } 96 | 97 | // RemoveNode delete a cluster node specified by addr 98 | func (c *Collector) RemoveNode(addr string) { 99 | c.unregisterAddr <- addr 100 | } 101 | 102 | // node level control 103 | // this function run as fake server 104 | // we need to care about the retry if the client has been unregister but the node not 105 | func (c *Collector) innerupdate() { 106 | ticker := time.NewTicker(retryPeriod) 107 | defer ticker.Stop() 108 | 109 | dialer := &websocket.Dialer{HandshakeTimeout: connectTimeout} 110 | glog.Info("collect innerupdate run...") 111 | for { 112 | select { 113 | case addr := <-c.registerAddr: 114 | // if this isn't a master, do nothing 115 | if c.disableConnection { 116 | glog.Warning("collector has disabled connection") 117 | continue 118 | } 119 | 120 | // add a new node 121 | if _, ok := c.clientAddrs[addr]; !ok { 122 | glog.V(5).Infof("collector adds node: %v", addr) 123 | // create client 124 | u := url.URL{Scheme: "ws", Host: addr, Path: "/collector"} 125 | conn, _, err := dialer.Dial(u.String(), nil) 126 | if err != nil { 127 | glog.Warningf("collector add node %v failed: %v", addr, err) 128 | } else { 129 | client := &Client{hub: c, conn: conn, send: make(chan []byte, 256), dead: false, retry: 0, ping: false} 130 | c.clientAddrs[addr] = client 131 | // register client 132 | c.register <- client 133 | glog.V(5).Infof("collector add node %v done", addr) 134 | go client.writePump() 135 | go client.readPump() 136 | } 137 | } else { 138 | glog.Warningf("collector has added node: %v", addr) 139 | } 140 | case addr := <-c.unregisterAddr: 141 | // remove a node if exists 142 | if client := c.clientAddrs[addr]; client != nil { 143 | glog.V(5).Infof("collector removes node: %v", addr) 144 | c.unregister <- client 145 | delete(c.clientAddrs, addr) 146 | } else { 147 | glog.V(5).Infof("collector cannot remove node %v as it is not in the cluster", addr) 148 | } 149 | case flag := <-c.rejectConnection: 150 | if flag { 151 | // possible node role changed: master -> standby, stop colloect data from nodes 152 | for addr, client := range c.clientAddrs { 153 | glog.V(5).Infof("clean client: %v", addr) 154 | c.unregister <- client 155 | delete(c.clientAddrs, addr) 156 | } 157 | c.disableConnection = true 158 | } else { 159 | c.disableConnection = false 160 | } 161 | case <-ticker.C: 162 | // see if any node need an retry 163 | for k, v := range c.clientAddrs { 164 | if v.dead { 165 | glog.V(5).Infof("collector reconnect to node %v retry: %v", k, v.retry) 166 | u := url.URL{Scheme: "ws", Host: k, Path: "/collector"} 167 | conn, _, err := dialer.Dial(u.String(), nil) 168 | if err != nil { 169 | glog.Warningf("collector reconnect to node %v failed: %v", k, err) 170 | v.retry++ 171 | } else { 172 | glog.V(5).Infof("collector reconnect to node %v success", k) 173 | client := &Client{hub: c, conn: conn, send: make(chan []byte, 256), dead: false, retry: 0, ping: false} 174 | c.clientAddrs[k] = client 175 | // register client 176 | c.register <- client 177 | glog.V(5).Infof("collector reconnect to node %v done", k) 178 | go client.writePump() 179 | go client.readPump() 180 | } 181 | } 182 | } 183 | case <-c.stop: 184 | return 185 | } 186 | } 187 | } 188 | 189 | // Stop shutdown the collector 190 | func (c *Collector) Stop() { 191 | c.Lock() 192 | defer c.Unlock() 193 | if !c.shutdown { 194 | close(c.stop) 195 | c.shutdown = true 196 | } 197 | } 198 | 199 | // ReportIn return the input channel of 'Report' 200 | func (c *Collector) ReportIn() chan<- *message.Report { 201 | return c.reportIn 202 | } 203 | 204 | // MessageIn return the input channel of single 'Message' 205 | func (c *Collector) MessageIn() chan<- *message.Message { 206 | return c.messageIn 207 | } 208 | 209 | // Register submit a client to the client pool 210 | func (c *Collector) Register() chan<- *Client { 211 | return c.register 212 | } 213 | 214 | // Unregister remove a client from client pool 215 | func (c *Collector) Unregister() chan<- *Client { 216 | return c.unregister 217 | } 218 | 219 | // ProcessData decode the report received from slaves 220 | func (c *Collector) ProcessData(data []byte) { 221 | // snappy decode 222 | dst := make([]byte, len(data)*10) 223 | if buf, err := snappy.Decode(dst, data); err != nil { 224 | glog.Warningf("snappy decode failed: %v", err) 225 | } else { 226 | r, err := message.DecodeReportFromBytes(buf) 227 | if err != nil { 228 | glog.Warningf("decode report failed: %v", err) 229 | return 230 | } 231 | // gather reports from remote slaves, this is only avaiable on master 232 | c.reportIn <- r 233 | } 234 | } 235 | 236 | // merge collected reports, used by master and standby master 237 | func (c *Collector) assembleReport(target, slice *message.Report) { 238 | glog.V(8).Info("[collector] merge report") 239 | // merge report 240 | target.Merge(slice) 241 | } 242 | 243 | // merge collected messages, used by slave 244 | func (c *Collector) assembleMessage(target *message.Report, slice *message.Message) { 245 | glog.V(7).Infof("[collector] merge message: %v", slice.SQL) 246 | // merge message 247 | slow := slice.Latency > float32(c.slowThreshold) 248 | target.AddMessage(slice, slow) 249 | // caculate qps 250 | key := slice.AssemblyKey 251 | c.qps.Add(key, 1) 252 | // caculate latency us 253 | c.latency.Add(key, int64(slice.Latency*1000)) 254 | c.latencyRange.Add(key, int64(slice.Latency*1000)) 255 | } 256 | 257 | // Run start the main assembling process on message and report level 258 | func (c *Collector) Run() { 259 | glog.Info("collector start...") 260 | go c.innerupdate() 261 | 262 | report := message.NewReport() 263 | ticker := time.NewTicker(c.reportPeriod) 264 | defer ticker.Stop() 265 | for { 266 | select { 267 | case client := <-c.register: 268 | c.clients[client] = true 269 | case client := <-c.unregister: 270 | if _, ok := c.clients[client]; ok { 271 | delete(c.clients, client) 272 | close(client.send) 273 | } 274 | case r := <-c.reportIn: 275 | c.assembleReport(report, r) 276 | // try to receive more reports 277 | for i := 0; i < len(c.reportIn); i++ { 278 | r = <-c.reportIn 279 | c.assembleReport(report, r) 280 | } 281 | case m := <-c.messageIn: 282 | c.assembleMessage(report, m) 283 | // try to receive more messages 284 | for i := 0; i < len(c.messageIn); i++ { 285 | m = <-c.messageIn 286 | c.assembleMessage(report, m) 287 | } 288 | case <-ticker.C: 289 | glog.V(7).Info("collector flush report") 290 | // report and flush merged data 291 | if c.disableConnection { 292 | // slave need to caculate the average values 293 | for _, db := range report.DB { 294 | for _, s := range db.Group.Summary { 295 | s.QPS = new(int) 296 | *s.QPS = int(c.qps.AverageInSecond(s.AssemblyKey)) 297 | sum := c.qps.Sum(s.AssemblyKey) 298 | if sum != 0 { 299 | s.AverageLatency = new(float32) 300 | *s.AverageLatency = float32(c.latency.Sum(s.AssemblyKey)/sum) / 1000 301 | } 302 | s.MinLatency = new(float32) 303 | s.MaxLatency = new(float32) 304 | s.Latency99 = new(float32) 305 | min, max, q99 := c.latencyRange.Get(s.AssemblyKey) 306 | *s.MinLatency = float32(min) / 1000 307 | *s.MaxLatency = float32(max) / 1000 308 | *s.Latency99 = float32(q99) / 1000 309 | } 310 | } 311 | } 312 | 313 | // report 314 | if len(report.DB) > 0 { 315 | if data, err := message.EncodeReportToBytes(report); err != nil { 316 | glog.Warningf("[collector] encode report failed: %v", err) 317 | } else { 318 | glog.V(8).Infof("[collector] send report %s", string(data)) 319 | // compress and report 320 | dst := make([]byte, len(data)) 321 | c.report <- snappy.Encode(dst, data) 322 | } 323 | report.Reset() 324 | } 325 | 326 | // see if need to refresh the ticker 327 | if c.configChanged { 328 | glog.V(7).Infof("[collector] ticker update: %v", c.reportPeriod) 329 | ticker.Stop() 330 | ticker = time.NewTicker(c.reportPeriod) 331 | c.configChanged = false 332 | } 333 | case <-c.stop: 334 | // stop the collector 335 | for client := range c.clients { 336 | close(client.send) 337 | } 338 | return 339 | } 340 | } 341 | } 342 | -------------------------------------------------------------------------------- /server/dispatcher.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/golang/glog" 7 | ) 8 | 9 | // Dispatcher is a hub to serve websocket, it runs on every node in the cluster 10 | type Dispatcher struct { 11 | clients map[*Client]bool // registered clients 12 | broadcast chan []byte // inbound messages 13 | register chan *Client // client register channel 14 | unregister chan *Client // client unregister channel 15 | pusher *Pusher // pool to push message 16 | } 17 | 18 | // NewDispatcher create a new Dispatcher object 19 | func NewDispatcher(pusher *Pusher) *Dispatcher { 20 | return &Dispatcher{ 21 | clients: make(map[*Client]bool), 22 | broadcast: make(chan []byte), 23 | register: make(chan *Client), 24 | unregister: make(chan *Client), 25 | pusher: pusher, 26 | } 27 | } 28 | 29 | // Run starts the push process 30 | func (d *Dispatcher) Run() { 31 | for { 32 | select { 33 | case client := <-d.register: 34 | // TODO: check if exist 35 | glog.V(6).Info("dispatcher register client") 36 | d.clients[client] = true 37 | case client := <-d.unregister: 38 | glog.V(6).Info("dispatcher unregister client") 39 | if _, ok := d.clients[client]; ok { 40 | delete(d.clients, client) 41 | close(client.send) 42 | } 43 | case message := <-d.broadcast: 44 | glog.V(6).Info("dispatcher receive report") 45 | // push data to dynamic servers 46 | for client := range d.clients { 47 | select { 48 | case client.send <- message: 49 | default: 50 | close(client.send) 51 | delete(d.clients, client) 52 | } 53 | } 54 | // push data to static server pool 55 | if d.pusher != nil { 56 | select { 57 | case d.pusher.send <- message: 58 | default: 59 | glog.Warning("[pusher] queue full") 60 | } 61 | } 62 | } 63 | } 64 | } 65 | 66 | // In return the broadcast channel 67 | func (d *Dispatcher) In() chan<- []byte { 68 | return d.broadcast 69 | } 70 | 71 | // ProcessData of Dispatcher is empty 72 | func (d *Dispatcher) ProcessData(data []byte) { 73 | // dispatcher don't need to read any data 74 | } 75 | 76 | // Register gets the register channel 77 | func (d *Dispatcher) Register() chan<- *Client { 78 | return d.register 79 | } 80 | 81 | // Unregister gets the unregister channel 82 | func (d *Dispatcher) Unregister() chan<- *Client { 83 | return d.unregister 84 | } 85 | 86 | func serveWs(dispatcher *Dispatcher, w http.ResponseWriter, r *http.Request) { 87 | conn, err := upgrader.Upgrade(w, r, nil) 88 | if err != nil { 89 | glog.Warningf("http connection upgrade failed: %v", err) 90 | return 91 | } 92 | 93 | // we don't care about 'dead' and 'retry' of the client in server mode 94 | client := &Client{hub: dispatcher, conn: conn, send: make(chan []byte, 256), ping: true} 95 | client.hub.Register() <- client 96 | go client.writePump() 97 | go client.readPump() 98 | } 99 | -------------------------------------------------------------------------------- /server/hub.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | // Hub is the interface processing Client 4 | type Hub interface { 5 | ProcessData(data []byte) 6 | Register() chan<- *Client 7 | Unregister() chan<- *Client 8 | } 9 | -------------------------------------------------------------------------------- /server/pusher.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "errors" 5 | "math/rand" 6 | "net/url" 7 | "sort" 8 | 9 | "github.com/golang/glog" 10 | "github.com/gorilla/websocket" 11 | ) 12 | 13 | // Slot keep the connection for a server 14 | type Slot struct { 15 | addr string 16 | path string 17 | connections []*Client 18 | dialer *websocket.Dialer 19 | } 20 | 21 | func (s *Slot) Len() int { 22 | return len(s.connections) 23 | } 24 | 25 | func (s *Slot) Less(i, j int) bool { 26 | return !s.connections[i].dead 27 | } 28 | 29 | func (s *Slot) Swap(i, j int) { 30 | s.connections[i], s.connections[j] = s.connections[j], s.connections[i] 31 | } 32 | 33 | func (s *Slot) newClient() (*Client, error) { 34 | u := url.URL{Scheme: "ws", Host: s.addr, Path: s.path} 35 | conn, _, err := s.dialer.Dial(u.String(), nil) 36 | if err != nil { 37 | return nil, err 38 | } 39 | client := &Client{hub: nil, conn: conn, send: make(chan []byte, 256), dead: false, retry: 0, ping: true} 40 | go client.writePump() 41 | go client.readPump() 42 | s.connections = append(s.connections, client) 43 | return client, nil 44 | } 45 | 46 | func (s *Slot) getClient() (*Client, error) { 47 | // clean dead connections 48 | sort.Sort(s) 49 | count := 0 50 | for _, v := range s.connections { 51 | if !v.dead { 52 | count++ 53 | } else { 54 | break 55 | } 56 | } 57 | s.connections = s.connections[:count] 58 | // get a connection 59 | if count == 0 { 60 | return s.newClient() 61 | } 62 | return s.connections[rand.Int()%count], nil 63 | } 64 | 65 | // ClientPool keep the connections for pushing data 66 | // TODO: perference support 67 | type ClientPool struct { 68 | slots []*Slot 69 | dialer *websocket.Dialer 70 | path string 71 | } 72 | 73 | func newClientPool(servers []string, path string, preconnect bool) *ClientPool { 74 | p := &ClientPool{ 75 | dialer: &websocket.Dialer{HandshakeTimeout: connectTimeout}, 76 | path: path, 77 | } 78 | 79 | for _, server := range servers { 80 | slot := &Slot{dialer: p.dialer, path: p.path, addr: server} 81 | p.slots = append(p.slots, slot) 82 | if preconnect { 83 | _, err := slot.getClient() 84 | if err != nil { 85 | glog.Warningf("[pool] preconnect to %v%v failed: %v", slot.addr, slot.path, err) 86 | } 87 | } 88 | } 89 | return p 90 | } 91 | 92 | func (p *ClientPool) getClient() (*Client, error) { 93 | length := len(p.slots) 94 | if length == 0 { 95 | return nil, errors.New("no server list") 96 | } 97 | 98 | pos := rand.Int() % len(p.slots) 99 | for i := 0; i < length; i++ { 100 | pos = (pos + i) % len(p.slots) 101 | slot := p.slots[pos] 102 | client, err := slot.getClient() 103 | if err != nil { 104 | glog.Warningf("[pool] get client of %v/%v failed: %v", slot.addr, slot.path, err) 105 | continue 106 | } else { 107 | return client, nil 108 | } 109 | } 110 | return nil, errors.New("[pool] all servers failed") 111 | } 112 | 113 | // Pusher always push the message to one server in the pool 114 | // TODO: a instance of special Client may be a cute implementation 115 | type Pusher struct { 116 | pool *ClientPool 117 | send chan []byte 118 | } 119 | 120 | func newPusher(servers []string, path string, preconnect bool) *Pusher { 121 | p := &Pusher{ 122 | pool: newClientPool(servers, path, preconnect), 123 | send: make(chan []byte, 256), 124 | } 125 | go p.Run() 126 | return p 127 | } 128 | 129 | // Run start the push process 130 | func (p *Pusher) Run() { 131 | for { 132 | m := <-p.send 133 | client, err := p.pool.getClient() 134 | if err != nil { 135 | glog.Warningf("[pusher] push message failed: %v", err) 136 | } else { 137 | select { 138 | case client.send <- m: 139 | default: 140 | close(client.send) 141 | glog.Warningf("[pusher] push message to %v failed: queue full", client.conn.RemoteAddr()) 142 | } 143 | } 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /server/server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "strings" 7 | "time" 8 | 9 | "github.com/golang/glog" 10 | 11 | "github.com/deatheyes/MysqlProbe/config" 12 | ) 13 | 14 | // InitWebsocketEnv set global config for websocket client and server 15 | func InitWebsocketEnv(config *config.Config) { 16 | if config.Websocket.WriteTimeoutMs > 0 { 17 | writeTimeout = time.Duration(config.Websocket.WriteTimeoutMs) * time.Millisecond 18 | } 19 | 20 | if config.Websocket.PingTimeoutS > 0 { 21 | pingTimeout = time.Duration(config.Websocket.PingTimeoutS) * time.Second 22 | } 23 | 24 | if config.Websocket.ReconnectPeriodS > 0 { 25 | retryPeriod = time.Duration(config.Websocket.ReconnectPeriodS) * time.Second 26 | } 27 | 28 | if config.Websocket.MaxMessageSize > 0 { 29 | maxMessageSize = config.Websocket.MaxMessageSize * 1024 30 | } 31 | 32 | glog.Infof("intialize websocket env done, connect timeout: %v, write timeout: %v, ping timeout: %v, retry period: %v, max message size: %v", 33 | connectTimeout, writeTimeout, pingTimeout, retryPeriod, maxMessageSize) 34 | } 35 | 36 | // Server manage all network input and output 37 | type Server struct { 38 | dispatcher *Dispatcher // dispatcher to serve the client 39 | collector *Collector // collector to gather message 40 | distributedSystem DistributedSystem // distributed system handling topo 41 | config *config.Config // config loaded form file 42 | } 43 | 44 | // NewServer create a server by config 45 | func NewServer(config *config.Config) *Server { 46 | s := &Server{ 47 | dispatcher: NewDispatcher(nil), 48 | config: config, 49 | } 50 | 51 | var pusher *Pusher 52 | if len(config.Pusher.Servers) == 0 { 53 | pusher = nil 54 | } else { 55 | servers := strings.Split(config.Pusher.Servers, ",") 56 | pusher = newPusher(servers, config.Pusher.Path, config.Pusher.Preconnect) 57 | } 58 | s.dispatcher = NewDispatcher(pusher) 59 | 60 | flag := true 61 | if config.Role != NodeRoleSlave { 62 | flag = false 63 | } 64 | s.collector = NewCollector(s.dispatcher.In(), time.Duration(config.Interval)*time.Second, config.SlowThresholdMs, flag) 65 | if config.Cluster.Gossip { 66 | s.distributedSystem = NewGossipSystem(s, config.Role, config.Cluster.Group, config.Cluster.Port) 67 | } else { 68 | s.distributedSystem = NewStaticSystem(s, config.Role, config.Cluster.Group) 69 | } 70 | return s 71 | } 72 | 73 | // Collector get the current instance of Collector 74 | func (s *Server) Collector() *Collector { 75 | return s.collector 76 | } 77 | 78 | // Dispatcher get the current instance of Dispatcher 79 | func (s *Server) Dispatcher() *Dispatcher { 80 | return s.dispatcher 81 | } 82 | 83 | // Run start the Server 84 | func (s *Server) Run() { 85 | s.distributedSystem.Run() 86 | go s.dispatcher.Run() 87 | go s.collector.Run() 88 | 89 | // websocket 90 | http.HandleFunc("/collector", func(w http.ResponseWriter, r *http.Request) { 91 | serveWs(s.dispatcher, w, r) 92 | }) 93 | // cluster control 94 | http.HandleFunc("/cluster/listnodes", func(w http.ResponseWriter, r *http.Request) { 95 | serveListNodes(s.distributedSystem, w, r) 96 | }) 97 | http.HandleFunc("/cluster/join", func(w http.ResponseWriter, r *http.Request) { 98 | serveJoin(s.distributedSystem, w, r) 99 | }) 100 | http.HandleFunc("/cluster/leave", func(w http.ResponseWriter, r *http.Request) { 101 | serveLeave(s.distributedSystem, w, r) 102 | }) 103 | http.HandleFunc("/config/update", func(w http.ResponseWriter, r *http.Request) { 104 | serveConfigUpdate(s.distributedSystem, w, r) 105 | }) 106 | err := http.ListenAndServe(fmt.Sprintf(":%d", s.config.Port), nil) 107 | if err != nil { 108 | glog.Fatalf("listen and serve failed: %v", err) 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /util/encode.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "crypto/md5" 5 | "fmt" 6 | ) 7 | 8 | // ReadLengthEncodedInteger is a length decoder 9 | func ReadLengthEncodedInteger(b []byte) (uint64, bool, int) { 10 | if len(b) == 0 { 11 | return 0, true, 1 12 | } 13 | 14 | switch b[0] { 15 | // 251: NULL 16 | case 0xfb: 17 | return 0, true, 1 18 | 19 | // 252: value of following 2 20 | case 0xfc: 21 | return uint64(b[1]) | uint64(b[2])<<8, false, 3 22 | 23 | // 253: value of following 3 24 | case 0xfd: 25 | return uint64(b[1]) | uint64(b[2])<<8 | uint64(b[3])<<16, false, 4 26 | 27 | // 254: value of following 8 28 | case 0xfe: 29 | return uint64(b[1]) | uint64(b[2])<<8 | uint64(b[3])<<16 | 30 | uint64(b[4])<<24 | uint64(b[5])<<32 | uint64(b[6])<<40 | 31 | uint64(b[7])<<48 | uint64(b[8])<<56, 32 | false, 9 33 | } 34 | 35 | // 0-250: value of first byte 36 | return uint64(b[0]), false, 1 37 | } 38 | 39 | // ReadStatus return the mysql reponse status 40 | func ReadStatus(b []byte) uint16 { 41 | return uint16(b[0]) | uint16(b[1])<<8 42 | } 43 | 44 | // Hash return the hash code of a string 45 | func Hash(key string) string { 46 | return fmt.Sprintf("%x", md5.Sum([]byte(key))) 47 | } 48 | -------------------------------------------------------------------------------- /util/mysql.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "bytes" 5 | "database/sql" 6 | "errors" 7 | "fmt" 8 | "sync" 9 | "time" 10 | 11 | // mysql dialect 12 | _ "github.com/go-sql-driver/mysql" 13 | "github.com/golang/glog" 14 | ) 15 | 16 | const ( 17 | mysqlShowProcessList = "show processlist" 18 | mysql = "mysql" 19 | localhost = "localhost" 20 | infoExpiration = 10 * time.Second // connection info expiration 21 | ) 22 | 23 | // ConnectionWatcher get the connection info by db client and refresh periodically 24 | type ConnectionWatcher struct { 25 | infoMap map[string]*DBConnectionInfo // connection info from db 26 | uname string // db user name 27 | passward string // db passward 28 | port uint16 // db uint16 29 | lastupdate time.Time // update time 30 | lastbytes []byte // byte for comparision 31 | sync.RWMutex 32 | } 33 | 34 | // NewConnectionWatcher create a instance of connection watcher 35 | func NewConnectionWatcher(uname, passward string, port uint16) *ConnectionWatcher { 36 | w := &ConnectionWatcher{ 37 | infoMap: make(map[string]*DBConnectionInfo), 38 | uname: uname, 39 | passward: passward, 40 | port: port, 41 | } 42 | w.init() 43 | return w 44 | } 45 | 46 | func (w *ConnectionWatcher) update() { 47 | m, err := GetMysqlConnectionInfo(w.uname, w.passward, w.port) 48 | if err != nil { 49 | glog.Warningf("[watcher] get connection info failed: %v", err) 50 | } 51 | 52 | var buffer []byte 53 | for _, v := range m { 54 | buffer = append(buffer, v.key()[:]...) 55 | } 56 | 57 | if bytes.Equal(buffer, w.lastbytes) { 58 | if len(buffer) == 0 { 59 | glog.Warning("[watcher] get empty connection info") 60 | } else { 61 | glog.V(6).Info("[watcher] connection info not change") 62 | } 63 | } else { 64 | w.Lock() 65 | w.infoMap = m 66 | w.Unlock() 67 | w.lastbytes = buffer 68 | glog.V(6).Info("[watcher] connection info update done") 69 | } 70 | } 71 | 72 | // Init run the update process 73 | func (w *ConnectionWatcher) init() { 74 | // init connection info first 75 | w.update() 76 | 77 | go func() { 78 | ticker := time.NewTicker(infoExpiration) 79 | for { 80 | <-ticker.C 81 | w.update() 82 | } 83 | }() 84 | } 85 | 86 | // Get return the connection info by key 87 | func (w *ConnectionWatcher) Get(key string) *DBConnectionInfo { 88 | w.RLock() 89 | defer w.RUnlock() 90 | 91 | return w.infoMap[key] 92 | } 93 | 94 | // DBConnectionInfo retrieves the db connection info 95 | type DBConnectionInfo struct { 96 | ID, User, Host, DB, Cmd, Time, State, Info, Sent, Examined []byte 97 | } 98 | 99 | // Key generate the bytes for comparison 100 | func (i *DBConnectionInfo) key() (buffer []byte) { 101 | buffer = append(buffer, i.ID[:]...) 102 | buffer = append(buffer, i.User[:]...) 103 | buffer = append(buffer, i.Host[:]...) 104 | buffer = append(buffer, i.DB[:]...) 105 | return 106 | } 107 | 108 | // GetMysqlConnectionInfo return all db connection info 109 | func GetMysqlConnectionInfo(user string, password string, port uint16) (map[string]*DBConnectionInfo, error) { 110 | str := fmt.Sprintf("%s:%s@tcp(127.0.0.1:%d)/", user, password, port) 111 | db, err := sql.Open(mysql, str) 112 | if err != nil { 113 | return nil, err 114 | } 115 | defer db.Close() 116 | 117 | rows, err := db.Query(mysqlShowProcessList) 118 | if err != nil { 119 | return nil, err 120 | } 121 | 122 | version := 0 123 | cols, err := rows.Columns() 124 | if err != nil { 125 | return nil, err 126 | } 127 | 128 | if len(cols) == 10 { 129 | version = 5 130 | } else if len(cols) == 8 { 131 | version = 8 132 | } 133 | 134 | if version == 0 { 135 | return nil, errors.New("unknown mysql version") 136 | } 137 | 138 | ret := make(map[string]*DBConnectionInfo) 139 | for rows.Next() { 140 | data := &DBConnectionInfo{} 141 | if version == 8 { 142 | if err := rows.Scan(&data.ID, &data.User, &data.Host, &data.DB, &data.Cmd, &data.Time, &data.State, &data.Info); err != nil { 143 | return nil, err 144 | } 145 | } else { 146 | if err := rows.Scan(&data.ID, &data.User, &data.Host, &data.DB, &data.Cmd, &data.Time, &data.State, &data.Info, &data.Sent, &data.Examined); err != nil { 147 | return nil, err 148 | } 149 | } 150 | if len(data.Host) == 0 || string(data.Host) == localhost || len(data.DB) == 0 { 151 | continue 152 | } 153 | glog.V(5).Infof("[watcher] connection %s db %s", data.Host, data.DB) 154 | ret[string(data.Host)] = data 155 | } 156 | return ret, nil 157 | } 158 | -------------------------------------------------------------------------------- /util/net.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "net" 5 | ) 6 | 7 | // GetLocalIPs return all the address 8 | func GetLocalIPs() ([]string, error) { 9 | addrs, err := net.InterfaceAddrs() 10 | if err != nil { 11 | return nil, err 12 | } 13 | 14 | var ret []string 15 | for _, addr := range addrs { 16 | ipnet, ok := addr.(*net.IPNet) 17 | if !ok || ipnet.IP.IsLoopback() { 18 | continue 19 | } 20 | 21 | if ipnet.IP.To4() != nil { 22 | ret = append(ret, ipnet.IP.String()) 23 | } 24 | } 25 | return ret, nil 26 | } 27 | -------------------------------------------------------------------------------- /util/statistics.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "errors" 5 | "math" 6 | "sync" 7 | "time" 8 | ) 9 | 10 | type bucket struct { 11 | startTime time.Time // the time this bucket activated 12 | counters map[string]int64 // counters 13 | } 14 | 15 | func newBucket(startTime time.Time) *bucket { 16 | return &bucket{ 17 | startTime: startTime, 18 | counters: make(map[string]int64), 19 | } 20 | } 21 | 22 | type buckets struct { 23 | ring []*bucket // data slice 24 | pos int // last bucket 25 | bucketSizeInNanoseconds int64 26 | numberOfBuckets int64 27 | timeInNanoseconds int64 28 | } 29 | 30 | func newBuckets(numberOfBuckets, timeInNanoseconds int64) *buckets { 31 | b := &buckets{ 32 | ring: make([]*bucket, numberOfBuckets), 33 | pos: 0, 34 | timeInNanoseconds: timeInNanoseconds, 35 | bucketSizeInNanoseconds: timeInNanoseconds / numberOfBuckets, 36 | numberOfBuckets: numberOfBuckets, 37 | } 38 | 39 | b.ring[0] = newBucket(time.Now()) 40 | 41 | return b 42 | } 43 | 44 | func (b *buckets) peekLast() *bucket { 45 | currentTime := time.Now() 46 | posInc := currentTime.Sub(b.ring[b.pos].startTime).Nanoseconds() / b.bucketSizeInNanoseconds 47 | currentPos := int((int64(b.pos) + posInc) % b.numberOfBuckets) 48 | gap := currentTime.Sub(b.ring[b.pos].startTime).Nanoseconds() % b.bucketSizeInNanoseconds 49 | if b.ring[currentPos] == nil || currentTime.Sub(b.ring[currentPos].startTime).Nanoseconds() > b.bucketSizeInNanoseconds { 50 | // empty bucket or a expired bucket 51 | b.ring[currentPos] = newBucket(currentTime.Add(-time.Duration(gap))) 52 | } 53 | b.pos = currentPos 54 | return b.ring[b.pos] 55 | } 56 | 57 | func (b *buckets) Add(key string, inc int64) { 58 | bucket := b.peekLast() 59 | bucket.counters[key] += int64(inc) 60 | } 61 | 62 | func (b *buckets) Sum(key string) int64 { 63 | var count int64 64 | currentTime := time.Now() 65 | for _, v := range b.ring { 66 | if v == nil { 67 | continue 68 | } 69 | 70 | if currentTime.Sub(v.startTime).Nanoseconds() > b.timeInNanoseconds { 71 | // expired bucket 72 | v = nil 73 | continue 74 | } 75 | 76 | if _, ok := v.counters[key]; ok { 77 | count += v.counters[key] 78 | } 79 | } 80 | return count 81 | } 82 | 83 | type rollingNumberEvent struct { 84 | key string 85 | val int64 86 | op string 87 | } 88 | 89 | // RollingNumber is struct for statistics 90 | type RollingNumber struct { 91 | timeInMilliseconds int64 // assemble interval 92 | numberOfBuckets int64 // assemble accuracy 93 | bucketSizeInMillseconds int64 // time span for each bucket 94 | buckets *buckets // data slices 95 | notify chan *rollingNumberEvent // channel to create a operation 96 | result chan int64 // channel to get operation result 97 | } 98 | 99 | // NewRollingNumber create a new rolling number 100 | func NewRollingNumber(timeInMilliseconds, numberOfBuckets int64) (*RollingNumber, error) { 101 | if timeInMilliseconds%numberOfBuckets != 0 { 102 | return nil, errors.New("The timeInMilliseconds must divide equally into numberOfBuckets") 103 | } 104 | 105 | n := &RollingNumber{ 106 | timeInMilliseconds: timeInMilliseconds, 107 | numberOfBuckets: numberOfBuckets, 108 | bucketSizeInMillseconds: timeInMilliseconds / numberOfBuckets, 109 | buckets: newBuckets(numberOfBuckets, timeInMilliseconds*1000000), 110 | notify: make(chan *rollingNumberEvent), 111 | result: make(chan int64), 112 | } 113 | go n.process() 114 | return n, nil 115 | } 116 | 117 | func (n *RollingNumber) process() { 118 | for { 119 | event := <-n.notify 120 | switch event.op { 121 | case "add": 122 | n.buckets.Add(event.key, event.val) 123 | case "sum": 124 | n.result <- n.buckets.Sum(event.key) 125 | } 126 | } 127 | } 128 | 129 | // Add increase the counter specified by key with val 130 | func (n *RollingNumber) Add(key string, val int64) { 131 | n.notify <- &rollingNumberEvent{ 132 | key: key, 133 | val: val, 134 | op: "add", 135 | } 136 | } 137 | 138 | // Sum return the sum of the counter sepecified by key 139 | func (n *RollingNumber) Sum(key string) int64 { 140 | n.notify <- &rollingNumberEvent{ 141 | key: key, 142 | op: "sum", 143 | } 144 | return <-n.result 145 | } 146 | 147 | // AverageInSecond caculate the average of the item specified by key in second 148 | func (n *RollingNumber) AverageInSecond(key string) int64 { 149 | return n.Sum(key) * 1000 / n.timeInMilliseconds 150 | } 151 | 152 | type point struct { 153 | value int64 154 | timestamp time.Time 155 | } 156 | 157 | type quantile struct { 158 | values []point // ring buffer 159 | ids []int // sort helper 160 | lastseen time.Time 161 | lastupdate time.Time 162 | head int 163 | tail int 164 | size int 165 | min int64 166 | max int64 167 | q99 int64 168 | } 169 | 170 | func newQuantile(size int) *quantile { 171 | return &quantile{ 172 | values: make([]point, size), 173 | ids: make([]int, size), 174 | head: 0, 175 | tail: 0, 176 | size: size, 177 | } 178 | 179 | } 180 | 181 | func (q *quantile) add(value int64) { 182 | q.values[q.tail].value = value 183 | q.lastseen = time.Now() 184 | q.values[q.tail].timestamp = q.lastseen 185 | 186 | q.tail = (q.tail + 1) % q.size 187 | if q.tail == q.head { 188 | q.head = (q.head + 1) % q.size 189 | } 190 | } 191 | 192 | func (q *quantile) get() (mint int64, max int64, q99 int64) { 193 | if time.Now().Sub(q.lastupdate) > interval { 194 | q.caculate() 195 | } 196 | return q.min, q.max, q.q99 197 | } 198 | 199 | func (q *quantile) rank(left, right, rank int) { 200 | if rank == 0 || left >= right { 201 | q.q99 = q.values[q.ids[left]].value 202 | return 203 | } 204 | 205 | p := q.ids[left] 206 | v := q.values[p].value 207 | i := left 208 | j := right 209 | for i < j { 210 | for i < j && q.values[q.ids[j]].value >= v { 211 | j-- 212 | } 213 | q.ids[i] = q.ids[j] 214 | 215 | for i < j && q.values[q.ids[i]].value <= v { 216 | i++ 217 | } 218 | q.ids[j] = q.ids[i] 219 | } 220 | q.ids[i] = p 221 | 222 | if i < rank { 223 | q.rank(i+1, right, rank-i-1) 224 | } else if i > rank { 225 | q.rank(left, i-1, rank) 226 | } else { 227 | q.q99 = q.values[p].value 228 | } 229 | } 230 | 231 | func (q *quantile) caculate() { 232 | timestamp := time.Now() 233 | q.lastupdate = timestamp 234 | flag := false 235 | count := 0 236 | len := 0 237 | for i := q.head; i != q.tail; i = (i + 1) % q.size { 238 | if q.values[i].timestamp.Add(interval).Before(timestamp) { 239 | q.values[q.head].value = 0 240 | q.head = (q.head + 1) % q.size 241 | continue 242 | } 243 | 244 | q.ids[len] = i 245 | len++ 246 | 247 | if !flag { 248 | flag = true 249 | if q.head >= q.tail { 250 | count = q.size - q.head + q.tail 251 | } else { 252 | count = q.tail - q.head 253 | } 254 | 255 | if count == 0 { 256 | q.min = 0 257 | q.max = 0 258 | q.q99 = 0 259 | return 260 | } 261 | count = int(math.Ceil(float64(count) * 0.99)) 262 | 263 | q.max = q.values[i].value 264 | q.min = q.values[i].value 265 | } 266 | 267 | if q.values[i].value < q.min { 268 | q.min = q.values[i].value 269 | } 270 | if q.values[i].value >= q.max { 271 | q.max = q.values[i].value 272 | } 273 | } 274 | 275 | q.rank(0, len-1, count-1) 276 | } 277 | 278 | // QuantileGroup caculate the 99 quantile, min and max 279 | type QuantileGroup struct { 280 | group map[string]*quantile 281 | expiration time.Duration 282 | size int 283 | sync.Mutex 284 | } 285 | 286 | // NewQuantileGroup create a QuantileGroup 287 | func NewQuantileGroup(expiration time.Duration, size int) *QuantileGroup { 288 | g := &QuantileGroup{ 289 | group: make(map[string]*quantile), 290 | expiration: expiration, 291 | size: size, 292 | } 293 | go g.run() 294 | return g 295 | } 296 | 297 | const interval = time.Second * 10 298 | 299 | func (g *QuantileGroup) run() { 300 | ticker := time.NewTicker(g.expiration) 301 | for { 302 | <-ticker.C 303 | timestamp := time.Now() 304 | 305 | g.Lock() 306 | for k, v := range g.group { 307 | if v.lastseen.Add(g.expiration).Before(timestamp) { 308 | delete(g.group, k) 309 | } 310 | } 311 | g.Unlock() 312 | } 313 | } 314 | 315 | // Add record one point 316 | func (g *QuantileGroup) Add(key string, value int64) { 317 | g.Lock() 318 | defer g.Unlock() 319 | 320 | if _, ok := g.group[key]; !ok { 321 | g.group[key] = newQuantile(g.size) 322 | } 323 | g.group[key].add(value) 324 | } 325 | 326 | // Get retrive min, max and q99 327 | func (g *QuantileGroup) Get(key string) (mint int64, max int64, q99 int64) { 328 | g.Lock() 329 | defer g.Unlock() 330 | 331 | if _, ok := g.group[key]; !ok { 332 | return 0, 0, 0 333 | } 334 | return g.group[key].get() 335 | } 336 | --------------------------------------------------------------------------------