├── .gitignore ├── .travis.yml ├── Dockerfile ├── Gomfile ├── LICENSE ├── Makefile ├── README.md ├── debian ├── changelog ├── compat ├── control ├── copyright ├── dirs ├── docs ├── prerm ├── rules └── statsgod.init ├── doc └── development.md ├── example.config.yml ├── extras ├── generator.go ├── loadtest.go ├── log_compare.go └── receiver.go ├── statsgod.go ├── statsgod ├── auth.go ├── auth_test.go ├── config.go ├── config_test.go ├── connectionpool.go ├── connectionpool_test.go ├── logger.go ├── logger_test.go ├── metric.go ├── metric_test.go ├── nilconn.go ├── nillconn_test.go ├── relay.go ├── relay_test.go ├── signals.go ├── signals_test.go ├── socket.go ├── socket_test.go ├── statistics.go ├── statistics_test.go └── statsgod_suite_test.go └── statsgod_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Debian build 11 | debian/statsgod.debhelper.log 12 | debian/statsgod.postinst.debhelper 13 | debian/statsgod.postrm.debhelper 14 | debian/statsgod.prerm.debhelper 15 | debian/statsgod.substvars 16 | debian/statsgod/* 17 | 18 | # Architecture specific extensions/prefixes 19 | *.[568vq] 20 | [568vq].out 21 | 22 | *.cgo1.go 23 | *.cgo2.c 24 | _cgo_defun.c 25 | _cgo_gotypes.go 26 | _cgo_export.* 27 | 28 | _testmain.go 29 | _vendor/* 30 | 31 | *.exe 32 | *.test 33 | 34 | coverage.out 35 | statsgod.coverprofile 36 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - 1.4 4 | - 1.5 5 | before_install: 6 | - sudo apt-get update 7 | - go get github.com/mattn/gom 8 | script: 9 | - pushd $TRAVIS_BUILD_DIR 10 | - make test 11 | - make deb 12 | - popd 13 | notifications: 14 | email: false 15 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Acquia, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM ubuntu:trusty 16 | MAINTAINER Acquia Engineering 17 | 18 | ENV GOLANG_VERSION 1.5.1 19 | ENV DEBIAN_FRONTEND noninteractive 20 | 21 | # Setup container dependencies 22 | RUN apt-get -y update && \ 23 | apt-get -y install build-essential \ 24 | dh-make debhelper cdbs python-support \ 25 | git mercurial curl \ 26 | && apt-get clean && \ 27 | rm -rf /var/cache/apt/* && \ 28 | rm -rf /var/lib/apt/lists/* && \ 29 | rm -rf /tmp/* && \ 30 | rm -rf /var/tmp/* 31 | 32 | RUN curl -sSL https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-amd64.tar.gz | tar -C /usr/lib/ -xz && \ 33 | mkdir -p /usr/share/go 34 | 35 | # Setup env and install GOM for Development 36 | ENV GOROOT /usr/lib/go 37 | ENV GOPATH /usr/share/go 38 | ENV PATH ${GOROOT}/bin:${GOPATH}/bin:$PATH 39 | 40 | RUN go get github.com/mattn/gom 41 | 42 | WORKDIR /usr/share/go/src/github.com/acquia/statsgod 43 | CMD ["bash"] 44 | -------------------------------------------------------------------------------- /Gomfile: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Acquia, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | gom 'github.com/jmcvetta/randutil' 17 | gom 'gopkg.in/yaml.v2' 18 | 19 | # Prior to go 1.4 vet and cover are retrieved from legacy URLs. 20 | group :test_legacy do 21 | gom 'code.google.com/p/go.tools/cmd/vet' 22 | gom 'code.google.com/p/go.tools/cmd/cover' 23 | end 24 | 25 | group :test_tip do 26 | gom 'golang.org/x/tools/cmd/vet' 27 | gom 'golang.org/x/tools/cmd/cover' 28 | end 29 | 30 | group :test do 31 | gom 'gopkg.in/check.v1' 32 | gom 'github.com/golang/lint/golint' 33 | gom 'github.com/onsi/ginkgo/ginkgo' 34 | gom 'github.com/onsi/gomega' 35 | end 36 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | 204 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Acquia, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | project=statsgod 17 | 18 | version=0.1 19 | 20 | PACKAGES=extras statsgod 21 | 22 | export PATH := $(abspath ./_vendor/bin):$(PATH) 23 | 24 | # Versions of golang prior to 1.4 use a different package URL. 25 | GOM_GROUPS_FLAG="test,test_tip" 26 | ifneq (,$(findstring $(shell go version | grep -e "go[0-9]\+\.[0-9]\+" -o),go1.2 go1.3)) 27 | GOM_GROUPS_FLAG="test,test_legacy" 28 | endif 29 | 30 | GOM=$(if $(TRAVIS),$(HOME)/gopath/bin/gom,gom) 31 | 32 | all: ${project} 33 | 34 | clean: 35 | rm -rf _vendor 36 | 37 | run: ${project} 38 | go run -race ${project}.go 39 | 40 | $(project): deps 41 | $(GOM) build -o $(GOPATH)/bin/statsgod 42 | 43 | deps: clean 44 | $(GOM) -groups=$(GOM_GROUPS_FLAG) install 45 | 46 | lint: deps 47 | $(GOM) exec go fmt ./... 48 | $(GOM) exec go vet -x ./... 49 | $(GOM) exec golint . 50 | $(foreach p, $(PACKAGES), $(GOM) exec golint ./$(p)/.; ) 51 | 52 | test: deps lint 53 | (test -f coverage.out && "$(TRAVIS)" == "true") || \ 54 | $(GOM) exec go test -covermode=count -coverprofile=coverage.out . 55 | (test -f statsgod/statsgod.coverprofile && "$(TRAVIS)" == "true") || \ 56 | $(GOM) exec ginkgo -cover=true ./statsgod/. 57 | 58 | deb: 59 | dpkg-buildpackage -uc -b -d -tc 60 | 61 | recv: 62 | go run -race extras/receiver/test_receiver.go 63 | 64 | run-container: 65 | docker run -v $(shell pwd)/:/usr/share/go/src/github.com/acquia/statsgod -it statsgod /bin/bash 66 | 67 | .PHONY: all clean run deps test 68 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | statsgod 2 | ======== 3 | [![Build Status](https://travis-ci.org/acquia/statsgod.png)](https://travis-ci.org/acquia/statsgod) 4 | 5 | Statsgod is a metric aggregation service inspired by the statsd project. Written in Golang, it increases performance and can be deployed without dependencies. This project uses the same metric string format as statsd, but adds new features like alternate sockets, authentication, etc. 6 | 7 | ## Usage 8 | 9 | Usage: statsgod [args] 10 | -config="config.yml": YAML config file path 11 | 12 | ### Example: 13 | 1. start the daemon. 14 | 15 | gom exec go run statsgod.go 16 | 17 | 2. Start a testing receiver. 18 | 19 | gom exec go run test_receiver.go 20 | 21 | 3. Send data to the daemon. Set a gauge to 3 for your.metric.name 22 | 23 | echo "your.metric.name:3|g" | nc localhost 8125 # TCP 24 | echo "your.metric.name:3|g" | nc -4u -w0 localhost 8126 # UDP 25 | echo "your.metric.name:3|g" | nc -U /tmp/statsgod.sock # Unix Socket 26 | 27 | ### Metric Format 28 | Data is sent over a socket connection as a string using the format: [namespace]:[value]|[type] where the namespace is a dot-delimeted string like "user.login.success". Values are floating point numbers represented as strings. The metric type uses the following values: 29 | 30 | - Gauge (g): constant metric, keeps the last value. 31 | - Counter (c): increment/decrement a given namespace. 32 | - Timer (ms): a timer that calculates averages (see below). 33 | - Set (s): a count of unique values sent during a flush period. 34 | 35 | Optionally you may denote that a metric has been sampled by adding "|@0.75" (where 0.75 is the sample rate as a float). Counters will inflate the value accordingly so that it can be accurately used to calculate a rate. 36 | 37 | An example data string would be "user.login.success:123|c|@0.9" 38 | 39 | ### Sending Metrics 40 | Client code can send metrics via any one of three sockets which listen concurrently: 41 | 42 | 1. TCP 43 | - Allows multiple metrics to be sent over a connection, separated by a newline character. 44 | - Connection will remain open until closed by the client. 45 | - Config: 46 | - connection.udp.enabled 47 | - connection.udp.host 48 | - connection.udp.port 49 | 50 | 2. UDP 51 | - Allows multiple metrics to be sent over a connection, separated by a newline character. Note, you should be careful to not exceed the maximum packet size (default 1024 bytes). 52 | - Config: 53 | - connection.udp.enabled 54 | - connection.udp.host 55 | - connection.udp.port 56 | - connection.udp.maxpacket (buffer size to read incoming packets) 57 | 58 | 3. Unix Domain Socket 59 | - Allows multiple metrics to be sent over a connection, separated by a newline character. 60 | - Connection will remain open until closed by the client. 61 | - Config: 62 | - connection.unix.enabled 63 | - config: connection.unix.file (path to the sock file) 64 | 65 | ## Configuration 66 | All runtime options are specified in a YAML file. Please see example.config.yml for defaults. e.g. 67 | 68 | go run statsgod.go -config=/etc/statsgod.yml 69 | 70 | ## Stats Types 71 | Statsgod provides support for the following metric types. 72 | 73 | 1. Counters - these are cumulative values that calculate the sum of all metrics sent. A rate is also calculated to determine how many values were sent during the flush interval: 74 | 75 | my.counter:1|c 76 | my.counter:1|c 77 | my.counter:1|c 78 | # flush produces a count and a rate: 79 | [prefix].my.counter.[suffix] [timestamp] 3 80 | [prefix].my.counter.[suffix] [timestamp] [3/(duration of flush interval in seconds)] 81 | 82 | 2. Gauges - these are a "last in" measurement which discards all previously sent values: 83 | 84 | my.gauge:1|g 85 | my.gauge:2|g 86 | my.gauge:3|g 87 | # flush only sends the last value: 88 | [prefix].my.gauge.[suffix] [timestamp] 3 89 | 90 | 3. Timers - these are timed values measured in milliseconds. Statsgod provides several calculated values based on the sent metrics: 91 | 92 | my.timer:100|ms 93 | my.timer:200|ms 94 | my.timer:300|ms 95 | # flush produces several calculated fields: 96 | [prefix].my.timer.mean_value.[suffix] [timestamp] [mean] 97 | [prefix].my.timer.median_value.[suffix] [timestamp] [median] 98 | [prefix].my.timer.min_value.[suffix] [timestamp] [min] 99 | [prefix].my.timer.max_value.[suffix] [timestamp] [max] 100 | [prefix].my.timer.mean_90.[suffix] [timestamp] [mean in 90th percentile] 101 | [prefix].my.timer.upper_90.[suffix] [timestamp] [upper in 90th percentile] 102 | [prefix].my.timer.sum_90.[suffix] [timestamp] [sum in 90th percentile] 103 | 104 | 4. Sets - these track the number of unique values sent during a flush interval: 105 | 106 | my.unique:1|s 107 | my.unique:2|s 108 | my.unique:2|s 109 | my.unique:1|s 110 | # flush produces a single value counting the unique metrics sent: 111 | [prefix].my.unique.[suffix] [timestamp] 2 112 | 113 | ## Prefix/Suffix 114 | Prefixes and suffixes noted above can be customized in the configuration. Metrics will render as [prefix].[type prefix].[metric namespace].[type suffix].[suffix]. You may also use empty strings in the config for any values you do not wish statsgod to prefix/suffix before relaying. 115 | 116 | namespace: 117 | prefix: "stats" 118 | prefixes: 119 | counters: "counts" 120 | gauges: "gauges" 121 | rates: "rates" 122 | sets: "sets" 123 | timers: "timers" 124 | suffix: "" 125 | suffixes: 126 | counters: "" 127 | gauges: "" 128 | rates: "" 129 | sets: "" 130 | timers: "" 131 | 132 | 133 | 134 | ## Authentication 135 | Auth is handled via the statsgod.Auth interface. Currently there are two types of authentication: no-auth and token-auth, which are specified in the configuration file: 136 | 137 | 1. No auth 138 | 139 | # config.yml 140 | service: 141 | auth: "none" 142 | 143 | Works as you might expect, all metrics strings are parsed without authentication or manipulation. This is the default behavior. 144 | 145 | 2. Token auth 146 | 147 | # config.yml 148 | service: 149 | auth: "token" 150 | tokens: 151 | "token-name": false 152 | "32a3c4970093": true 153 | 154 | "token" checks the configuration file for a valid auth token. The config file may specify as many tokens as needed in the service.tokens map. These are written as "string": bool where the string is the token and the bool is whether or not the token is valid. Please note that these are read into memory when the proces is started, so changes to the token map require a restart. 155 | 156 | When sending metrics, the token is specified at the beginning of the metric namespace followed by a dot. For example, a metric "32a3c4970093.my.metric:123|g" would look in the config tokens for the string "32a3c4970093" and see if that is set to true. If valid, the process will strip the token from the namespace, only parsing and aggregating "my.metric:123|g". NOTE: since metric namespaces are dot-delimited, you cannot use a dot in a token. 157 | 158 | ## Signal handling 159 | The statsgod service is equipped to handle the following signals: 160 | 161 | 1. Shut down the sockets and clean up before exiting. 162 | - SIGABRT 163 | - SIGINT 164 | - SIGTERM 165 | - SIGQUIT 166 | 167 | 2. Reload\* the configuration without restarting. 168 | - SIGHUP 169 | 170 | \* When reloading configuration, not all values will affect the current runtime. The following are only available on start up and not currently reloadable: 171 | 172 | - connection.\* 173 | - relay.\* 174 | - stats.percentile 175 | - debug.verbose 176 | - debug.profile 177 | 178 | ## Development 179 | [Read more about the development process.](doc/development.md) 180 | 181 | ## License 182 | Except as otherwise noted this software is licensed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0.html) 183 | 184 | Licensed under the Apache License, Version 2.0 (the "License"); 185 | you may not use this file except in compliance with the License. 186 | You may obtain a copy of the License at 187 | 188 | http://www.apache.org/licenses/LICENSE-2.0 189 | 190 | Unless required by applicable law or agreed to in writing, software 191 | distributed under the License is distributed on an "AS IS" BASIS, 192 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 193 | See the License for the specific language governing permissions and 194 | limitations under the License. 195 | -------------------------------------------------------------------------------- /debian/changelog: -------------------------------------------------------------------------------- 1 | statsgod (0.2-1) trusty; urgency=low 2 | 3 | * Initial release 4 | 5 | -- root Mon, 02 Nov 2015 20:11:44 +0000 6 | -------------------------------------------------------------------------------- /debian/compat: -------------------------------------------------------------------------------- 1 | 9 2 | -------------------------------------------------------------------------------- /debian/control: -------------------------------------------------------------------------------- 1 | Source: statsgod 2 | Section: web 3 | Priority: optional 4 | Maintainer: Acquia 5 | Build-Depends: debhelper (>= 9.0.0) 6 | Standards-Version: 3.9.5 7 | Homepage: http://github.com/acquia/statsgod 8 | 9 | Package: statsgod 10 | Section: web 11 | Priority: extra 12 | Architecture: amd64 13 | Depends: ${shlibs:Depends}, ${misc:Depends} 14 | Description: Network metric aggregator. 15 | Metrics are sent using TCP, UDP or Unix sockets then aggregated and relayed to 16 | a backend service. 17 | -------------------------------------------------------------------------------- /debian/copyright: -------------------------------------------------------------------------------- 1 | Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ 2 | Upstream-Name: statsgod 3 | Source: https://github.com/acquia/statsgod 4 | 5 | Files: debian/* 6 | Copyright: 2015 Acquia, Inc. 7 | License: Apache 2.0 8 | Licensed under the Apache License, Version 2.0 (the "License"); 9 | you may not use this file except in compliance with the License. 10 | You may obtain a copy of the License at 11 | . 12 | http://www.apache.org/licenses/LICENSE-2.0 13 | . 14 | Unless required by applicable law or agreed to in writing, software 15 | distributed under the License is distributed on an "AS IS" BASIS, 16 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | See the License for the specific language governing permissions and 18 | limitations under the License. 19 | -------------------------------------------------------------------------------- /debian/dirs: -------------------------------------------------------------------------------- 1 | etc/statsgod 2 | usr/bin 3 | -------------------------------------------------------------------------------- /debian/docs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acquia/statsgod/bdc50074f2876517e235196ca5c26f66442e1fce/debian/docs -------------------------------------------------------------------------------- /debian/prerm: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Stop the service. 4 | /etc/init.d/statsgod stop || true 5 | -------------------------------------------------------------------------------- /debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | # -*- makefile -*- 3 | 4 | # Uncomment this to turn on verbose mode. 5 | #export DH_VERBOSE=1 6 | 7 | %: 8 | dh $@ 9 | 10 | override_dh_auto_clean: 11 | $(MAKE) clean 12 | 13 | override_dh_auto_build: 14 | $(MAKE) 15 | 16 | override_dh_builddeb: 17 | # Force gzip compression for older system wth dpkg version < 1.15.6 18 | dh_builddeb -- -Zgzip 19 | 20 | override_dh_auto_install: 21 | mkdir -p $(CURDIR)/debian/statsgod/usr/bin 22 | mkdir -p $(CURDIR)/debian/statsgod/etc/statsgod 23 | install -Dm755 $(GOPATH)/bin/statsgod $(CURDIR)/debian/statsgod/usr/bin/statsgod 24 | install -m644 $(CURDIR)/example.config.yml $(CURDIR)/debian/statsgod/etc/statsgod/config.yml 25 | dh_installinit --no-start --name statsgod 26 | 27 | .PHONY: override_dh_auto_clean override_dh_auto_build override_dh_builddeb override_dh_auto_install 28 | -------------------------------------------------------------------------------- /debian/statsgod.init: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | ### BEGIN INIT INFO 3 | # Provides: statsgod 4 | # Required-Start: $remote_fs $syslog 5 | # Required-Stop: $remote_fs $syslog 6 | # Default-Start: 2 3 4 5 7 | # Default-Stop: 0 1 6 8 | # Short-Description: statsgod service 9 | # Description: Statsgod provides socket listeners to aggregate metrics and 10 | # relay them to a remote backend. 11 | ### END INIT INFO 12 | 13 | PATH=/sbin:/usr/sbin:/bin:/usr/bin 14 | DESC="Statsgod metric aggregator" 15 | NAME=statsgod 16 | DAEMON=/usr/bin/$NAME 17 | DAEMON_ARGS="" 18 | SCRIPTNAME=/etc/init.d/$NAME 19 | RUN_DIR=/var/run/$NAME 20 | LOG_DIR=/var/log/$NAME 21 | PIDFILE=$RUN_DIR/$NAME.pid 22 | STATSGOD_USER=statsgod 23 | STATSGOD_SOCK=$RUN_DIR/$NAME.sock 24 | 25 | # Look for $STATSGOD_USER, if it does not exist, start the daemon as the current user. 26 | id -u $STATSGOD_USER > /dev/null 2>&1 || STATSGOD_USER=$(whoami) 27 | 28 | # Exit if the package is not installed 29 | [ -x "$DAEMON" ] || exit 0 30 | 31 | # Read configuration variable file if it is present 32 | [ -r /etc/default/$NAME ] && . /etc/default/$NAME 33 | 34 | # Load the VERBOSE setting and other rcS variables 35 | . /lib/init/vars.sh 36 | 37 | # Define LSB log_* functions. 38 | # Depend on lsb-base (>= 3.2-14) to ensure that this file is present 39 | # and status_of_proc is working. 40 | . /lib/lsb/init-functions 41 | 42 | # 43 | # Function that starts the daemon/service 44 | # 45 | do_start() 46 | { 47 | # Ensure that the log and run directories are present. 48 | # This script will need to be executed as a super user. 49 | mkdir -p $LOG_DIR 50 | chown -R $STATSGOD_USER $LOG_DIR 51 | chmod 775 $LOG_DIR 52 | mkdir -p $RUN_DIR 53 | chown -R $STATSGOD_USER $RUN_DIR 54 | chmod 775 $RUN_DIR 55 | # Ensure that we can handle the socket traffic. 56 | ulimit -n 65000 57 | # Return 58 | # 0 if daemon has been started 59 | # 1 if daemon was already running 60 | # 2 if daemon could not be started 61 | start-stop-daemon --start --quiet --pidfile $PIDFILE --chuid $STATSGOD_USER \ 62 | --make-pidfile --background --test \ 63 | --startas /bin/bash -- -c "exec $DAEMON $DAEMON_ARGS > /dev/null 2>&1"\ 64 | || return 1 65 | 66 | # If the service died unexpectedly, remove the sock. 67 | [ -e $STATSGOD_SOCK ] && rm $STATSGOD_SOCK 68 | 69 | start-stop-daemon --start --quiet --pidfile $PIDFILE --chuid $STATSGOD_USER \ 70 | --make-pidfile --background \ 71 | --startas /bin/bash -- -c "exec $DAEMON $DAEMON_ARGS >> /var/log/statsgod/statsgod.log 2>> /var/log/statsgod/statsgod.err"\ 72 | || return 2 73 | # Add code here, if necessary, that waits for the process to be ready 74 | # to handle requests from services started subsequently which depend 75 | # on this one. As a last resort, sleep for some time. 76 | } 77 | 78 | # 79 | # Function that stops the daemon/service 80 | # 81 | do_stop() 82 | { 83 | # Return 84 | # 0 if daemon has been stopped 85 | # 1 if daemon was already stopped 86 | # 2 if daemon could not be stopped 87 | # other if a failure occurred 88 | start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --name $NAME 89 | RETVAL="$?" 90 | [ "$RETVAL" = 2 ] && return 2 91 | # Wait for children to finish too if this is a daemon that forks 92 | # and if the daemon is only ever run from this initscript. 93 | # If the above conditions are not satisfied then add some other code 94 | # that waits for the process to drop all resources that could be 95 | # needed by services started subsequently. A last resort is to 96 | # sleep for some time. 97 | start-stop-daemon --stop --quiet --oknodo --retry=0/30/KILL/5 --exec $DAEMON 98 | [ "$?" = 2 ] && return 2 99 | # Many daemons don't delete their pidfiles when they exit. 100 | rm -f $PIDFILE 101 | return "$RETVAL" 102 | } 103 | 104 | # 105 | # Function that sends a SIGHUP to the daemon/service 106 | # 107 | do_reload() { 108 | # 109 | # If the daemon can reload its configuration without 110 | # restarting (for example, when it is sent a SIGHUP), 111 | # then implement that here. 112 | # 113 | start-stop-daemon --stop --signal 1 --quiet --pidfile $PIDFILE --name $NAME 114 | return 0 115 | } 116 | 117 | case "$1" in 118 | start) 119 | [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME" 120 | do_start 121 | case "$?" in 122 | 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; 123 | 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; 124 | esac 125 | ;; 126 | stop) 127 | [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME" 128 | do_stop 129 | case "$?" in 130 | 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; 131 | 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; 132 | esac 133 | ;; 134 | status) 135 | status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $? 136 | ;; 137 | #reload|force-reload) 138 | # 139 | # If do_reload() is not implemented then leave this commented out 140 | # and leave 'force-reload' as an alias for 'restart'. 141 | # 142 | #log_daemon_msg "Reloading $DESC" "$NAME" 143 | #do_reload 144 | #log_end_msg $? 145 | #;; 146 | restart|force-reload) 147 | # 148 | # If the "reload" option is implemented then remove the 149 | # 'force-reload' alias 150 | # 151 | log_daemon_msg "Restarting $DESC" "$NAME" 152 | do_stop 153 | case "$?" in 154 | 0|1) 155 | do_start 156 | case "$?" in 157 | 0) log_end_msg 0 ;; 158 | 1) log_end_msg 1 ;; # Old process is still running 159 | *) log_end_msg 1 ;; # Failed to start 160 | esac 161 | ;; 162 | *) 163 | # Failed to stop 164 | log_end_msg 1 165 | ;; 166 | esac 167 | ;; 168 | *) 169 | #echo "Usage: $SCRIPTNAME {start|stop|restart|reload|force-reload}" >&2 170 | echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 171 | exit 3 172 | ;; 173 | esac 174 | 175 | : 176 | -------------------------------------------------------------------------------- /doc/development.md: -------------------------------------------------------------------------------- 1 | ## Development 2 | To download all dependencies and compile statsgod 3 | 4 | go get -u github.com/mattn/gom 5 | mkdir -p $GOPATH/src/github.com/acquia/statsgod 6 | git clone https://github.com/acquia/statsgod $GOPATH/src/github.com/acquia/statsgod 7 | cd $GOPATH/src/github.com/acquia/statsgod 8 | gom install 9 | gom build -o $GOPATH/bin/statsgod 10 | 11 | 12 | To build the debian package in a docker container 13 | 14 | docker build -t statsgod . 15 | docker run -v $(pwd)/:/usr/share/go/src/github.com/acquia/statsgod -it statsgod /bin/bash -c "make && make deb" 16 | 17 | ## Testing 18 | For automated tests we are using http://onsi.github.io/ginkgo/ and http://onsi.github.io/gomega/. We have a combination of unit, integration and benchmark tests which are executed by Travis.ci. 19 | 20 | make test 21 | 22 | ## Load Testing and QA 23 | For load testing and manual QA we have a script in /extras/loadtest.go that can be used to soak the system. There are a lot of options with invocation, so take a moment to read through and understand them. The most used flags specify the number of metrics, amount of concurrency and type of connection to test (tcp, tcp with connection pool, udp, unix and unix with connection pool). 24 | 25 | Here is an example using the -logSent flag which will emit a line for each metric string sent. For statsgod.go, use the config option config.debug.receipt: true to emit the received/parsed metrics. Using the log\_compare.go utility you can compare each metric sent to each received/parsed metric received to ensure that nothing was lost. 26 | 27 | 1. Start statsgod.go with config.debug.receipt set to true and redirect to a file: 28 | 29 | gom exec go run statsgod.go 2>&1 > /tmp/statsgod.output 30 | 31 | 2. Start loadtest.go with the -logSent flag and redirect to a file: 32 | 33 | gom exec go run extras/loadtest.go [options] -logSent=true 2>&1 > /tmp/statsgod.input 34 | 35 | 3. After collecting input and output, compare using the log\_compare.go utility: 36 | 37 | gom exec go run extras/log_compare.go -in=/tmp/statsgod.input -out=/tmp/statsgod.output 38 | 39 | ## Profiling 40 | To profile this program we have been using the runtime.pprof library. Build a binary, start the daemon and run it through a real-world set of tests to fully exercise the code. After the program exits it will create a file in the same directory called "statsgod.prof". This profile, along with the binary, can be used to learn more about the runtime. 41 | 42 | Config: 43 | debug.profile = true 44 | 45 | gom build -o sg # Build a binary called "sg" 46 | ./sg -config=/path/to/config # start the daemon and don't forget to set config.debug.profile = true 47 | # run some tests and stop the daemon. "sg" is the binary and "statsgod.prof" is the profile. 48 | go tool pprof --help # See all of the options. 49 | go tool pprof --dot sg statsgod.prof # Generate a dot file for graphviz. 50 | go tool pprof --gif sg statsgod.prof # Generate a gif of the visualization. 51 | go tool pprof sg statsgod.prof # Interactive mode: 52 | (pprof) top30 -cum # View the top 30 functions sorted by cumulative sample counts. 53 | 54 | 55 | -------------------------------------------------------------------------------- /example.config.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Acquia, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | ### 17 | # Main server configuration 18 | ### 19 | service: 20 | name: "statsgod" 21 | auth: "none" # One of "none" or "token". 22 | tokens: 23 | "token-name": false # A list of tokens for auth: "ConfigToken". Use the 24 | # format: [token]: [valid] where [token] is the token 25 | # string and [valid] is a boolean value describing 26 | # whether or not the token is valid. 27 | hostname: "" # Leave hostname empty to use the current hostname. 28 | 29 | ### 30 | # Socket connections 31 | ### 32 | connection: 33 | tcp: 34 | enabled: true 35 | host: 127.0.0.1 36 | port: 8125 37 | udp: 38 | enabled: true 39 | host: 127.0.0.1 40 | port: 8126 41 | maxpacket: 1024 42 | unix: 43 | enabled: true 44 | file: /var/run/statsgod/statsgod.sock 45 | 46 | ### 47 | # Backend relay carbon server configuration 48 | ### 49 | carbon: 50 | host: 127.0.0.1 51 | port: 2003 52 | 53 | ### 54 | # Relay connection options 55 | ### 56 | relay: 57 | type: carbon # One of "mock" or "carbon". 58 | concurrency: 1 # Normally use 1 for carbon. 59 | timeout: 30s # How long to sit idle on a connection. 60 | flush: 10s # How frequently to flush to the relay. 61 | 62 | ### 63 | # Namespace options 64 | ### 65 | namespace: 66 | prefix: "stats" 67 | prefixes: 68 | counters: "counts" 69 | gauges: "gauges" 70 | rates: "rates" 71 | sets: "sets" 72 | timers: "timers" 73 | suffix: "" 74 | suffixes: 75 | counters: "" 76 | gauges: "" 77 | rates: "" 78 | sets: "" 79 | timers: "" 80 | 81 | ### 82 | # Stats options 83 | ### 84 | stats: 85 | percentile: 86 | - 80 87 | ### 88 | # Debug options 89 | ### 90 | debug: 91 | verbose: false # Increases the logging output. 92 | receipt: false # Logs a message with the parsed metric. 93 | profile: false # Profile the program with runtime/pprof. 94 | relay: false # Relay internal runtime information as metrics. 95 | -------------------------------------------------------------------------------- /extras/generator.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Acquia, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Package generator is a sample metric generator app. 18 | package main 19 | 20 | /** 21 | * Test generator used to send data to the statsgod process. 22 | */ 23 | 24 | import ( 25 | "flag" 26 | "fmt" 27 | "github.com/jmcvetta/randutil" 28 | "math/rand" 29 | "net" 30 | "time" 31 | ) 32 | 33 | var statsHost = flag.String("statsHost", "localhost", "Stats Hostname") 34 | var statsPortTcp = flag.Int("statsPortTcp", 8125, "Statsgod TCP Port") 35 | var statsPortUdp = flag.Int("statsPortUdp", 8126, "Statsgod UDP Port") 36 | var statsSock = flag.String("statsSock", "/var/run/statsgod/statsgod.sock", "The location of the socket file.") 37 | var numMetrics = flag.Int("numMetrics", 10, "Number of metrics") 38 | var flushTime = flag.Duration("flushTime", 2000*time.Millisecond, "Flush time") 39 | var sleepTime = flag.Duration("sleepTime", 10*time.Nanosecond, "Sleep time") 40 | var concurrency = flag.Int("concurrency", 1, "How many concurrent generators to run.") 41 | 42 | const ( 43 | // AvailableMemory is amount of available memory for the process. 44 | AvailableMemory = 10 << 20 // 10 MB, for example 45 | // AverageMemoryPerRequest is how much memory we want to use per request. 46 | AverageMemoryPerRequest = 10 << 10 // 10 KB 47 | // MAXREQS is how many requests. 48 | MAXREQS = AvailableMemory / AverageMemoryPerRequest 49 | // ConnectionTypeTcp is the enum value for TCP connections. 50 | ConnectionTypeTcp = 1 51 | // ConnectionTypeUdp is the enum value for UDP connections. 52 | ConnectionTypeUdp = 2 53 | // ConnectionTypeUnix is the enum value for Unix connections. 54 | ConnectionTypeUnix = 3 55 | ) 56 | 57 | // Metric is our main data type. 58 | type Metric struct { 59 | key string // Name of the metric. 60 | metricType string // What type of metric is it (gauge, counter, timer) 61 | connectionType int // Whether we are connecting TCP, UDP or Unix. 62 | } 63 | 64 | func main() { 65 | // Load command line options. 66 | flag.Parse() 67 | 68 | logger(fmt.Sprintf("Creating %d metrics for %d processes", *numMetrics, *concurrency)) 69 | 70 | for i := 0; i < *concurrency; i++ { 71 | var store = make([]Metric, 0) 72 | store = generateMetricNames(*numMetrics, store) 73 | statsPipeline := make(chan Metric, MAXREQS) 74 | fmt.Printf("New store: %v\n", store) 75 | 76 | // Every X seconds we want to flush the metrics 77 | go loadTestMetrics(store, statsPipeline) 78 | 79 | // Constantly process background Stats queue. 80 | go handleStatsQueue(statsPipeline) 81 | } 82 | 83 | select {} // block forever 84 | 85 | } 86 | 87 | func loadTestMetrics(store []Metric, statsPipeline chan Metric) { 88 | flushTicker := time.Tick(*flushTime) 89 | fmt.Printf("Flushing every %v\n", *flushTime) 90 | 91 | for { 92 | select { 93 | case <-flushTicker: 94 | fmt.Println("Tick...") 95 | for _, metric := range store { 96 | statsPipeline <- metric 97 | } 98 | } 99 | } 100 | } 101 | 102 | func handleStatsQueue(statsPipeline chan Metric) { 103 | for { 104 | metric := <-statsPipeline 105 | go sendMetricToStats(metric) 106 | } 107 | } 108 | 109 | func generateMetricNames(numMetrics int, store []Metric) []Metric { 110 | metricTypes := []string{ 111 | "c", 112 | "g", 113 | "ms", 114 | } 115 | 116 | r := rand.New(rand.NewSource(time.Now().UnixNano())) 117 | 118 | for i := 0; i < numMetrics; i++ { 119 | newMetricName, _ := randutil.String(20, randutil.Alphabet) 120 | newMetricNS := fmt.Sprintf("statsgod.test.%s", newMetricName) 121 | newMetricCT, _ := randutil.IntRange(1, 4) 122 | store = append(store, Metric{ 123 | key: newMetricNS, 124 | metricType: metricTypes[r.Intn(len(metricTypes))], 125 | connectionType: newMetricCT, 126 | }) 127 | } 128 | 129 | return store 130 | } 131 | 132 | // sendSingleMetricToGraphite formats a message and a value and time and sends to Graphite. 133 | func sendMetricToStats(metric Metric) { 134 | var payload string 135 | fmt.Printf("Sending metric %s.%s to stats on conn %d\n", metric.metricType, metric.key, metric.connectionType) 136 | 137 | rand.Seed(time.Now().UnixNano()) 138 | 139 | if metric.metricType == "ms" { 140 | payload = fmt.Sprintf("%s:%d|ms", metric.key, rand.Intn(1000)) 141 | } else if metric.metricType == "c" { 142 | payload = fmt.Sprintf("%s:1|c", metric.key) 143 | } else { 144 | payload = fmt.Sprintf("%s:%d|g", metric.key, rand.Intn(100)) 145 | } 146 | 147 | //sv := strconv.FormatFloat(float64(v), 'f', 6, 32) 148 | //payload := fmt.Sprintf("%s %s %s", key, sv, t) 149 | //Trace.Printf("Payload: %v", payload) 150 | 151 | // Send to the connection 152 | var sendErr error 153 | switch metric.connectionType { 154 | case 1: 155 | c, err := net.Dial("tcp", fmt.Sprintf("%s:%d", *statsHost, *statsPortTcp)) 156 | sendErr = err 157 | if err == nil { 158 | fmt.Fprintf(c, payload) 159 | defer c.Close() 160 | } 161 | case 2: 162 | c, err := net.Dial("udp", fmt.Sprintf("%s:%d", *statsHost, *statsPortUdp)) 163 | sendErr = err 164 | if err == nil { 165 | fmt.Fprintf(c, payload) 166 | defer c.Close() 167 | } 168 | case 3: 169 | c, err := net.Dial("unix", *statsSock) 170 | sendErr = err 171 | if err == nil { 172 | fmt.Fprintf(c, payload) 173 | defer c.Close() 174 | } 175 | } 176 | if sendErr != nil { 177 | fmt.Println("Could not connect to remote stats server") 178 | return 179 | } 180 | 181 | } 182 | 183 | func logger(msg string) { 184 | fmt.Println(msg) 185 | } 186 | -------------------------------------------------------------------------------- /extras/loadtest.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Acquia, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "flag" 21 | "fmt" 22 | "github.com/acquia/statsgod/statsgod" 23 | "github.com/jmcvetta/randutil" 24 | "io/ioutil" 25 | "math/rand" 26 | "net" 27 | "os" 28 | "time" 29 | ) 30 | 31 | var statsHost = flag.String("statsHost", "localhost", "Statsgod Hostname.") 32 | var statsPortTcp = flag.Int("statsPortTcp", 8125, "Statsgod TCP Port.") 33 | var statsPortUdp = flag.Int("statsPortUdp", 8126, "Statsgod UDP Port.") 34 | var statsSock = flag.String("statsSock", "/var/run/statsgod/statsgod.sock", "The location of the socket file.") 35 | var statsPoolCount = flag.Int("statsPoolCount", 5, "How many active connections to maintain.") 36 | var numMetrics = flag.Int("numMetrics", 10, "Number of metrics per thread.") 37 | var flushTime = flag.Duration("flushTime", 1*time.Second, "How frequently to send metrics.") 38 | var concurrency = flag.Int("concurrency", 1, "How many concurrent generators to run.") 39 | var runTime = flag.Duration("runTime", 30*time.Second, "How long to run the test.") 40 | var connType = flag.Int("connType", 0, "0 for all, 1 for TCP, 2 for TCP Pool, 3 for UDP, 4 for Unix, 5 for Unix Pool.") 41 | var logSent = flag.Bool("logSent", false, "Log each metric sent.") 42 | var token = flag.String("token", "", "An auth token to prepend to the metric string.") 43 | 44 | // Metric is our main data type. 45 | type Metric struct { 46 | key string // Name of the metric. 47 | metricType string // What type of metric is it (gauge, counter, set, timer) 48 | metricValue int // The value of the metric to send. 49 | connectionType int // Whether we are connecting TCP, UDP or Unix. 50 | } 51 | 52 | // Connection Types enum. 53 | const ( 54 | ConnectionTypeTcp = 1 55 | ConnectionTypeTcpPool = 2 56 | ConnectionTypeUdp = 3 57 | ConnectionTypeUnix = 4 58 | ConnectionTypeUnixPool = 5 59 | ) 60 | 61 | // Track connections/errors. 62 | var ( 63 | connectionCountTcpPool int 64 | connectionCountTcp int 65 | connectionCountUdp int 66 | connectionCountUnixPool int 67 | connectionCountUnix int 68 | connectionErrorTcpPool int 69 | connectionErrorTcp int 70 | connectionErrorUdp int 71 | connectionErrorUnixPool int 72 | connectionErrorUnix int 73 | tcpPool *statsgod.ConnectionPool 74 | unixPool *statsgod.ConnectionPool 75 | ) 76 | 77 | var logger = *statsgod.CreateLogger(ioutil.Discard, os.Stdout, os.Stdout, os.Stderr) 78 | 79 | func main() { 80 | flag.Parse() 81 | fmt.Printf("Starting test with %d metrics on %d concurrent threads for %s.\n", *numMetrics, *concurrency, *runTime) 82 | 83 | tcpPool, _ = statsgod.CreateConnectionPool(*statsPoolCount, fmt.Sprintf("%s:%d", *statsHost, *statsPortTcp), statsgod.ConnPoolTypeTcp, 10*time.Second, logger) 84 | unixPool, _ = statsgod.CreateConnectionPool(*statsPoolCount, *statsSock, statsgod.ConnPoolTypeUnix, 10*time.Second, logger) 85 | 86 | startTime := time.Now() 87 | 88 | finishChannel := make(chan int) 89 | flushChannel := make(chan Metric) 90 | 91 | // If the user specified an auth token, construct it here. 92 | metricToken := "" 93 | if *token != "" { 94 | metricToken = *token + "." 95 | } 96 | 97 | // Establish threads to send data concurrently. 98 | for i := 0; i < *concurrency; i++ { 99 | var store = make([]Metric, 0) 100 | store = generateMetricNames(*numMetrics, store, metricToken) 101 | go sendTestMetrics(store, flushChannel, finishChannel) 102 | go flushMetrics(flushChannel, finishChannel) 103 | } 104 | 105 | finishTicker := time.Tick(*runTime) 106 | flushTicker := time.Tick(*flushTime) 107 | runloop: 108 | for { 109 | select { 110 | case <-flushTicker: 111 | if !*logSent { 112 | fmt.Printf("-") 113 | } 114 | case <-finishTicker: 115 | break runloop 116 | } 117 | } 118 | close(finishChannel) 119 | 120 | totalTime := time.Since(startTime) 121 | 122 | // Print the output. 123 | printReqPerSecond("TCP", connectionCountTcp, connectionErrorTcp, totalTime) 124 | printReqPerSecond("TCP (pool)", connectionCountTcpPool, connectionErrorTcpPool, totalTime) 125 | printReqPerSecond("UDP", connectionCountUdp, connectionErrorUdp, totalTime) 126 | printReqPerSecond("Unix", connectionCountUnix, connectionErrorUnix, totalTime) 127 | printReqPerSecond("Unix (pool)", connectionCountUnixPool, connectionErrorUnixPool, totalTime) 128 | totalCount := connectionCountTcpPool + connectionCountTcp + connectionCountUdp + connectionCountUnixPool + connectionCountUnix 129 | totalError := connectionErrorTcpPool + connectionErrorTcp + connectionErrorUdp + connectionErrorUnixPool + connectionErrorUnix 130 | printReqPerSecond("Total", totalCount, totalError, totalTime) 131 | } 132 | 133 | // printReqPerSecond prints the results in a human-readable format. 134 | func printReqPerSecond(title string, total int, errors int, runTime time.Duration) { 135 | rate := (total - errors) / (int(runTime / time.Second)) 136 | errRate := float64(errors) / float64(total) 137 | fmt.Printf("\n%s:\n-Connections: %d\n-Errors: %d\n-Error Rate: %.6f\n-Req/sec: %d\n", title, total, errors, errRate, rate) 138 | } 139 | 140 | // sendTestMetrics reads from the store and send all metrics to the flush channel on a timer. 141 | func sendTestMetrics(store []Metric, flushChannel chan Metric, finishChannel chan int) { 142 | flushTicker := time.Tick(*flushTime) 143 | 144 | for { 145 | select { 146 | case <-flushTicker: 147 | for _, metric := range store { 148 | flushChannel <- metric 149 | } 150 | case <-finishChannel: 151 | return 152 | } 153 | } 154 | 155 | } 156 | 157 | // flushMetrics continually reads from the flush channel and sends metrics. 158 | func flushMetrics(flushChannel chan Metric, finishChannel chan int) { 159 | 160 | for { 161 | select { 162 | case metric := <-flushChannel: 163 | sendMetricToStats(metric) 164 | case <-finishChannel: 165 | return 166 | } 167 | } 168 | } 169 | 170 | // generateMetricNames generates a specified number of random metric types. 171 | func generateMetricNames(numMetrics int, store []Metric, metricToken string) []Metric { 172 | metricTypes := []string{ 173 | "c", 174 | "g", 175 | "ms", 176 | "s", 177 | } 178 | 179 | rand.Seed(time.Now().UnixNano()) 180 | r := rand.New(rand.NewSource(time.Now().UnixNano())) 181 | 182 | for i := 0; i < numMetrics; i++ { 183 | newMetricName, _ := randutil.String(20, randutil.Alphabet) 184 | newMetricNS := fmt.Sprintf("%sstatsgod.test.%s", metricToken, newMetricName) 185 | newMetricCT := 1 186 | if *connType > 0 && *connType < 6 { 187 | newMetricCT = *connType 188 | } else { 189 | newMetricCT, _ = randutil.IntRange(1, 6) 190 | } 191 | 192 | metricType := metricTypes[r.Intn(len(metricTypes))] 193 | 194 | store = append(store, Metric{ 195 | key: newMetricNS, 196 | metricType: metricType, 197 | metricValue: 0, 198 | connectionType: newMetricCT, 199 | }) 200 | } 201 | 202 | return store 203 | } 204 | 205 | // sendMetricToStats formats a message and a value and time and sends to statsgod. 206 | func sendMetricToStats(metric Metric) { 207 | 208 | rand.Seed(time.Now().UnixNano()) 209 | var metricValue int 210 | switch metric.metricType { 211 | case "c": 212 | metricValue = 1 213 | case "g": 214 | metricValue = rand.Intn(100) 215 | case "ms": 216 | metricValue = rand.Intn(1000) 217 | case "s": 218 | metricValue = rand.Intn(100) 219 | } 220 | stringValue := fmt.Sprintf("%s:%d|%s", metric.key, metricValue, metric.metricType) 221 | // Send to the designated connection 222 | switch metric.connectionType { 223 | case ConnectionTypeTcp: 224 | c, err := net.Dial("tcp", fmt.Sprintf("%s:%d", *statsHost, *statsPortTcp)) 225 | connectionCountTcp++ 226 | if err == nil { 227 | c.Write([]byte(stringValue + "\n")) 228 | logSentMetric(stringValue) 229 | defer c.Close() 230 | } else { 231 | connectionErrorTcp++ 232 | return 233 | } 234 | case ConnectionTypeTcpPool: 235 | c, err := tcpPool.GetConnection(logger) 236 | connectionCountTcpPool++ 237 | if err == nil { 238 | _, err := c.Write([]byte(stringValue + "\n")) 239 | logSentMetric(stringValue) 240 | if err != nil { 241 | connectionErrorTcpPool++ 242 | defer tcpPool.ReleaseConnection(c, true, logger) 243 | } else { 244 | defer tcpPool.ReleaseConnection(c, false, logger) 245 | } 246 | } else { 247 | connectionErrorTcp++ 248 | return 249 | } 250 | case ConnectionTypeUdp: 251 | c, err := net.Dial("udp", fmt.Sprintf("%s:%d", *statsHost, *statsPortUdp)) 252 | connectionCountUdp++ 253 | if err == nil { 254 | c.Write([]byte(stringValue)) 255 | logSentMetric(stringValue) 256 | defer c.Close() 257 | } else { 258 | connectionErrorUdp++ 259 | return 260 | } 261 | case ConnectionTypeUnix: 262 | c, err := net.Dial("unix", *statsSock) 263 | connectionCountUnix++ 264 | if err == nil { 265 | c.Write([]byte(stringValue)) 266 | logSentMetric(stringValue) 267 | defer c.Close() 268 | } else { 269 | connectionErrorUnix++ 270 | return 271 | } 272 | case ConnectionTypeUnixPool: 273 | c, err := unixPool.GetConnection(logger) 274 | connectionCountUnixPool++ 275 | if err == nil { 276 | _, err := c.Write([]byte(stringValue + "\n")) 277 | logSentMetric(stringValue) 278 | if err != nil { 279 | connectionErrorUnixPool++ 280 | defer unixPool.ReleaseConnection(c, true, logger) 281 | } else { 282 | defer unixPool.ReleaseConnection(c, false, logger) 283 | } 284 | } else { 285 | connectionErrorUnixPool++ 286 | return 287 | } 288 | } 289 | } 290 | 291 | // logSentMetric will log the metric string for QA spot checking. 292 | func logSentMetric(metric string) { 293 | if *logSent { 294 | fmt.Printf("%s\n", metric) 295 | } 296 | } 297 | -------------------------------------------------------------------------------- /extras/log_compare.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Acquia, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "flag" 21 | "fmt" 22 | "io/ioutil" 23 | "os" 24 | "regexp" 25 | "strings" 26 | ) 27 | 28 | var logInputFile = flag.String("in", "/tmp/statsgod.input", "The stdout from loadtest.go") 29 | var logOutputFile = flag.String("out", "/tmp/statsgod.output", "The stdout from statsgod.go") 30 | var token = flag.String("token", "", "An auth token to prepend to the metric string.") 31 | 32 | func main() { 33 | flag.Usage = func() { 34 | usage := `Usage of %s: 35 | 36 | 1. Start statsgod.go with config.debug.receipt set to true and redirect to a file: 37 | go run statsgod.go 2>&1 /tmp/statsgod.output 38 | 39 | 2. Start loadtest.go with the -logSent flag and redirect to a file: 40 | go run extras/loadtest.go [options] -logSent=true 2>&1 /tmp/statsgod.input 41 | 42 | 3. After collecting input and output, compare using this utility: 43 | go run extras/log_compare.go -in=/tmp/statsgod.input -out=/tmp/statsgod.output 44 | 45 | ` 46 | fmt.Fprintf(os.Stderr, usage, os.Args[0]) 47 | flag.PrintDefaults() 48 | } 49 | 50 | flag.Parse() 51 | 52 | inputBytes, err := ioutil.ReadFile(*logInputFile) 53 | inputString := string(inputBytes) 54 | if err != nil { 55 | panic("Could not open input file.") 56 | } 57 | inputLines := strings.Split(inputString, "\n") 58 | inputStrings := make(map[string]int) 59 | 60 | outputBytes, err := ioutil.ReadFile(*logOutputFile) 61 | outputString := string(outputBytes) 62 | if err != nil { 63 | panic("Could not open output file.") 64 | } 65 | outputLines := strings.Split(outputString, "\n") 66 | outputStrings := make(map[string]int) 67 | var outputLine string 68 | 69 | inputLen := len(inputLines) 70 | outputLen := len(outputLines) 71 | totalLines := inputLen 72 | 73 | fmt.Printf("Lines in input file (%s): %d\n", *logInputFile, inputLen) 74 | fmt.Printf("Lines in output file (%s): %d\n", *logOutputFile, outputLen) 75 | 76 | var metricType string 77 | var metricKey string 78 | 79 | if inputLen < outputLen { 80 | totalLines = outputLen 81 | } 82 | 83 | outputRegex := regexp.MustCompile("^.*Metric: \\{([^\\s]+)\\s([^\\s]+)\\s.*[\\s|\\[]([0-9\\.]+)\\].*$") 84 | 85 | inputStringsLen := 0 86 | outputStringsLen := 0 87 | 88 | // If the user specified an auth token, construct it here. 89 | metricToken := "" 90 | if *token != "" { 91 | metricToken = *token + "\\." 92 | } 93 | 94 | for i := 0; i < totalLines; i++ { 95 | metricType = "" 96 | metricKey = "" 97 | 98 | if i < inputLen { 99 | inputR, _ := regexp.Match("^"+metricToken+"[^\\s]+\\:[0-9\\.]+\\|(c|g|s|ms).*$", []byte(inputLines[i])) 100 | if inputR { 101 | // Strip off the token if it exists as the output won't specify it. 102 | if metricToken != "" { 103 | metricKey = strings.Replace(inputLines[i], *token+".", "", 1) 104 | } else { 105 | metricKey = inputLines[i] 106 | } 107 | 108 | inputStringsLen++ 109 | if inputStrings[metricKey] == 0 { 110 | inputStrings[metricKey] = 1 111 | } else { 112 | inputStrings[metricKey]++ 113 | } 114 | } 115 | } 116 | 117 | if i < outputLen { 118 | outputR := outputRegex.FindAllStringSubmatch(outputLines[i], -1) 119 | if len(outputR) > 0 && len(outputR[0]) == 4 { 120 | switch outputR[0][2] { 121 | case "0": 122 | metricType = "c" 123 | case "1": 124 | metricType = "g" 125 | case "2": 126 | metricType = "s" 127 | case "3": 128 | metricType = "ms" 129 | } 130 | outputLine = fmt.Sprintf("%s:%s|%s", outputR[0][1], outputR[0][3], metricType) 131 | outputStringsLen++ 132 | if outputStrings[outputLine] == 0 { 133 | outputStrings[outputLine] = 1 134 | } else { 135 | outputStrings[outputLine]++ 136 | } 137 | } 138 | } 139 | 140 | } 141 | 142 | fmt.Printf("Input metrics: %d\n", inputStringsLen) 143 | fmt.Printf("Output metrics: %d\n", outputStringsLen) 144 | 145 | totalLines = inputStringsLen 146 | if inputStringsLen < outputStringsLen { 147 | totalLines = outputStringsLen 148 | } 149 | 150 | matchCount := 0 151 | errorCount := 0 152 | 153 | for inputMetric, inputCount := range inputStrings { 154 | if inputCount == outputStrings[inputMetric] { 155 | matchCount++ 156 | delete(inputStrings, inputMetric) 157 | delete(outputStrings, inputMetric) 158 | } else { 159 | errorCount++ 160 | } 161 | } 162 | 163 | fmt.Printf("Consolidated metric matches: %d\n", matchCount) 164 | fmt.Printf("Consolidated metric errors: %d\n", errorCount) 165 | fmt.Printf("Input not matched: %v\n", inputStrings) 166 | fmt.Printf("Output not matched: %v\n", outputStrings) 167 | } 168 | -------------------------------------------------------------------------------- /extras/receiver.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Acquia, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Package receiver is a sample receiver app. 18 | package main 19 | 20 | /** 21 | * Test receiver used to debug statsgod that listens on localhost:2003 22 | */ 23 | 24 | import ( 25 | "bufio" 26 | "fmt" 27 | "io" 28 | "net" 29 | "os" 30 | ) 31 | 32 | func main() { 33 | ln, err := net.Listen("tcp", ":2003") 34 | if err != nil { 35 | fmt.Println(err) 36 | os.Exit(1) 37 | } 38 | for { 39 | conn, err := ln.Accept() 40 | if err != nil { 41 | fmt.Println("Error: ", err) 42 | continue 43 | } 44 | go handleConnection(conn) 45 | } 46 | } 47 | 48 | func handleConnection(c net.Conn) { 49 | // TODO - this isn't working for new lines? 50 | status, err := bufio.NewReader(c).ReadString('\n') 51 | if err != nil && err != io.EOF { 52 | fmt.Println("Error: ", err) 53 | os.Exit(1) 54 | } 55 | fmt.Printf("New client: %v\n%s", c.RemoteAddr(), status) 56 | 57 | c.Close() 58 | fmt.Printf("Connection from %v closed.\n", c.RemoteAddr()) 59 | } 60 | -------------------------------------------------------------------------------- /statsgod.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Acquia, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /** 18 | * Package main: statsgod is a metrics aggregator inspired by statsd. The 19 | * main feature is to provide a server which accepts metrics over time, 20 | * aggregates them and forwards on to permanent storage. 21 | */ 22 | package main 23 | 24 | import ( 25 | "flag" 26 | "fmt" 27 | "github.com/acquia/statsgod/statsgod" 28 | "io/ioutil" 29 | "os" 30 | "runtime/pprof" 31 | ) 32 | 33 | const ( 34 | // AvailableMemory is amount of available memory for the process. 35 | AvailableMemory = 10 << 20 // 10 MB, for example 36 | // AverageMemoryPerRequest is how much memory we want to use per request. 37 | AverageMemoryPerRequest = 10 << 10 // 10 KB 38 | // MaxReqs is how many requests. 39 | MaxReqs = AvailableMemory / AverageMemoryPerRequest 40 | ) 41 | 42 | // parseChannel containing received metric strings. 43 | var parseChannel = make(chan string, MaxReqs) 44 | 45 | // relayChannel containing the Metric objects. 46 | var relayChannel = make(chan *statsgod.Metric, MaxReqs) 47 | 48 | // finishChannel is used to respond to a quit signal. 49 | var finishChannel = make(chan int) 50 | 51 | // quit is a shared value to instruct looping goroutines to stop. 52 | var quit = false 53 | 54 | // CLI flags. 55 | var configFile = flag.String("config", "/etc/statsgod/config.yml", "YAML config file path") 56 | 57 | func main() { 58 | // Load command line options. 59 | flag.Parse() 60 | 61 | // Load the YAML config. 62 | var config, _ = statsgod.CreateConfig(*configFile) 63 | 64 | // Set up the logger based on the configuration. 65 | var logger statsgod.Logger 66 | if config.Debug.Verbose { 67 | logger = *statsgod.CreateLogger(os.Stdout, os.Stdout, os.Stdout, os.Stderr) 68 | logger.Info.Println("Debugging mode enabled") 69 | logger.Info.Printf("Loaded Config: %v", config) 70 | } else { 71 | logger = *statsgod.CreateLogger(ioutil.Discard, os.Stdout, os.Stdout, os.Stderr) 72 | } 73 | 74 | // Set up profiling. 75 | if config.Debug.Profile { 76 | f, err := os.Create("statsgod.prof") 77 | if err != nil { 78 | panic(err) 79 | } 80 | pprof.StartCPUProfile(f) 81 | } 82 | 83 | // Set up the backend relay. 84 | relay := statsgod.CreateRelay(config, logger) 85 | 86 | // Set up the authentication. 87 | auth := statsgod.CreateAuth(config) 88 | 89 | // Parse the incoming messages and convert to metrics. 90 | go statsgod.ParseMetrics(parseChannel, relayChannel, auth, logger, &quit) 91 | 92 | // Flush the metrics to the remote stats collector. 93 | for i := 0; i < config.Relay.Concurrency; i++ { 94 | go statsgod.RelayMetrics(relay, relayChannel, logger, &config, &quit) 95 | } 96 | 97 | var socketTcp statsgod.Socket 98 | var socketUdp statsgod.Socket 99 | var socketUnix statsgod.Socket 100 | 101 | // Listen on the TCP socket. 102 | if config.Connection.Tcp.Enabled { 103 | tcpAddr := fmt.Sprintf("%s:%d", config.Connection.Tcp.Host, config.Connection.Tcp.Port) 104 | socketTcp = statsgod.CreateSocket(statsgod.SocketTypeTcp, tcpAddr).(*statsgod.SocketTcp) 105 | go socketTcp.Listen(parseChannel, logger, &config) 106 | } 107 | 108 | // Listen on the UDP socket. 109 | if config.Connection.Udp.Enabled { 110 | udpAddr := fmt.Sprintf("%s:%d", config.Connection.Udp.Host, config.Connection.Udp.Port) 111 | socketUdp = statsgod.CreateSocket(statsgod.SocketTypeUdp, udpAddr).(*statsgod.SocketUdp) 112 | go socketUdp.Listen(parseChannel, logger, &config) 113 | } 114 | 115 | // Listen on the Unix socket. 116 | if config.Connection.Unix.Enabled { 117 | socketUnix = statsgod.CreateSocket(statsgod.SocketTypeUnix, config.Connection.Unix.File).(*statsgod.SocketUnix) 118 | go socketUnix.Listen(parseChannel, logger, &config) 119 | } 120 | 121 | // Listen for OS signals. 122 | statsgod.ListenForSignals(finishChannel, &config, configFile, logger) 123 | 124 | // Wait until the program is finished. 125 | select { 126 | case <-finishChannel: 127 | logger.Info.Println("Exiting program.") 128 | quit = true 129 | if config.Connection.Tcp.Enabled { 130 | socketTcp.Close(logger) 131 | } 132 | if config.Connection.Udp.Enabled { 133 | socketUdp.Close(logger) 134 | } 135 | if config.Connection.Unix.Enabled { 136 | socketUnix.Close(logger) 137 | } 138 | if config.Debug.Profile { 139 | pprof.StopCPUProfile() 140 | } 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /statsgod/auth.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Acquia, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Package statsgod - this library manages authentication methods. All auth 18 | // currently happens at the metric level. This means that all authentication 19 | // parameters must be included in the metric string itself before it is parsed. 20 | package statsgod 21 | 22 | import ( 23 | "errors" 24 | "strings" 25 | ) 26 | 27 | const ( 28 | // AuthTypeNone defines a runtime with no authentication. 29 | AuthTypeNone = "none" 30 | // AuthTypeConfigToken is a token-based auth which is retrieved from 31 | // the main configuration file. 32 | AuthTypeConfigToken = "token" 33 | ) 34 | 35 | // CreateAuth is a factory to create an Auth object. 36 | func CreateAuth(config ConfigValues) Auth { 37 | switch config.Service.Auth { 38 | case AuthTypeConfigToken: 39 | tokenAuth := new(AuthConfigToken) 40 | tokenAuth.Tokens = config.Service.Tokens 41 | return tokenAuth 42 | 43 | case AuthTypeNone: 44 | fallthrough 45 | default: 46 | return new(AuthNone) 47 | } 48 | } 49 | 50 | // Auth is an interface describing statsgod authentication objects. 51 | type Auth interface { 52 | // Authenticate takes a metric string and authenticates it. The metric 53 | // is passed by reference in case the Auth object needs to manipulate 54 | // it, such as removing a token. 55 | Authenticate(metric *string) (bool, error) 56 | } 57 | 58 | // AuthNone does nothing and allows all traffic. 59 | type AuthNone struct { 60 | } 61 | 62 | // AuthConfigToken checks the configuration file for a valid auth token. 63 | type AuthConfigToken struct { 64 | // Tokens contains a list of valid auth tokens with a token as the key 65 | // which maps to a boolean defining the validity of the token. 66 | Tokens map[string]bool 67 | } 68 | 69 | // Authenticate conforms to Auth.Authenticate() 70 | func (a AuthNone) Authenticate(metric *string) (bool, error) { 71 | return true, nil 72 | } 73 | 74 | // Authenticate conforms to Auth.Authenticate() 75 | func (a AuthConfigToken) Authenticate(metric *string) (bool, error) { 76 | authSplit := strings.SplitN(*metric, ".", 2) 77 | if len(authSplit) == 2 { 78 | token, exists := a.Tokens[authSplit[0]] 79 | if !exists || !token { 80 | authError := errors.New("Invalid authentication token") 81 | return false, authError 82 | } 83 | } else { 84 | authError := errors.New("Missing authentication token") 85 | return false, authError 86 | } 87 | *metric = authSplit[1] 88 | return true, nil 89 | } 90 | -------------------------------------------------------------------------------- /statsgod/auth_test.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Acquia, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package statsgod_test 18 | 19 | import ( 20 | . "github.com/acquia/statsgod/statsgod" 21 | . "github.com/onsi/ginkgo" 22 | . "github.com/onsi/gomega" 23 | "reflect" 24 | ) 25 | 26 | var _ = Describe("Auth", func() { 27 | 28 | var testToken = "test-token" 29 | var testMetric = "foo.bar:123|g" 30 | var config ConfigValues 31 | 32 | // The boolean value is whether or not the token auth should succeed. 33 | var testMetrics = map[string]bool{ 34 | "": false, 35 | "foo": false, 36 | testMetric: false, 37 | testToken + ".foo": true, 38 | testToken: false, 39 | testToken + "." + testMetric: true, 40 | } 41 | 42 | Describe("Testing the authentication", func() { 43 | BeforeEach(func() { 44 | config, _ = CreateConfig("") 45 | }) 46 | 47 | Context("when using the factory function", func() { 48 | It("should be a complete structure", func() { 49 | // Create an token config auth. 50 | config.Service.Auth = AuthTypeConfigToken 51 | tokenAuth := CreateAuth(config) 52 | Expect(tokenAuth).ShouldNot(BeNil()) 53 | Expect(reflect.TypeOf(tokenAuth).String()).Should(Equal("*statsgod.AuthConfigToken")) 54 | 55 | // Create a no auth. 56 | config.Service.Auth = AuthTypeNone 57 | noAuth := CreateAuth(config) 58 | Expect(noAuth).ShouldNot(BeNil()) 59 | Expect(reflect.TypeOf(noAuth).String()).Should(Equal("*statsgod.AuthNone")) 60 | 61 | // No auth should be default. 62 | config.Service.Auth = "foo" 63 | defaultAuth := CreateAuth(config) 64 | Expect(defaultAuth).ShouldNot(BeNil()) 65 | Expect(reflect.TypeOf(defaultAuth).String()).Should(Equal("*statsgod.AuthNone")) 66 | 67 | }) 68 | }) 69 | 70 | Context("when using AuthNone", func() { 71 | It("should always authenticate", func() { 72 | config.Service.Auth = AuthTypeNone 73 | noAuth := CreateAuth(config) 74 | 75 | // No auth should always authenticate. 76 | for metric, _ := range testMetrics { 77 | auth, _ := noAuth.Authenticate(&metric) 78 | Expect(auth).Should(BeTrue()) 79 | } 80 | }) 81 | }) 82 | 83 | Context("when using AuthConfigToken", func() { 84 | It("AuthConfigToken should only allow valid tokens", func() { 85 | config.Service.Auth = AuthTypeConfigToken 86 | tokenAuth := CreateAuth(config).(*AuthConfigToken) 87 | tokenAuth.Tokens = map[string]bool{testToken: true} 88 | 89 | // Auth should only authenticate with a valid token. 90 | for metric, valid := range testMetrics { 91 | auth, _ := tokenAuth.Authenticate(&metric) 92 | Expect(auth).Should(Equal(valid)) 93 | } 94 | 95 | // Test that the token is stripped from the metric. 96 | validMetric := testToken + "." + testMetric 97 | tokenAuth.Authenticate(&validMetric) 98 | Expect(validMetric).Should(Equal(testMetric)) 99 | 100 | // Invalidate the token to ensure that auth fails. 101 | tokenAuth.Tokens[testToken] = false 102 | invalidMetric := testToken + "." + testMetric 103 | invalidAuth, _ := tokenAuth.Authenticate(&invalidMetric) 104 | Expect(invalidAuth).Should(BeFalse()) 105 | }) 106 | }) 107 | }) 108 | }) 109 | -------------------------------------------------------------------------------- /statsgod/config.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Acquia, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Package statsgod - This library handles the file-based runtime configuration. 18 | package statsgod 19 | 20 | import ( 21 | "gopkg.in/yaml.v2" 22 | "io/ioutil" 23 | "os" 24 | "regexp" 25 | "time" 26 | ) 27 | 28 | // ConfigValues describes the data type that configuration is loaded into. The 29 | // values from the YAML config file map directly to these values. e.g. 30 | // 31 | // service: 32 | // name: statsgod 33 | // debug: false 34 | // 35 | // Map to: 36 | // config.Service.Name = "statsgod" 37 | // config.Service.Debug = false 38 | // 39 | // All values specified in the ConfigValues struct should also have a default 40 | // value set in LoadFile() to ensure a safe runtime environment. 41 | type ConfigValues struct { 42 | Service struct { 43 | Name string 44 | Debug bool 45 | Auth string 46 | Tokens map[string]bool 47 | Hostname string 48 | } 49 | Connection struct { 50 | Tcp struct { 51 | Enabled bool 52 | Host string 53 | Port int 54 | } 55 | Udp struct { 56 | Enabled bool 57 | Host string 58 | Port int 59 | Maxpacket int 60 | } 61 | Unix struct { 62 | Enabled bool 63 | File string 64 | } 65 | } 66 | Relay struct { 67 | Type string 68 | Concurrency int 69 | Timeout time.Duration 70 | Flush time.Duration 71 | } 72 | Namespace struct { 73 | Prefix string 74 | Prefixes struct { 75 | Counters string 76 | Gauges string 77 | Rates string 78 | Sets string 79 | Timers string 80 | } 81 | Suffix string 82 | Suffixes struct { 83 | Counters string 84 | Gauges string 85 | Rates string 86 | Sets string 87 | Timers string 88 | } 89 | } 90 | Carbon struct { 91 | Host string 92 | Port int 93 | } 94 | Stats struct { 95 | Percentile []int 96 | } 97 | Debug struct { 98 | Verbose bool 99 | Receipt bool 100 | Profile bool 101 | Relay bool 102 | } 103 | } 104 | 105 | // CreateConfig is a factory for creating ConfigValues. 106 | func CreateConfig(filePath string) (ConfigValues, error) { 107 | config := new(ConfigValues) 108 | err := config.LoadFile(filePath) 109 | return *config, err 110 | } 111 | 112 | // LoadFile will read configuration from a specified file. 113 | func (config *ConfigValues) LoadFile(filePath string) error { 114 | var err error 115 | 116 | // Establish all of the default values. 117 | 118 | // Service 119 | config.Service.Name = "statsgod" 120 | config.Service.Debug = false 121 | config.Service.Auth = "none" 122 | config.Service.Hostname = "" 123 | 124 | // Connection 125 | config.Connection.Tcp.Enabled = true 126 | config.Connection.Tcp.Host = "127.0.0.1" 127 | config.Connection.Tcp.Port = 8125 128 | config.Connection.Udp.Enabled = true 129 | config.Connection.Udp.Host = "127.0.0.1" 130 | config.Connection.Udp.Port = 8126 131 | config.Connection.Udp.Maxpacket = 1024 132 | config.Connection.Unix.Enabled = true 133 | config.Connection.Unix.File = "/var/run/statsgod/statsgod.sock" 134 | 135 | // Relay 136 | config.Relay.Type = RelayTypeCarbon 137 | config.Relay.Concurrency = 1 138 | config.Relay.Timeout = 30 * time.Second 139 | config.Relay.Flush = 10 * time.Second 140 | 141 | // Carbon 142 | config.Carbon.Host = "127.0.0.1" 143 | config.Carbon.Port = 2003 144 | 145 | // Namespace 146 | config.Namespace.Prefix = "stats" 147 | config.Namespace.Prefixes.Counters = "counts" 148 | config.Namespace.Prefixes.Gauges = "gauges" 149 | config.Namespace.Prefixes.Rates = "rates" 150 | config.Namespace.Prefixes.Sets = "sets" 151 | config.Namespace.Prefixes.Timers = "timers" 152 | config.Namespace.Suffix = "" 153 | config.Namespace.Suffixes.Counters = "" 154 | config.Namespace.Suffixes.Gauges = "" 155 | config.Namespace.Suffixes.Rates = "" 156 | config.Namespace.Suffixes.Sets = "" 157 | config.Namespace.Suffixes.Timers = "" 158 | 159 | // Debug 160 | config.Debug.Verbose = false 161 | config.Debug.Receipt = false 162 | config.Debug.Profile = false 163 | config.Debug.Relay = false 164 | 165 | // Attempt to read in the file. 166 | if filePath != "" { 167 | contents, readError := ioutil.ReadFile(filePath) 168 | if readError != nil { 169 | err = readError 170 | } else { 171 | err = yaml.Unmarshal([]byte(contents), &config) 172 | } 173 | } 174 | 175 | // The yaml parser will append array values, so to avoid duplicates we 176 | // only add the default when there are none specified in the yaml. 177 | if len(config.Stats.Percentile) == 0 { 178 | config.Stats.Percentile = []int{80} 179 | } 180 | 181 | // Similarly with the tokens, which is a map, only create a default if 182 | // one was not read in from the yaml. 183 | if len(config.Service.Tokens) == 0 { 184 | config.Service.Tokens = map[string]bool{"token-name": false} 185 | } 186 | 187 | // If the hostname is empty, use the current one. 188 | config.Service.Hostname = GetHostname(config.Service.Hostname) 189 | return err 190 | } 191 | 192 | // GetHostname determines the current hostname if the provided default is empty. 193 | func GetHostname(defaultValue string) (hostname string) { 194 | hostname = "unknown" 195 | if defaultValue == "" { 196 | hn, err := os.Hostname() 197 | if err == nil { 198 | hostname = hn 199 | } 200 | } else { 201 | hostname = defaultValue 202 | } 203 | re := regexp.MustCompile("[^a-zA-Z0-9]") 204 | hostname = re.ReplaceAllString(hostname, "-") 205 | 206 | return 207 | } 208 | -------------------------------------------------------------------------------- /statsgod/config_test.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Acquia, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package statsgod_test 18 | 19 | import ( 20 | . "github.com/acquia/statsgod/statsgod" 21 | . "github.com/onsi/ginkgo" 22 | . "github.com/onsi/gomega" 23 | ) 24 | 25 | var _ = Describe("Config", func() { 26 | 27 | var ( 28 | config ConfigValues 29 | yaml ConfigValues 30 | fileErr error 31 | exampleConfigFile string = "../example.config.yml" 32 | ) 33 | 34 | Describe("Loading runtime configuration", func() { 35 | Context("Loading default values", func() { 36 | config, _ = CreateConfig("") 37 | It("should contain defaults", func() { 38 | // Service 39 | Expect(config.Service.Name).ShouldNot(Equal(nil)) 40 | Expect(config.Service.Auth).ShouldNot(Equal(nil)) 41 | Expect(config.Service.Tokens).ShouldNot(Equal(nil)) 42 | Expect(config.Service.Hostname).ShouldNot(Equal(nil)) 43 | 44 | // Connection 45 | Expect(config.Connection.Tcp.Enabled).ShouldNot(Equal(nil)) 46 | Expect(config.Connection.Tcp.Host).ShouldNot(Equal(nil)) 47 | Expect(config.Connection.Tcp.Port).ShouldNot(Equal(nil)) 48 | Expect(config.Connection.Udp.Enabled).ShouldNot(Equal(nil)) 49 | Expect(config.Connection.Udp.Host).ShouldNot(Equal(nil)) 50 | Expect(config.Connection.Udp.Port).ShouldNot(Equal(nil)) 51 | Expect(config.Connection.Udp.Maxpacket).ShouldNot(Equal(nil)) 52 | Expect(config.Connection.Unix.Enabled).ShouldNot(Equal(nil)) 53 | Expect(config.Connection.Unix.File).ShouldNot(Equal(nil)) 54 | 55 | // Relay 56 | Expect(config.Relay.Type).ShouldNot(Equal(nil)) 57 | Expect(config.Relay.Concurrency).ShouldNot(Equal(nil)) 58 | Expect(config.Relay.Timeout).ShouldNot(Equal(nil)) 59 | Expect(config.Relay.Flush).ShouldNot(Equal(nil)) 60 | 61 | // Carbon 62 | Expect(config.Carbon.Host).ShouldNot(Equal(nil)) 63 | Expect(config.Carbon.Port).ShouldNot(Equal(nil)) 64 | 65 | // Stats 66 | Expect(config.Stats.Percentile).ShouldNot(Equal(nil)) 67 | 68 | // Namespace 69 | Expect(config.Namespace.Prefix).ShouldNot(Equal(nil)) 70 | Expect(config.Namespace.Prefixes.Counters).ShouldNot(Equal(nil)) 71 | Expect(config.Namespace.Prefixes.Gauges).ShouldNot(Equal(nil)) 72 | Expect(config.Namespace.Prefixes.Rates).ShouldNot(Equal(nil)) 73 | Expect(config.Namespace.Prefixes.Sets).ShouldNot(Equal(nil)) 74 | Expect(config.Namespace.Prefixes.Timers).ShouldNot(Equal(nil)) 75 | Expect(config.Namespace.Suffix).ShouldNot(Equal(nil)) 76 | Expect(config.Namespace.Suffixes.Counters).ShouldNot(Equal(nil)) 77 | Expect(config.Namespace.Suffixes.Gauges).ShouldNot(Equal(nil)) 78 | Expect(config.Namespace.Suffixes.Rates).ShouldNot(Equal(nil)) 79 | Expect(config.Namespace.Suffixes.Sets).ShouldNot(Equal(nil)) 80 | Expect(config.Namespace.Suffixes.Timers).ShouldNot(Equal(nil)) 81 | 82 | // Debug 83 | Expect(config.Debug.Verbose).ShouldNot(Equal(nil)) 84 | Expect(config.Debug.Receipt).ShouldNot(Equal(nil)) 85 | Expect(config.Debug.Profile).ShouldNot(Equal(nil)) 86 | Expect(config.Debug.Relay).ShouldNot(Equal(nil)) 87 | }) 88 | }) 89 | 90 | Context("Loading config file", func() { 91 | yaml, fileErr = CreateConfig(exampleConfigFile) 92 | It("should match the defaults", func() { 93 | Expect(fileErr).Should(BeNil()) 94 | 95 | hostname := GetHostname("") 96 | hostnameDefault := GetHostname("default") 97 | Expect(hostnameDefault).Should(Equal("default")) 98 | 99 | // Service 100 | Expect(yaml.Service.Name).Should(Equal(config.Service.Name)) 101 | Expect(yaml.Service.Auth).Should(Equal(config.Service.Auth)) 102 | Expect(yaml.Service.Tokens).Should(Equal(config.Service.Tokens)) 103 | Expect(hostname).Should(Equal(config.Service.Hostname)) 104 | 105 | // Connection 106 | Expect(yaml.Connection.Tcp.Enabled).Should(Equal(config.Connection.Tcp.Enabled)) 107 | Expect(yaml.Connection.Tcp.Host).Should(Equal(config.Connection.Tcp.Host)) 108 | Expect(yaml.Connection.Tcp.Port).Should(Equal(config.Connection.Tcp.Port)) 109 | Expect(yaml.Connection.Udp.Enabled).Should(Equal(config.Connection.Udp.Enabled)) 110 | Expect(yaml.Connection.Udp.Host).Should(Equal(config.Connection.Udp.Host)) 111 | Expect(yaml.Connection.Udp.Port).Should(Equal(config.Connection.Udp.Port)) 112 | Expect(yaml.Connection.Udp.Maxpacket).Should(Equal(config.Connection.Udp.Maxpacket)) 113 | Expect(yaml.Connection.Unix.Enabled).Should(Equal(config.Connection.Unix.Enabled)) 114 | Expect(yaml.Connection.Unix.File).Should(Equal(config.Connection.Unix.File)) 115 | 116 | // Relay 117 | Expect(yaml.Relay.Type).Should(Equal(config.Relay.Type)) 118 | Expect(yaml.Relay.Concurrency).Should(Equal(config.Relay.Concurrency)) 119 | Expect(yaml.Relay.Timeout).Should(Equal(config.Relay.Timeout)) 120 | Expect(yaml.Relay.Flush).Should(Equal(config.Relay.Flush)) 121 | 122 | // Carbon 123 | Expect(yaml.Carbon.Host).Should(Equal(config.Carbon.Host)) 124 | Expect(yaml.Carbon.Port).Should(Equal(config.Carbon.Port)) 125 | 126 | // Stats 127 | Expect(yaml.Stats.Percentile).Should(Equal(config.Stats.Percentile)) 128 | 129 | // Namespace 130 | Expect(yaml.Namespace.Prefix).Should(Equal(config.Namespace.Prefix)) 131 | Expect(yaml.Namespace.Prefixes.Counters).Should(Equal(config.Namespace.Prefixes.Counters)) 132 | Expect(yaml.Namespace.Prefixes.Gauges).Should(Equal(config.Namespace.Prefixes.Gauges)) 133 | Expect(yaml.Namespace.Prefixes.Rates).Should(Equal(config.Namespace.Prefixes.Rates)) 134 | Expect(yaml.Namespace.Prefixes.Sets).Should(Equal(config.Namespace.Prefixes.Sets)) 135 | Expect(yaml.Namespace.Prefixes.Timers).Should(Equal(config.Namespace.Prefixes.Timers)) 136 | Expect(yaml.Namespace.Suffix).Should(Equal(config.Namespace.Suffix)) 137 | Expect(yaml.Namespace.Suffixes.Counters).Should(Equal(config.Namespace.Suffixes.Counters)) 138 | Expect(yaml.Namespace.Suffixes.Gauges).Should(Equal(config.Namespace.Suffixes.Gauges)) 139 | Expect(yaml.Namespace.Suffixes.Rates).Should(Equal(config.Namespace.Suffixes.Rates)) 140 | Expect(yaml.Namespace.Suffixes.Sets).Should(Equal(config.Namespace.Suffixes.Sets)) 141 | Expect(yaml.Namespace.Suffixes.Timers).Should(Equal(config.Namespace.Suffixes.Timers)) 142 | 143 | // Debug 144 | Expect(yaml.Debug.Verbose).Should(Equal(config.Debug.Verbose)) 145 | Expect(yaml.Debug.Receipt).Should(Equal(config.Debug.Receipt)) 146 | Expect(yaml.Debug.Profile).Should(Equal(config.Debug.Profile)) 147 | Expect(yaml.Debug.Relay).Should(Equal(config.Debug.Relay)) 148 | 149 | }) 150 | }) 151 | 152 | Context("Loading bogus file", func() { 153 | It("should throw an error", func() { 154 | _, noFileErr := CreateConfig("noFile") 155 | Expect(noFileErr).ShouldNot(Equal(nil)) 156 | }) 157 | }) 158 | 159 | Context("Re-loading file", func() { 160 | It("should update the values", func() { 161 | runtimeConf, _ := CreateConfig(exampleConfigFile) 162 | originalName := runtimeConf.Service.Name 163 | runtimeConf.Service.Name = "SomethingNew" 164 | runtimeConf.LoadFile(exampleConfigFile) 165 | Expect(runtimeConf.Service.Name).Should(Equal(originalName)) 166 | }) 167 | }) 168 | }) 169 | 170 | }) 171 | -------------------------------------------------------------------------------- /statsgod/connectionpool.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Acquia, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package statsgod 18 | 19 | import ( 20 | "errors" 21 | "fmt" 22 | "net" 23 | "time" 24 | ) 25 | 26 | const ( 27 | // ConnPoolTypeTcp is an enum describing a TCP connection pool. 28 | ConnPoolTypeTcp = iota 29 | // ConnPoolTypeUnix is an enum describing a Unix Socket connection pool. 30 | ConnPoolTypeUnix 31 | // ConnPoolTypeNone is for testing. 32 | ConnPoolTypeNone 33 | ) 34 | 35 | // ConnectionPool maintains a channel of connections to a remote host. 36 | type ConnectionPool struct { 37 | // Size indicates the number of connections to keep open. 38 | Size int 39 | // Connections is the channel to push new/reused connections onto. 40 | Connections chan net.Conn 41 | // Addr is the string representing the address of the socket. 42 | Addr string 43 | // Type is the type of connection the pool will make. 44 | Type int 45 | // Timeout is the amount of time to wait for a connection. 46 | Timeout time.Duration 47 | // ErrorCount tracks the number of connection errors that have occured. 48 | ErrorCount int 49 | } 50 | 51 | // CreateConnectionPool creates instances of ConnectionPool. 52 | func CreateConnectionPool(size int, addr string, connType int, timeout time.Duration, logger Logger) (*ConnectionPool, error) { 53 | var pool = new(ConnectionPool) 54 | pool.Size = size 55 | pool.Addr = addr 56 | pool.Type = connType 57 | pool.Timeout = timeout 58 | pool.ErrorCount = 0 59 | pool.Connections = make(chan net.Conn, size) 60 | 61 | errorCount := 0 62 | for i := 0; i < size; i++ { 63 | added, err := pool.CreateConnection(logger) 64 | if !added || err != nil { 65 | errorCount++ 66 | } 67 | } 68 | 69 | if errorCount > 0 { 70 | err := fmt.Errorf("%d connections failed", errorCount) 71 | return pool, err 72 | } 73 | 74 | return pool, nil 75 | } 76 | 77 | // CreateConnection attempts to contact the remote relay host. 78 | func (pool *ConnectionPool) CreateConnection(logger Logger) (bool, error) { 79 | 80 | if len(pool.Connections) < pool.Size { 81 | logger.Info.Printf("Connecting to %s", pool.Addr) 82 | // Establish a new connection and set the timeout accordingly. 83 | var connType string 84 | switch pool.Type { 85 | case ConnPoolTypeTcp: 86 | connType = "tcp" 87 | case ConnPoolTypeUnix: 88 | connType = "unix" 89 | default: 90 | err := errors.New("Unable to create a connection of the specified type.") 91 | return false, err 92 | } 93 | conn, err := net.Dial(connType, pool.Addr) 94 | if err != nil { 95 | pool.ErrorCount++ 96 | logger.Error.Println("Connection Error.", err) 97 | return false, err 98 | } 99 | conn.SetDeadline(time.Now().Add(pool.Timeout)) 100 | pool.Connections <- conn 101 | return true, nil 102 | } 103 | 104 | err := errors.New("Attempt to add too many connections to the pool.") 105 | return false, err 106 | } 107 | 108 | // GetConnection retrieves a connection from the pool. 109 | func (pool *ConnectionPool) GetConnection(logger Logger) (net.Conn, error) { 110 | select { 111 | case conn := <-pool.Connections: 112 | return conn, nil 113 | case <-time.After(pool.Timeout): 114 | logger.Error.Println("No connections available.") 115 | err := errors.New("Connection timeout.") 116 | nilConn := NilConn{} 117 | return nilConn, err 118 | } 119 | } 120 | 121 | // ReleaseConnection releases a connection back to the pool. 122 | func (pool *ConnectionPool) ReleaseConnection(conn net.Conn, recreate bool, logger Logger) (bool, error) { 123 | // recreate signifies that there was something wrong with the connection and 124 | // that we should make a new one. 125 | if recreate { 126 | conn.Close() 127 | added, err := pool.CreateConnection(logger) 128 | if !added || err != nil { 129 | logger.Error.Println("Could not release connection.", err) 130 | return false, err 131 | } 132 | return true, nil 133 | } 134 | 135 | // Reset the timeout and put it back on the channel. 136 | conn.SetDeadline(time.Now().Add(pool.Timeout)) 137 | pool.Connections <- conn 138 | return true, nil 139 | } 140 | -------------------------------------------------------------------------------- /statsgod/connectionpool_test.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Acquia, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package statsgod_test 18 | 19 | import ( 20 | "fmt" 21 | . "github.com/acquia/statsgod/statsgod" 22 | . "github.com/onsi/ginkgo" 23 | . "github.com/onsi/gomega" 24 | "io/ioutil" 25 | "net" 26 | "strconv" 27 | "strings" 28 | "time" 29 | ) 30 | 31 | var _ = Describe("Connection Pool", func() { 32 | var ( 33 | tmpPort int 34 | logger Logger 35 | maxConnections int = 2 36 | host string = "127.0.0.1" 37 | timeout time.Duration = 1 * time.Second 38 | ) 39 | 40 | Describe("Testing the basic structure", func() { 41 | It("should contain values", func() { 42 | var pool = new(ConnectionPool) 43 | Expect(pool.Size).ShouldNot(Equal(nil)) 44 | Expect(pool.Addr).ShouldNot(Equal(nil)) 45 | Expect(pool.Timeout).ShouldNot(Equal(nil)) 46 | Expect(pool.ErrorCount).ShouldNot(Equal(nil)) 47 | 48 | Expect(len(pool.Connections)).Should(Equal(0)) 49 | 50 | }) 51 | }) 52 | 53 | Describe("Testing the connection pool functionality", func() { 54 | BeforeEach(func() { 55 | logger = *CreateLogger(ioutil.Discard, ioutil.Discard, ioutil.Discard, ioutil.Discard) 56 | tmpPort = StartTemporaryListener() 57 | }) 58 | 59 | AfterEach(func() { 60 | StopTemporaryListener() 61 | }) 62 | 63 | Context("when we create a new connection pool", func() { 64 | It("should contain values", func() { 65 | addr := fmt.Sprintf("%s:%d", host, tmpPort) 66 | pool, _ := CreateConnectionPool(maxConnections, addr, ConnPoolTypeTcp, timeout, logger) 67 | Expect(pool.Size).Should(Equal(maxConnections)) 68 | Expect(pool.Addr).Should(Equal(addr)) 69 | Expect(pool.Timeout).Should(Equal(timeout)) 70 | Expect(pool.ErrorCount).Should(Equal(0)) 71 | Expect(cap(pool.Connections)).Should(Equal(maxConnections)) 72 | Expect(len(pool.Connections)).Should(Equal(maxConnections)) 73 | }) 74 | 75 | // Test that we get an error if there is no listener. 76 | It("should throw an error if there is no listener", func() { 77 | addr := fmt.Sprintf("%s:%d", host, tmpPort) 78 | StopTemporaryListener() 79 | _, errTcp := CreateConnectionPool(maxConnections, addr, ConnPoolTypeTcp, timeout, logger) 80 | Expect(errTcp).ShouldNot(Equal(nil)) 81 | 82 | _, errUnix := CreateConnectionPool(maxConnections, "/dev/null", ConnPoolTypeUnix, timeout, logger) 83 | Expect(errUnix).ShouldNot(BeNil()) 84 | 85 | _, errType := CreateConnectionPool(maxConnections, "/dev/null", ConnPoolTypeNone, timeout, logger) 86 | Expect(errType).ShouldNot(BeNil()) 87 | }) 88 | 89 | }) 90 | 91 | Context("when we use the connection pool", func() { 92 | It("should contain the right number of connections", func() { 93 | addr := fmt.Sprintf("%s:%d", host, tmpPort) 94 | pool, _ := CreateConnectionPool(maxConnections, addr, ConnPoolTypeTcp, timeout, logger) 95 | 96 | // Check that we established the correct number of connections. 97 | Expect(len(pool.Connections)).Should(Equal(maxConnections)) 98 | 99 | // Check one out and ensure that the length of the channel changes. 100 | connOne, _ := pool.GetConnection(logger) 101 | Expect(len(pool.Connections)).Should(Equal(maxConnections - 1)) 102 | 103 | // Check another one out and ensure that the length of the channel changes. 104 | connTwo, _ := pool.GetConnection(logger) 105 | Expect(len(pool.Connections)).Should(Equal(maxConnections - 2)) 106 | 107 | // Test that we timeout if there are no available connections. 108 | _, err := pool.GetConnection(logger) 109 | Expect(err).ShouldNot(Equal(nil)) 110 | 111 | // Release the connections and check that we are again at max connections. 112 | pool.ReleaseConnection(connOne, false, logger) 113 | pool.ReleaseConnection(connTwo, false, logger) 114 | Expect(len(pool.Connections)).Should(Equal(maxConnections)) 115 | 116 | // Test that we can recreate connections 117 | connThree, _ := pool.GetConnection(logger) 118 | connThree.Close() 119 | pool.ReleaseConnection(connThree, true, logger) 120 | Expect(len(pool.Connections)).Should(Equal(maxConnections)) 121 | 122 | // Test that we cannot create more connections than the pool allows. 123 | _, err = pool.CreateConnection(logger) 124 | Expect(err).ShouldNot(Equal(nil)) 125 | }) 126 | 127 | It("should throw an error if there is no listener.", func() { 128 | addr := fmt.Sprintf("%s:%d", host, tmpPort) 129 | pool, _ := CreateConnectionPool(maxConnections, addr, ConnPoolTypeTcp, timeout, logger) 130 | StopTemporaryListener() 131 | 132 | // Test that we get an error if there is no listener. 133 | badConnection, _ := pool.GetConnection(logger) 134 | _, releaseErr := pool.ReleaseConnection(badConnection, true, logger) 135 | Expect(releaseErr).ShouldNot(Equal(nil)) 136 | }) 137 | 138 | }) 139 | }) 140 | }) 141 | 142 | // tmpListener tracks a local dummy tcp connection. 143 | var tmpListener net.Listener 144 | 145 | // StartTemporaryListener starts a dummy tcp listener. 146 | func StartTemporaryListener() int { 147 | // @todo: move this to a setup/teardown (Issue #29) 148 | // Temporarily listen for the test connection 149 | conn, err := net.Listen("tcp", "127.0.0.1:0") 150 | if err != nil { 151 | panic(err) 152 | } 153 | tmpListener = conn 154 | laddr := strings.Split(conn.Addr().String(), ":") 155 | if len(laddr) < 2 { 156 | panic("Could not get port of listener.") 157 | } 158 | 159 | port, err := strconv.ParseInt(laddr[1], 10, 32) 160 | 161 | if err != nil { 162 | panic("Could not get port of listener.") 163 | } 164 | 165 | return int(port) 166 | } 167 | 168 | // StopTemporaryListener stops the dummy tcp listener. 169 | func StopTemporaryListener() { 170 | if tmpListener != nil { 171 | tmpListener.Close() 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /statsgod/logger.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Acquia, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Package statsgod - this library handles the logging during runtime. 18 | package statsgod 19 | 20 | import ( 21 | "io" 22 | "log" 23 | ) 24 | 25 | // Logger is a container for our log levels. 26 | type Logger struct { 27 | // Trace log level. 28 | Trace *log.Logger 29 | // Info log level. 30 | Info *log.Logger 31 | // Warning log level. 32 | Warning *log.Logger 33 | // Error log level. 34 | Error *log.Logger 35 | } 36 | 37 | // CreateLogger is a factory to instantiate a Logger struct. 38 | func CreateLogger( 39 | traceHandle io.Writer, 40 | infoHandle io.Writer, 41 | warningHandle io.Writer, 42 | errorHandle io.Writer) *Logger { 43 | 44 | var logger = new(Logger) 45 | 46 | logger.Trace = log.New(traceHandle, 47 | "TRACE: ", 48 | log.Ldate|log.Ltime|log.Lshortfile) 49 | 50 | logger.Info = log.New(infoHandle, 51 | "INFO: ", 52 | log.Ldate|log.Ltime|log.Lshortfile) 53 | 54 | logger.Warning = log.New(warningHandle, 55 | "WARNING: ", 56 | log.Ldate|log.Ltime|log.Lshortfile) 57 | 58 | logger.Error = log.New(errorHandle, 59 | "ERROR: ", 60 | log.Ldate|log.Ltime|log.Lshortfile) 61 | 62 | return logger 63 | } 64 | -------------------------------------------------------------------------------- /statsgod/logger_test.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Acquia, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package statsgod_test 18 | 19 | import ( 20 | . "github.com/acquia/statsgod/statsgod" 21 | . "github.com/onsi/ginkgo" 22 | . "github.com/onsi/gomega" 23 | "io/ioutil" 24 | ) 25 | 26 | var _ = Describe("Logger", func() { 27 | 28 | Describe("Creating a silent logger", func() { 29 | It("should be a legit Logger struct", func() { 30 | logger := *CreateLogger(ioutil.Discard, ioutil.Discard, ioutil.Discard, ioutil.Discard) 31 | Expect(logger.Trace).ShouldNot(Equal(nil)) 32 | Expect(logger.Info).ShouldNot(Equal(nil)) 33 | Expect(logger.Warning).ShouldNot(Equal(nil)) 34 | Expect(logger.Error).ShouldNot(Equal(nil)) 35 | }) 36 | }) 37 | 38 | }) 39 | -------------------------------------------------------------------------------- /statsgod/metric.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Acquia, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Package statsgod - This library is responsible for parsing and defining what 18 | // a "metric" is that we are going to be aggregating. 19 | package statsgod 20 | 21 | import ( 22 | "errors" 23 | "fmt" 24 | "sort" 25 | "strconv" 26 | "time" 27 | ) 28 | 29 | // Metric strings look like my.namespaced.value:123|g|@0.9 30 | const ( 31 | // SeparatorNamespaceValue is the character separating the namespace and value 32 | // in the metric string. 33 | SeparatorNamespaceValue = ":" 34 | // SeparatorValueType is the character separating the value and metric type in 35 | // the metric string. 36 | SeparatorValueType = "|" 37 | // SeparatorTypeSample is the character separating the type and an optional 38 | // sample rate. 39 | SeparatorTypeSample = "@" 40 | ) 41 | 42 | const ( 43 | // RuneColon is the rune value for a colon (:). 44 | RuneColon rune = 58 45 | // RunePipe is the rune value for a pipe (|). 46 | RunePipe rune = 124 47 | // RuneAt is the rune value for an "at" sign (@). 48 | RuneAt rune = 64 49 | // RuneSpace is the rune value for a space ( ). 50 | RuneSpace rune = 32 51 | // RuneNull is the rune value for an null byte. 52 | RuneNull rune = 0 53 | ) 54 | 55 | const ( 56 | // MaximumMetricLength is the number of runes allowed in a metric string. 57 | MaximumMetricLength = 512 58 | ) 59 | 60 | const ( 61 | // MetricTypeCounter describes a counter, sent as "c" 62 | MetricTypeCounter = 0 63 | // MetricTypeGauge describes a gauge, sent as "g" 64 | MetricTypeGauge = 1 65 | // MetricTypeSet describes a set, set as "s" 66 | MetricTypeSet = 2 67 | // MetricTypeTimer describes a timer, set as "ms" 68 | MetricTypeTimer = 3 69 | // MetricTypeUnknown describes a malformed metric type 70 | MetricTypeUnknown = 4 71 | ) 72 | 73 | // MetricQuantile tracks a specified quantile measurement. 74 | type MetricQuantile struct { 75 | Quantile int // The specified percentile. 76 | Boundary float64 // The calculated quantile value. 77 | AllValues ValueSlice // All of the values. 78 | Mean float64 // The mean value within the quantile. 79 | Median float64 // The median value within the quantile. 80 | Max float64 // The maxumum value within the quantile. 81 | Sum float64 // The sum value within the quantile. 82 | } 83 | 84 | // Metric is our main data type. 85 | type Metric struct { 86 | Key string // Name of the metric. 87 | MetricType int // What type of metric is it (gauge, counter, timer) 88 | TotalHits float64 // Number of times it has been used. 89 | LastValue float64 // The last value stored. 90 | ValuesPerSecond float64 // The number of values per second. 91 | MinValue float64 // The min value. 92 | MaxValue float64 // The max value. 93 | MeanValue float64 // The cumulative mean. 94 | MedianValue float64 // The cumulative median. 95 | Quantiles []MetricQuantile // A list of quantile calculations. 96 | AllValues ValueSlice // All of the values. 97 | FlushTime int // What time are we sending Graphite? 98 | LastFlushed int // When did we last flush this out? 99 | SampleRate float64 // The sample rate of the metric. 100 | } 101 | 102 | // CreateSimpleMetric is a helper to quickly create a metric with the minimum information. 103 | func CreateSimpleMetric(key string, value float64, metricType int) *Metric { 104 | metric := new(Metric) 105 | metric.Key = key 106 | metric.MetricType = metricType 107 | metric.LastValue = value 108 | metric.AllValues = append(metric.AllValues, metric.LastValue) 109 | metric.TotalHits = float64(1.0) 110 | 111 | return metric 112 | } 113 | 114 | // ParseMetricString parses a metric string, and if it is properly constructed, 115 | // create a Metric structure. Expects the format [namespace]:[value]|[type] 116 | func ParseMetricString(metricString string) (*Metric, error) { 117 | var metric = new(Metric) 118 | sampleRate := float64(1.0) 119 | 120 | delimeters := [4]rune{ 121 | RuneColon, // ":" 122 | RunePipe, // "|" 123 | RunePipe, // "|" 124 | RuneAt, // "@" 125 | } 126 | 127 | var metricStrings [5][MaximumMetricLength]rune 128 | delimeterCount := 0 129 | metricCount := 0 130 | var metricRuneCount [5]int 131 | for _, ch := range metricString { 132 | if ch == RuneSpace || ch == RuneNull { 133 | continue 134 | } 135 | if delimeterCount < len(delimeters) && 136 | metricCount < len(metricStrings)-1 && 137 | ch == delimeters[delimeterCount] { 138 | delimeterCount++ 139 | metricCount++ 140 | } else { 141 | metricStrings[metricCount][metricRuneCount[metricCount]] = ch 142 | metricRuneCount[metricCount]++ 143 | } 144 | if metricRuneCount[metricCount] == MaximumMetricLength { 145 | break 146 | } 147 | } 148 | 149 | if metricRuneCount[0] == 0 || 150 | metricRuneCount[1] == 0 || 151 | metricRuneCount[2] == 0 { 152 | return metric, fmt.Errorf("Invalid data string, missing elements: '%s'", metricString) 153 | } 154 | 155 | value, valueErr := strconv.ParseFloat(string(metricStrings[1][:metricRuneCount[1]]), 32) 156 | if valueErr != nil { 157 | return metric, fmt.Errorf("Invalid data string, bad value: '%s'", metricString) 158 | } 159 | 160 | metricType, err := getMetricType(string(metricStrings[2][:metricRuneCount[2]])) 161 | if err != nil { 162 | return metric, fmt.Errorf("Invalid data string, bad type: '%s'", metricString) 163 | } 164 | 165 | sample, rateErr := strconv.ParseFloat(string(metricStrings[4][:metricRuneCount[4]]), 32) 166 | if rateErr == nil && sample > float64(0) && sample <= float64(1) { 167 | sampleRate = sample 168 | } 169 | 170 | // If a sample rate was applied, we inflate the hit count to extrapolate 171 | // the actual rate. 172 | if sampleRate < float64(1.0) { 173 | metric.TotalHits = float64(1.0) / sampleRate 174 | } else { 175 | metric.TotalHits = float64(1.0) 176 | } 177 | 178 | // The string was successfully parsed. Convert to a Metric structure. 179 | metric.Key = string(metricStrings[0][:metricRuneCount[0]]) 180 | metric.MetricType = metricType 181 | metric.LastValue = value 182 | metric.AllValues = append(metric.AllValues, metric.LastValue) 183 | metric.SampleRate = sampleRate 184 | 185 | return metric, nil 186 | } 187 | 188 | // AggregateMetric adds the metric to the specified storage map or aggregates 189 | // it with an existing metric which has the same namespace. 190 | func AggregateMetric(metrics map[string]Metric, metric Metric) { 191 | existingMetric, metricExists := metrics[metric.Key] 192 | 193 | // If the metric exists in the specified map, we either take the last value or 194 | // sum the values and increment the hit count. 195 | if metricExists { 196 | // Occasionally metrics are sampled and may inflate their numbers. 197 | existingMetric.TotalHits += metric.TotalHits 198 | 199 | existingMetric.AllValues = append(existingMetric.AllValues, metric.LastValue) 200 | 201 | switch { 202 | case metric.MetricType == MetricTypeCounter: 203 | existingMetric.LastValue += metric.LastValue * metric.TotalHits 204 | case metric.MetricType == MetricTypeGauge: 205 | existingMetric.LastValue = metric.LastValue 206 | case metric.MetricType == MetricTypeSet: 207 | existingMetric.LastValue = metric.LastValue 208 | case metric.MetricType == MetricTypeTimer: 209 | existingMetric.LastValue = metric.LastValue 210 | } 211 | 212 | metrics[metric.Key] = existingMetric 213 | } else { 214 | metrics[metric.Key] = metric 215 | } 216 | } 217 | 218 | // ProcessMetric will create additional calculations based on the type of metric. 219 | func ProcessMetric(metric *Metric, flushDuration time.Duration, quantiles []int, logger Logger) { 220 | flushInterval := flushDuration / time.Second 221 | 222 | sort.Sort(metric.AllValues) 223 | switch metric.MetricType { 224 | case MetricTypeCounter: 225 | metric.ValuesPerSecond = metric.LastValue / float64(flushInterval) 226 | case MetricTypeGauge: 227 | metric.MedianValue = metric.AllValues.Median() 228 | metric.MeanValue = metric.AllValues.Mean() 229 | case MetricTypeSet: 230 | metric.LastValue = float64(metric.AllValues.UniqueCount()) 231 | case MetricTypeTimer: 232 | metric.MinValue, metric.MaxValue, _ = metric.AllValues.Minmax() 233 | metric.MedianValue = metric.AllValues.Median() 234 | metric.MeanValue = metric.AllValues.Mean() 235 | metric.ValuesPerSecond = metric.TotalHits / float64(flushInterval) 236 | 237 | metric.Quantiles = make([]MetricQuantile, 0) 238 | for _, q := range quantiles { 239 | percentile := float64(q) / float64(100) 240 | quantile := new(MetricQuantile) 241 | quantile.Quantile = q 242 | 243 | // Make calculations based on the desired quantile. 244 | quantile.Boundary = metric.AllValues.Quantile(percentile) 245 | for _, value := range metric.AllValues { 246 | if value > quantile.Boundary { 247 | break 248 | } 249 | quantile.AllValues = append(quantile.AllValues, value) 250 | } 251 | _, quantile.Max, _ = quantile.AllValues.Minmax() 252 | quantile.Mean = quantile.AllValues.Mean() 253 | quantile.Median = quantile.AllValues.Median() 254 | quantile.Sum = quantile.AllValues.Sum() 255 | metric.Quantiles = append(metric.Quantiles, *quantile) 256 | } 257 | } 258 | } 259 | 260 | // getMetricType converts a single-character metric format to a full term. 261 | func getMetricType(short string) (int, error) { 262 | switch { 263 | case "c" == short: 264 | return MetricTypeCounter, nil 265 | case "g" == short: 266 | return MetricTypeGauge, nil 267 | case "s" == short: 268 | return MetricTypeSet, nil 269 | case "ms" == short: 270 | return MetricTypeTimer, nil 271 | } 272 | return MetricTypeUnknown, errors.New("unknown metric type") 273 | } 274 | 275 | // ParseMetrics parses the strings received from clients and creates Metric structures. 276 | func ParseMetrics(parseChannel chan string, relayChannel chan *Metric, auth Auth, logger Logger, quit *bool) { 277 | 278 | var authOk bool 279 | var authErr error 280 | 281 | for { 282 | // Process the channel as soon as requests come in. If they are valid Metric 283 | // structures, we move them to a new channel to be flushed on an interval. 284 | select { 285 | case metricString := <-parseChannel: 286 | // Authenticate the metric. 287 | authOk, authErr = auth.Authenticate(&metricString) 288 | if authErr != nil || !authOk { 289 | logger.Error.Printf("Auth Error: %v, %s", authOk, authErr) 290 | break 291 | } 292 | 293 | metric, err := ParseMetricString(metricString) 294 | if err != nil { 295 | logger.Error.Printf("Invalid metric: %s, %s", metricString, err) 296 | break 297 | } 298 | // Push the metric onto the channel to be aggregated and flushed. 299 | relayChannel <- metric 300 | case <-time.After(time.Second): 301 | // Test for a quit signal. 302 | } 303 | if *quit { 304 | break 305 | } 306 | } 307 | } 308 | -------------------------------------------------------------------------------- /statsgod/metric_test.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Acquia, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package statsgod_test 18 | 19 | import ( 20 | "crypto/rand" 21 | "fmt" 22 | . "github.com/acquia/statsgod/statsgod" 23 | . "github.com/onsi/ginkgo" 24 | . "github.com/onsi/gomega" 25 | "io/ioutil" 26 | "math" 27 | "time" 28 | ) 29 | 30 | var metricBenchmarkTimeLimit = 0.25 31 | 32 | // acceptablePrecision is used to round values to an acceptable test precision. 33 | var acceptablePrecision = float64(10000) 34 | 35 | // Creates a Metric struct with default values. 36 | func getDefaultMetricStructure() Metric { 37 | var metric = new(Metric) 38 | 39 | return *metric 40 | } 41 | 42 | // Adjusts values to an acceptable precision for testing. 43 | func testPrecision(value float64) float64 { 44 | return math.Floor(value*acceptablePrecision) / acceptablePrecision 45 | } 46 | 47 | // Generates a random alphanumeric string. 48 | func randomString(strSize int) string { 49 | 50 | dictionary := ".0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" 51 | 52 | var bytes = make([]byte, strSize) 53 | rand.Read(bytes) 54 | for k, v := range bytes { 55 | bytes[k] = dictionary[v%byte(len(dictionary))] 56 | } 57 | return string(bytes) 58 | } 59 | 60 | var _ = Describe("Metrics", func() { 61 | 62 | Describe("Testing the basic structure", func() { 63 | metric := getDefaultMetricStructure() 64 | It("should contain values", func() { 65 | // Ensure that the expected values exist. 66 | Expect(metric.Key).ShouldNot(Equal(nil)) 67 | Expect(metric.MetricType).ShouldNot(Equal(nil)) 68 | Expect(metric.TotalHits).ShouldNot(Equal(nil)) 69 | Expect(metric.LastValue).ShouldNot(Equal(nil)) 70 | Expect(metric.MinValue).ShouldNot(Equal(nil)) 71 | Expect(metric.MaxValue).ShouldNot(Equal(nil)) 72 | Expect(metric.MeanValue).ShouldNot(Equal(nil)) 73 | Expect(metric.MedianValue).ShouldNot(Equal(nil)) 74 | Expect(metric.FlushTime).ShouldNot(Equal(nil)) 75 | Expect(metric.LastFlushed).ShouldNot(Equal(nil)) 76 | Expect(metric.SampleRate).ShouldNot(Equal(nil)) 77 | 78 | // Slices when empty evaluate to nil, check for len instead. 79 | Expect(len(metric.Quantiles)).Should(Equal(0)) 80 | Expect(len(metric.AllValues)).Should(Equal(0)) 81 | }) 82 | }) 83 | 84 | Describe("Testing the functionality", func() { 85 | Context("when we parse metrics", func() { 86 | // Test that we can correctly parse a metric string. 87 | It("should contain correct values", func() { 88 | metricOne, err := ParseMetricString("test.three:3|g|@0.75") 89 | Expect(err).Should(BeNil()) 90 | Expect(metricOne).ShouldNot(Equal(nil)) 91 | Expect(metricOne.Key).Should(Equal("test.three")) 92 | Expect(metricOne.LastValue).Should(Equal(float64(3))) 93 | Expect(metricOne.MetricType).Should(Equal(MetricTypeGauge)) 94 | Expect(metricOne.SampleRate).Should(Equal(float64(0.75))) 95 | }) 96 | 97 | It("should handle spaces and null bytes correctly", func() { 98 | spaceMetrics := []string{ 99 | " test.four:4|g|@0.75 ", 100 | " test.four:4|g|@0.75 ", 101 | " test.four:4 | g | @0.75 ", 102 | " test.four:4 |g |@0.75 ", 103 | " test.four:4| g | @0.75 ", 104 | " test.four:4 |g| @0.75 ", 105 | " test.four:4| g |@0.75 ", 106 | " test.four:4| g |@0.75\x00", 107 | " test.four:4| g |@0.75 \x00 ", 108 | "\x00test.four:4| g |@0.75 \x00", 109 | } 110 | var metricOne *Metric 111 | var err error 112 | for _, ms := range spaceMetrics { 113 | metricOne, err = ParseMetricString(ms) 114 | Expect(err).Should(BeNil()) 115 | Expect(metricOne).ShouldNot(Equal(nil)) 116 | Expect(metricOne.Key).Should(Equal("test.four")) 117 | Expect(metricOne.LastValue).Should(Equal(float64(4))) 118 | Expect(metricOne.MetricType).Should(Equal(MetricTypeGauge)) 119 | Expect(metricOne.SampleRate).Should(Equal(float64(0.75))) 120 | } 121 | }) 122 | 123 | It("should invalidate a namespace that is too long", func() { 124 | longName := randomString(MaximumMetricLength) 125 | metricString := fmt.Sprintf("statsgod.test.%s:3|g|@0.75", longName) 126 | _, err := ParseMetricString(metricString) 127 | Expect(err).ShouldNot(BeNil()) 128 | }) 129 | 130 | It("should contain reasonable defaults", func() { 131 | metricOne, _ := ParseMetricString("test.three:3|g") 132 | Expect(metricOne).ShouldNot(Equal(nil)) 133 | Expect(metricOne.TotalHits).Should(Equal(float64(1.0))) 134 | Expect(metricOne.SampleRate).Should(Equal(float64(1.0))) 135 | }) 136 | 137 | // Test that incorrect strings trigger errors. 138 | It("should throw errors on malformed strings", func() { 139 | _, errOne := ParseMetricString("test3|g") 140 | Expect(errOne).ShouldNot(Equal(nil)) 141 | _, errTwo := ParseMetricString("test:3g") 142 | Expect(errTwo).ShouldNot(Equal(nil)) 143 | _, errThree := ParseMetricString("test:3|gauge") 144 | Expect(errThree).ShouldNot(Equal(nil)) 145 | _, errFour := ParseMetricString("test:three|g") 146 | Expect(errFour).ShouldNot(Equal(nil)) 147 | }) 148 | 149 | Measure("it should be able to parse metric strings quickly.", func(b Benchmarker) { 150 | runtime := b.Time("runtime", func() { 151 | metric, _ := ParseMetricString("test:1|g|@0.25") 152 | metric, _ = ParseMetricString("test:2|c|@0.5") 153 | metric, _ = ParseMetricString("test:3|ms|@0.75") 154 | Expect(metric).ShouldNot(Equal(nil)) 155 | }) 156 | 157 | Expect(runtime.Seconds()).Should(BeNumerically("<", metricBenchmarkTimeLimit), "it should be able to parse metric strings quickly.") 158 | 159 | }, 100000) 160 | 161 | }) 162 | 163 | Context("when we create simple metrics", func() { 164 | It("should create a metric with the correct values", func() { 165 | key := "test" 166 | value := float64(123) 167 | metricType := MetricTypeGauge 168 | metric := CreateSimpleMetric(key, value, metricType) 169 | Expect(metric.Key).Should(Equal(key)) 170 | Expect(metric.MetricType).Should(Equal(metricType)) 171 | Expect(metric.LastValue).Should(Equal(value)) 172 | Expect(metric.AllValues[0]).Should(Equal(value)) 173 | Expect(metric.TotalHits).Should(Equal(float64(1.0))) 174 | }) 175 | }) 176 | 177 | Context("when we aggregate metrics", func() { 178 | metrics := make(map[string]Metric) 179 | testMetrics := []string{ 180 | "test.one:1|c", 181 | "test.one:2|c|@0.25", 182 | "test.one:3|c", 183 | "test.one:1|c", 184 | "test.one:1|c|@0.5", 185 | "test.one:1|c|@0.95", 186 | "test.one:5|c", 187 | "test.one:1|c|@0.75", 188 | "test.two:3|c", 189 | "test.three:45|g", 190 | "test.three:35|g", 191 | "test.three:33|g", 192 | "test.four:100|ms|@0.25", 193 | "test.four:150|ms|@0.5", 194 | "test.four:250|ms", 195 | "test.four:50|ms|@0.95", 196 | "test.four:150|ms|@0.95", 197 | "test.four:120|ms|@0.75", 198 | "test.four:130|ms|@0.95", 199 | } 200 | 201 | for _, metricString := range testMetrics { 202 | m, _ := ParseMetricString(metricString) 203 | AggregateMetric(metrics, *m) 204 | } 205 | 206 | It("should combine counters and keep last for metrics and timers", func() { 207 | // Test that we now have two metrics stored. 208 | Expect(len(metrics)).Should(Equal(4)) 209 | 210 | // Test that the equal namespaces sum values and increment hits. 211 | existingMetric, metricExists := metrics["test.one"] 212 | Expect(metricExists).Should(Equal(true)) 213 | Expect(len(existingMetric.AllValues)).Should(Equal(8)) 214 | Expect(testPrecision(existingMetric.TotalHits)).Should(Equal(float64(12.3859))) 215 | Expect(testPrecision(existingMetric.LastValue)).Should(Equal(float64(22.3859))) 216 | 217 | existingMetric, metricExists = metrics["test.two"] 218 | Expect(metricExists).Should(Equal(true)) 219 | Expect(len(existingMetric.AllValues)).Should(Equal(1)) 220 | Expect(existingMetric.TotalHits).Should(Equal(float64(1.0))) 221 | Expect(existingMetric.LastValue).Should(Equal(float64(3.0))) 222 | 223 | existingMetric, metricExists = metrics["test.three"] 224 | Expect(metricExists).Should(Equal(true)) 225 | Expect(len(existingMetric.AllValues)).Should(Equal(3)) 226 | Expect(existingMetric.TotalHits).Should(Equal(float64(3.0))) 227 | Expect(existingMetric.LastValue).Should(Equal(float64(33.0))) 228 | 229 | existingMetric, metricExists = metrics["test.four"] 230 | Expect(metricExists).Should(Equal(true)) 231 | Expect(len(existingMetric.AllValues)).Should(Equal(7)) 232 | Expect(testPrecision(existingMetric.TotalHits)).Should(Equal(float64(11.4912))) 233 | Expect(existingMetric.LastValue).Should(Equal(float64(130.0))) 234 | }) 235 | 236 | Measure("it should aggregate metrics quickly.", func(b Benchmarker) { 237 | metrics := make(map[string]Metric) 238 | metric, _ := ParseMetricString("test:1|c") 239 | runtime := b.Time("runtime", func() { 240 | AggregateMetric(metrics, *metric) 241 | }) 242 | 243 | Expect(runtime.Seconds()).Should(BeNumerically("<", metricBenchmarkTimeLimit), "it should aggregate metrics quickly.") 244 | }, 100000) 245 | }) 246 | 247 | Context("when we process metrics", func() { 248 | logger := *CreateLogger(ioutil.Discard, ioutil.Discard, ioutil.Discard, ioutil.Discard) 249 | // This is a full test of the goroutine that takes input strings 250 | // from the socket and converts them to Metrics. 251 | It("should parse from the channel and populate the relay channel", func() { 252 | config, _ := CreateConfig("") 253 | config.Service.Auth = AuthTypeConfigToken 254 | config.Service.Tokens["test-token"] = true 255 | 256 | parseChannel := make(chan string, 2) 257 | relayChannel := make(chan *Metric, 2) 258 | auth := CreateAuth(config) 259 | quit := false 260 | go ParseMetrics(parseChannel, relayChannel, auth, logger, &quit) 261 | parseChannel <- "test-token.test.one:123|c" 262 | parseChannel <- "test-token.test.two:234|g" 263 | parseChannel <- "bad-metric" 264 | parseChannel <- "test-token.bad-metric|g" 265 | parseChannel <- "bad-token.test.two:234|g" 266 | for len(parseChannel) > 0 { 267 | // Wait for the channel to be emptied. 268 | time.Sleep(time.Microsecond) 269 | } 270 | quit = true 271 | Expect(len(parseChannel)).Should(Equal(0)) 272 | Expect(len(relayChannel)).Should(Equal(2)) 273 | metricOne := <-relayChannel 274 | metricTwo := <-relayChannel 275 | Expect(metricOne.LastValue).Should(Equal(float64(123))) 276 | Expect(metricTwo.LastValue).Should(Equal(float64(234))) 277 | }) 278 | 279 | metrics := make(map[string]Metric) 280 | 281 | // Test that counters aggregate and process properly. 282 | for i := 0; i < 11; i++ { 283 | metricCount, _ := ParseMetricString("test.count:1|c") 284 | AggregateMetric(metrics, *metricCount) 285 | } 286 | It("should calculate the total and rate properly", func() { 287 | metricCount := metrics["test.count"] 288 | ProcessMetric(&metricCount, time.Second*10, []int{80}, logger) 289 | Expect(metricCount.ValuesPerSecond).Should(Equal(1.1)) 290 | Expect(metricCount.LastValue).Should(Equal(float64(11))) 291 | }) 292 | 293 | It("should magnify sample rates properly", func() { 294 | metricSample, _ := ParseMetricString("test.count:1|c|@0.1") 295 | Expect(testPrecision(metricSample.TotalHits)).Should(Equal(float64(9.9999))) 296 | metricSample, _ = ParseMetricString("test.count:1|c|@0.25") 297 | Expect(metricSample.TotalHits).Should(Equal(float64(4.0))) 298 | metricSample, _ = ParseMetricString("test.count:3|c|@0.5") 299 | Expect(metricSample.TotalHits).Should(Equal(float64(2.0))) 300 | metricSample, _ = ParseMetricString("test.count:4|c|@0.75") 301 | Expect(testPrecision(metricSample.TotalHits)).Should(Equal(float64(1.3333))) 302 | metricSample, _ = ParseMetricString("test.count:3|c|@0.9") 303 | Expect(testPrecision(metricSample.TotalHits)).Should(Equal(float64(1.1111))) 304 | metricSample, _ = ParseMetricString("test.count:3|c|@0.95") 305 | Expect(testPrecision(metricSample.TotalHits)).Should(Equal(float64(1.0526))) 306 | }) 307 | 308 | // Test that gauges average properly. 309 | for i := 1; i < 11; i++ { 310 | gauge := float64(i) * float64(i) 311 | metricGauge, _ := ParseMetricString(fmt.Sprintf("test.gauge:%f|g", gauge)) 312 | AggregateMetric(metrics, *metricGauge) 313 | } 314 | It("should average gauges properly", func() { 315 | metricGauge := metrics["test.gauge"] 316 | ProcessMetric(&metricGauge, time.Second*10, []int{80}, logger) 317 | Expect(metricGauge.MedianValue).Should(Equal(30.5)) 318 | Expect(metricGauge.MeanValue).Should(Equal(38.5)) 319 | Expect(metricGauge.LastValue).Should(Equal(100.0)) 320 | }) 321 | 322 | // Test all of the timer calculations. 323 | for i := 3; i < 14; i++ { 324 | timer := float64(i) * float64(i) 325 | metricTimer, _ := ParseMetricString(fmt.Sprintf("test.timer:%f|ms|@0.9", timer)) 326 | AggregateMetric(metrics, *metricTimer) 327 | } 328 | It("should calculate timer values properly", func() { 329 | metricTimer := metrics["test.timer"] 330 | ProcessMetric(&metricTimer, time.Second*10, []int{50, 75, 90}, logger) 331 | 332 | Expect(len(metricTimer.Quantiles)).Should(Equal(3)) 333 | 334 | Expect(metricTimer.MinValue).Should(Equal(float64(9))) 335 | Expect(metricTimer.MaxValue).Should(Equal(float64(169))) 336 | Expect(metricTimer.MeanValue).Should(Equal(float64(74))) 337 | Expect(metricTimer.MedianValue).Should(Equal(float64(64))) 338 | Expect(metricTimer.LastValue).Should(Equal(float64(169))) 339 | Expect(testPrecision(metricTimer.TotalHits)).Should(Equal(float64(12.2222))) 340 | Expect(testPrecision(metricTimer.ValuesPerSecond)).Should(Equal(float64(1.2222))) 341 | // Quantiles 342 | q := metricTimer.Quantiles[0] 343 | Expect(q.Mean).Should(Equal(float64(33.166666666666664))) 344 | Expect(q.Median).Should(Equal(float64(30.5))) 345 | Expect(q.Max).Should(Equal(float64(64))) 346 | Expect(q.Sum).Should(Equal(float64(199))) 347 | 348 | q = metricTimer.Quantiles[1] 349 | GinkgoWriter.Write([]byte(fmt.Sprintf("%v", q))) 350 | Expect(q.Mean).Should(Equal(float64(47.5))) 351 | Expect(q.Median).Should(Equal(float64(42.5))) 352 | Expect(q.Max).Should(Equal(float64(100))) 353 | Expect(q.Sum).Should(Equal(float64(380))) 354 | 355 | q = metricTimer.Quantiles[2] 356 | Expect(q.Mean).Should(Equal(float64(64.5))) 357 | Expect(q.Median).Should(Equal(float64(56.5))) 358 | Expect(q.Max).Should(Equal(float64(144))) 359 | Expect(q.Sum).Should(Equal(float64(645))) 360 | }) 361 | 362 | // Test the set-metric-type unique values. 363 | for i := 1; i < 21; i++ { 364 | set := math.Floor(float64(i) / float64(2)) 365 | metricSet, _ := ParseMetricString(fmt.Sprintf("test.set:%f|s", set)) 366 | AggregateMetric(metrics, *metricSet) 367 | } 368 | It("should calculate set unique values properly", func() { 369 | metricSet := metrics["test.set"] 370 | ProcessMetric(&metricSet, time.Second*10, []int{90}, logger) 371 | Expect(metricSet.LastValue).Should(Equal(float64(11))) 372 | }) 373 | 374 | Measure("it should process metrics quickly.", func(b Benchmarker) { 375 | metric := metrics["test.gauge"] 376 | runtime := b.Time("runtime", func() { 377 | ProcessMetric(&metric, time.Second*10, []int{80}, logger) 378 | }) 379 | 380 | Expect(runtime.Seconds()).Should(BeNumerically("<", metricBenchmarkTimeLimit), "it should process metrics quickly.") 381 | }, 100000) 382 | 383 | }) 384 | }) 385 | 386 | }) 387 | -------------------------------------------------------------------------------- /statsgod/nilconn.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Acquia, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package statsgod 18 | 19 | import ( 20 | "net" 21 | "time" 22 | ) 23 | 24 | // NilConnPanicMsg is the message that the NilConn implementation sends in panic. 25 | const ( 26 | NilConnPanicMsg = "Use of a nil connection." 27 | ) 28 | 29 | // NilConn is an implementation of the net.Conn interface. We use it so that 30 | // we can return a value from GetConnection. Without the NilConn, it would 31 | // attempt to convert nil into a net.Conn which causes an error. In this case 32 | // we can handle it gracefully and panic for any access instead of attemting 33 | // to access a nil pointer. 34 | type NilConn struct{} 35 | 36 | // Read implements net.Conn.Read() 37 | func (c NilConn) Read(b []byte) (n int, err error) { 38 | panic(NilConnPanicMsg) 39 | } 40 | 41 | // Write implements net.Conn.Write() 42 | func (c NilConn) Write(b []byte) (n int, err error) { 43 | panic(NilConnPanicMsg) 44 | } 45 | 46 | // Close implements net.Conn.Close() 47 | func (c NilConn) Close() error { 48 | panic(NilConnPanicMsg) 49 | } 50 | 51 | // LocalAddr implements net.Conn.LocalAddr() 52 | func (c NilConn) LocalAddr() net.Addr { 53 | panic(NilConnPanicMsg) 54 | } 55 | 56 | // RemoteAddr implements net.Conn.RemoteAddr() 57 | func (c NilConn) RemoteAddr() net.Addr { 58 | panic(NilConnPanicMsg) 59 | } 60 | 61 | // SetDeadline implements net.Conn.SetDeadline() 62 | func (c NilConn) SetDeadline(t time.Time) error { 63 | panic(NilConnPanicMsg) 64 | } 65 | 66 | // SetReadDeadline implements net.Conn.SetReadDeadline() 67 | func (c NilConn) SetReadDeadline(t time.Time) error { 68 | panic(NilConnPanicMsg) 69 | } 70 | 71 | // SetWriteDeadline implements net.Conn.SetWriteDeadline() 72 | func (c NilConn) SetWriteDeadline(t time.Time) error { 73 | panic(NilConnPanicMsg) 74 | } 75 | -------------------------------------------------------------------------------- /statsgod/nillconn_test.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Acquia, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package statsgod_test 18 | 19 | import ( 20 | . "github.com/acquia/statsgod/statsgod" 21 | . "github.com/onsi/ginkgo" 22 | . "github.com/onsi/gomega" 23 | "time" 24 | ) 25 | 26 | var _ = Describe("NilConn", func() { 27 | 28 | Describe("Testing the basic structure", func() { 29 | Context("when creating a NilConn", func() { 30 | It("should be a complete structure", func() { 31 | // Test the NilConn implementation. 32 | nilConn := NilConn{} 33 | nilBuf := make([]byte, 0) 34 | nilTime := time.Now() 35 | Expect(func() { nilConn.Read(nilBuf) }).Should(Panic()) 36 | Expect(func() { nilConn.Write(nilBuf) }).Should(Panic()) 37 | Expect(func() { nilConn.Close() }).Should(Panic()) 38 | Expect(func() { nilConn.LocalAddr() }).Should(Panic()) 39 | Expect(func() { nilConn.RemoteAddr() }).Should(Panic()) 40 | Expect(func() { nilConn.SetDeadline(nilTime) }).Should(Panic()) 41 | Expect(func() { nilConn.SetReadDeadline(nilTime) }).Should(Panic()) 42 | Expect(func() { nilConn.SetWriteDeadline(nilTime) }).Should(Panic()) 43 | }) 44 | }) 45 | }) 46 | }) 47 | -------------------------------------------------------------------------------- /statsgod/relay.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Acquia, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Package statsgod - This library handles the relaying of data to a backend 18 | // storage. This backend could be a webservice like carbon or a filesystem 19 | // or a mock for testing. All backend implementations should conform to the 20 | // MetricRelay interface. 21 | package statsgod 22 | 23 | import ( 24 | "bytes" 25 | "fmt" 26 | "runtime" 27 | "strconv" 28 | "time" 29 | ) 30 | 31 | const ( 32 | // RelayTypeCarbon is an enum describing a carbon backend relay. 33 | RelayTypeCarbon = "carbon" 34 | // RelayTypeMock is an enum describing a mock backend relay. 35 | RelayTypeMock = "mock" 36 | // NamespaceTypeCounter is an enum for counter namespacing. 37 | NamespaceTypeCounter = iota 38 | // NamespaceTypeGauge is an enum for gauge namespacing. 39 | NamespaceTypeGauge 40 | // NamespaceTypeRate is an enum for rate namespacing. 41 | NamespaceTypeRate 42 | // NamespaceTypeSet is an enum for rate namespacing. 43 | NamespaceTypeSet 44 | // NamespaceTypeTimer is an enum for timer namespacing. 45 | NamespaceTypeTimer 46 | // Megabyte represents the number of bytes in a megabyte. 47 | Megabyte = 1048576 48 | ) 49 | 50 | // MetricRelay defines the interface for a back end implementation. 51 | type MetricRelay interface { 52 | Relay(metric Metric, logger Logger) bool 53 | } 54 | 55 | // CreateRelay is a factory for instantiating remote relays. 56 | func CreateRelay(config ConfigValues, logger Logger) MetricRelay { 57 | switch config.Relay.Type { 58 | case RelayTypeCarbon: 59 | // Create a relay to carbon. 60 | relay := new(CarbonRelay) 61 | relay.FlushInterval = config.Relay.Flush 62 | relay.Percentile = config.Stats.Percentile 63 | relay.SetPrefixesAndSuffixes(config) 64 | // Create a connection pool for the relay to use. 65 | pool, err := CreateConnectionPool(config.Relay.Concurrency, fmt.Sprintf("%s:%d", config.Carbon.Host, config.Carbon.Port), ConnPoolTypeTcp, config.Relay.Timeout, logger) 66 | if err != nil { 67 | panic(fmt.Sprintf("Fatal error, could not create a connection pool to %s:%d", config.Carbon.Host, config.Carbon.Port)) 68 | } 69 | relay.ConnectionPool = pool 70 | logger.Info.Println("Relaying metrics to carbon backend") 71 | return relay 72 | case RelayTypeMock: 73 | fallthrough 74 | default: 75 | relay := new(MockRelay) 76 | relay.FlushInterval = config.Relay.Flush 77 | relay.Percentile = config.Stats.Percentile 78 | logger.Info.Println("Relaying metrics to mock backend") 79 | return relay 80 | } 81 | } 82 | 83 | // CarbonRelay implements MetricRelay. 84 | type CarbonRelay struct { 85 | FlushInterval time.Duration 86 | Percentile []int 87 | ConnectionPool *ConnectionPool 88 | Prefixes map[int]string 89 | Suffixes map[int]string 90 | } 91 | 92 | // SetPrefixesAndSuffixes is a helper to set the prefixes and suffixes from 93 | // config when relaying data. 94 | func (c *CarbonRelay) SetPrefixesAndSuffixes(config ConfigValues) { 95 | prefix := "" 96 | suffix := "" 97 | c.Prefixes = map[int]string{} 98 | c.Suffixes = map[int]string{} 99 | 100 | configPrefixes := map[int]string{ 101 | NamespaceTypeCounter: config.Namespace.Prefixes.Counters, 102 | NamespaceTypeGauge: config.Namespace.Prefixes.Gauges, 103 | NamespaceTypeRate: config.Namespace.Prefixes.Rates, 104 | NamespaceTypeSet: config.Namespace.Prefixes.Sets, 105 | NamespaceTypeTimer: config.Namespace.Prefixes.Timers, 106 | } 107 | 108 | configSuffixes := map[int]string{ 109 | NamespaceTypeCounter: config.Namespace.Suffixes.Counters, 110 | NamespaceTypeGauge: config.Namespace.Suffixes.Gauges, 111 | NamespaceTypeRate: config.Namespace.Suffixes.Rates, 112 | NamespaceTypeSet: config.Namespace.Suffixes.Sets, 113 | NamespaceTypeTimer: config.Namespace.Suffixes.Timers, 114 | } 115 | 116 | // Global prefix. 117 | if config.Namespace.Prefix != "" { 118 | prefix = config.Namespace.Prefix + "." 119 | } 120 | 121 | // Type prefixes. 122 | for metricType, typePrefix := range configPrefixes { 123 | c.Prefixes[metricType] = prefix 124 | if typePrefix != "" { 125 | c.Prefixes[metricType] = prefix + typePrefix + "." 126 | } 127 | } 128 | 129 | // Global suffix. 130 | if config.Namespace.Suffix != "" { 131 | suffix = "." + config.Namespace.Suffix 132 | } 133 | 134 | // Type suffixes 135 | for metricType, typeSuffix := range configSuffixes { 136 | c.Suffixes[metricType] = suffix 137 | if typeSuffix != "" { 138 | c.Suffixes[metricType] = "." + typeSuffix + suffix 139 | } 140 | } 141 | } 142 | 143 | // ApplyPrefixAndSuffix interpolates the configured prefix and suffix with the 144 | // metric namespace string. 145 | func (c CarbonRelay) ApplyPrefixAndSuffix(namespace string, metricType int) string { 146 | var metricNs bytes.Buffer 147 | 148 | metricNs.WriteString(c.Prefixes[metricType]) 149 | metricNs.WriteString(namespace) 150 | metricNs.WriteString(c.Suffixes[metricType]) 151 | return metricNs.String() 152 | } 153 | 154 | // Relay implements MetricRelay::Relay(). 155 | func (c CarbonRelay) Relay(metric Metric, logger Logger) bool { 156 | ProcessMetric(&metric, c.FlushInterval, c.Percentile, logger) 157 | // @todo: are we ever setting flush time? 158 | stringTime := strconv.Itoa(metric.FlushTime) 159 | var key string 160 | var qkey string 161 | 162 | switch metric.MetricType { 163 | case MetricTypeGauge: 164 | key = c.ApplyPrefixAndSuffix(metric.Key, NamespaceTypeGauge) 165 | sendCarbonMetric(key, metric.LastValue, stringTime, true, c, logger) 166 | case MetricTypeCounter: 167 | key = c.ApplyPrefixAndSuffix(metric.Key, NamespaceTypeRate) 168 | sendCarbonMetric(key, metric.ValuesPerSecond, stringTime, true, c, logger) 169 | 170 | key = c.ApplyPrefixAndSuffix(metric.Key, NamespaceTypeCounter) 171 | sendCarbonMetric(key, metric.LastValue, stringTime, true, c, logger) 172 | case MetricTypeSet: 173 | key = c.ApplyPrefixAndSuffix(metric.Key, NamespaceTypeSet) 174 | sendCarbonMetric(key, metric.LastValue, stringTime, true, c, logger) 175 | case MetricTypeTimer: 176 | key = c.ApplyPrefixAndSuffix(metric.Key, NamespaceTypeRate) 177 | sendCarbonMetric(key, metric.ValuesPerSecond, stringTime, true, c, logger) 178 | 179 | // Cumulative values. 180 | key = c.ApplyPrefixAndSuffix(metric.Key+".mean_value", NamespaceTypeTimer) 181 | sendCarbonMetric(key, metric.MeanValue, stringTime, true, c, logger) 182 | 183 | key = c.ApplyPrefixAndSuffix(metric.Key+".median_value", NamespaceTypeTimer) 184 | sendCarbonMetric(key, metric.MedianValue, stringTime, true, c, logger) 185 | 186 | key = c.ApplyPrefixAndSuffix(metric.Key+".max_value", NamespaceTypeTimer) 187 | sendCarbonMetric(key, metric.MaxValue, stringTime, true, c, logger) 188 | 189 | key = c.ApplyPrefixAndSuffix(metric.Key+".min_value", NamespaceTypeTimer) 190 | sendCarbonMetric(key, metric.MinValue, stringTime, true, c, logger) 191 | 192 | // Quantile values. 193 | for _, q := range metric.Quantiles { 194 | qkey = strconv.FormatInt(int64(q.Quantile), 10) 195 | 196 | key = c.ApplyPrefixAndSuffix(metric.Key+".mean_"+qkey, NamespaceTypeTimer) 197 | sendCarbonMetric(key, q.Mean, stringTime, true, c, logger) 198 | 199 | key = c.ApplyPrefixAndSuffix(metric.Key+".median_"+qkey, NamespaceTypeTimer) 200 | sendCarbonMetric(key, q.Median, stringTime, true, c, logger) 201 | 202 | key = c.ApplyPrefixAndSuffix(metric.Key+".upper_"+qkey, NamespaceTypeTimer) 203 | sendCarbonMetric(key, q.Max, stringTime, true, c, logger) 204 | 205 | key = c.ApplyPrefixAndSuffix(metric.Key+".sum_"+qkey, NamespaceTypeTimer) 206 | sendCarbonMetric(key, q.Sum, stringTime, true, c, logger) 207 | } 208 | } 209 | return true 210 | } 211 | 212 | // sendCarbonMetric formats a message and a value and time and sends to Graphite. 213 | func sendCarbonMetric(key string, v float64, t string, retry bool, relay CarbonRelay, logger Logger) bool { 214 | var releaseErr error 215 | dataSent := false 216 | 217 | // Send to the remote host. 218 | conn, connErr := relay.ConnectionPool.GetConnection(logger) 219 | 220 | if connErr != nil { 221 | logger.Error.Println("Could not connect to remote host.", connErr) 222 | // If there was an error connecting, recreate the connection and retry. 223 | _, releaseErr = relay.ConnectionPool.ReleaseConnection(conn, true, logger) 224 | } else { 225 | var payload bytes.Buffer 226 | sv := strconv.FormatFloat(float64(v), 'f', 6, 32) 227 | 228 | payload.WriteString(key) 229 | payload.WriteString(" ") 230 | payload.WriteString(sv) 231 | payload.WriteString(" ") 232 | payload.WriteString(t) 233 | payload.WriteString("\n") 234 | 235 | // Write to the connection. 236 | _, writeErr := fmt.Fprint(conn, payload.String()) 237 | if writeErr != nil { 238 | // If there was an error writing, recreate the connection and retry. 239 | _, releaseErr = relay.ConnectionPool.ReleaseConnection(conn, true, logger) 240 | } else { 241 | // If the metric was written, just release that connection back. 242 | _, releaseErr = relay.ConnectionPool.ReleaseConnection(conn, false, logger) 243 | dataSent = true 244 | } 245 | } 246 | 247 | // For some reason we were unable to release this connection. 248 | if releaseErr != nil { 249 | logger.Error.Println("Relay connection not released.", releaseErr) 250 | } 251 | 252 | // If data was not sent, likely a socket timeout, we'll retry one time. 253 | if !dataSent && retry { 254 | dataSent = sendCarbonMetric(key, v, t, false, relay, logger) 255 | } 256 | 257 | return dataSent 258 | } 259 | 260 | // MockRelay implements MetricRelay. 261 | type MockRelay struct { 262 | FlushInterval time.Duration 263 | Percentile []int 264 | } 265 | 266 | // Relay implements MetricRelay::Relay(). 267 | func (c MockRelay) Relay(metric Metric, logger Logger) bool { 268 | ProcessMetric(&metric, c.FlushInterval, c.Percentile, logger) 269 | logger.Trace.Printf("Mock flush: %v", metric) 270 | return true 271 | } 272 | 273 | // RelayMetrics relays the metrics in-memory to the permanent storage facility. 274 | // At this point we are receiving Metric structures from a channel that need to 275 | // be aggregated by the specified namespace. We do this immediately, then when 276 | // the specified flush interval passes, we send aggregated metrics to storage. 277 | func RelayMetrics(relay MetricRelay, relayChannel chan *Metric, logger Logger, config *ConfigValues, quit *bool) { 278 | // Use a tick channel to determine if a flush message has arrived. 279 | tick := time.Tick(config.Relay.Flush) 280 | logger.Info.Printf("Flushing every %v", config.Relay.Flush) 281 | 282 | // Track the flush cycle metrics. 283 | var flushStart time.Time 284 | var flushStop time.Time 285 | var flushCount int 286 | 287 | // Internal storage. 288 | metrics := make(map[string]Metric) 289 | 290 | for { 291 | if *quit { 292 | break 293 | } 294 | 295 | // Process the metrics as soon as they arrive on the channel. If nothing has 296 | // been added during the flush interval duration, continue the loop to allow 297 | // it to flush the data. 298 | select { 299 | case metric := <-relayChannel: 300 | AggregateMetric(metrics, *metric) 301 | if config.Debug.Receipt { 302 | logger.Info.Printf("Metric: %v", metrics[metric.Key]) 303 | } 304 | case <-time.After(config.Relay.Flush): 305 | // Nothing to read, attempt to flush. 306 | } 307 | 308 | // After reading from the metrics channel, we check the ticks channel. If there 309 | // is a tick, flush the in-memory metrics. 310 | select { 311 | case <-tick: 312 | // Prepare the runtime metrics. 313 | PrepareRuntimeMetrics(metrics, config) 314 | 315 | // Time and flush the received metrics. 316 | flushCount = len(metrics) 317 | flushStart = time.Now() 318 | RelayAllMetrics(relay, metrics, logger) 319 | flushStop = time.Now() 320 | 321 | // Prepare and flush the internal metrics. 322 | PrepareFlushMetrics(metrics, config, flushStart, flushStop, flushCount) 323 | RelayAllMetrics(relay, metrics, logger) 324 | default: 325 | // Flush interval hasn't passed yet. 326 | } 327 | } 328 | } 329 | 330 | // RelayAllMetrics is a helper to iterate over a Metric map and flush all to the relay. 331 | func RelayAllMetrics(relay MetricRelay, metrics map[string]Metric, logger Logger) { 332 | for key, metric := range metrics { 333 | metric.FlushTime = int(time.Now().Unix()) 334 | relay.Relay(metric, logger) 335 | delete(metrics, key) 336 | } 337 | } 338 | 339 | // PrepareRuntimeMetrics creates key runtime metrics to monitor the health of the system. 340 | func PrepareRuntimeMetrics(metrics map[string]Metric, config *ConfigValues) { 341 | if config.Debug.Relay { 342 | memStats := &runtime.MemStats{} 343 | runtime.ReadMemStats(memStats) 344 | 345 | var nsBuffer bytes.Buffer 346 | nsBuffer.WriteString("statsgod.") 347 | nsBuffer.WriteString(config.Service.Hostname) 348 | nsBuffer.WriteString(".runtime.memory") 349 | 350 | // Prepare a metric for the memory allocated. 351 | var heapBuffer bytes.Buffer 352 | heapBuffer.WriteString(nsBuffer.String()) 353 | heapBuffer.WriteString(".heapalloc") 354 | heapAllocValue := float64(memStats.HeapAlloc) / float64(Megabyte) 355 | heapAllocMetric := CreateSimpleMetric(heapBuffer.String(), heapAllocValue, MetricTypeGauge) 356 | metrics[heapAllocMetric.Key] = *heapAllocMetric 357 | 358 | // Prepare a metric for the memory allocated that is still in use. 359 | var allocBuffer bytes.Buffer 360 | allocBuffer.WriteString(nsBuffer.String()) 361 | allocBuffer.WriteString(".alloc") 362 | allocValue := float64(memStats.Alloc) / float64(Megabyte) 363 | allocMetric := CreateSimpleMetric(allocBuffer.String(), allocValue, MetricTypeGauge) 364 | metrics[allocMetric.Key] = *allocMetric 365 | 366 | // Prepare a metric for the memory obtained from the system. 367 | var sysBuffer bytes.Buffer 368 | sysBuffer.WriteString(nsBuffer.String()) 369 | sysBuffer.WriteString(".sys") 370 | sysValue := float64(memStats.Sys) / float64(Megabyte) 371 | sysMetric := CreateSimpleMetric(sysBuffer.String(), sysValue, MetricTypeGauge) 372 | metrics[sysMetric.Key] = *sysMetric 373 | } 374 | } 375 | 376 | // PrepareFlushMetrics creates metrics that represent the speed and size of the flushes. 377 | func PrepareFlushMetrics(metrics map[string]Metric, config *ConfigValues, flushStart time.Time, flushStop time.Time, flushCount int) { 378 | if config.Debug.Relay { 379 | 380 | var nsBuffer bytes.Buffer 381 | nsBuffer.WriteString("statsgod.") 382 | nsBuffer.WriteString(config.Service.Hostname) 383 | nsBuffer.WriteString(".flush") 384 | 385 | // Prepare the duration metric. 386 | var durationBuffer bytes.Buffer 387 | durationBuffer.WriteString(nsBuffer.String()) 388 | durationBuffer.WriteString(".duration") 389 | durationValue := float64(int64(flushStop.Sub(flushStart).Nanoseconds()) / int64(time.Millisecond)) 390 | durationMetric := CreateSimpleMetric(durationBuffer.String(), durationValue, MetricTypeTimer) 391 | metrics[durationMetric.Key] = *durationMetric 392 | 393 | // Prepare the counter metric. 394 | var countBuffer bytes.Buffer 395 | countBuffer.WriteString(nsBuffer.String()) 396 | countBuffer.WriteString(".count") 397 | countMetric := CreateSimpleMetric(countBuffer.String(), float64(flushCount), MetricTypeGauge) 398 | metrics[countMetric.Key] = *countMetric 399 | } 400 | } 401 | -------------------------------------------------------------------------------- /statsgod/relay_test.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Acquia, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package statsgod_test 18 | 19 | import ( 20 | . "github.com/acquia/statsgod/statsgod" 21 | . "github.com/onsi/ginkgo" 22 | . "github.com/onsi/gomega" 23 | "io/ioutil" 24 | "reflect" 25 | "time" 26 | ) 27 | 28 | var _ = Describe("Relay", func() { 29 | var ( 30 | tmpPort int 31 | logger Logger 32 | config ConfigValues 33 | ) 34 | 35 | Describe("Testing the basic structure", func() { 36 | BeforeEach(func() { 37 | logger = *CreateLogger(ioutil.Discard, ioutil.Discard, ioutil.Discard, ioutil.Discard) 38 | config, _ = CreateConfig("") 39 | config.Relay.Type = RelayTypeMock 40 | tmpPort = StartTemporaryListener() 41 | // In case we switch to the carbon relay, we need to use 42 | // the temporary port number. 43 | config.Carbon.Port = tmpPort 44 | }) 45 | 46 | AfterEach(func() { 47 | StopTemporaryListener() 48 | }) 49 | 50 | Context("when using the factory function", func() { 51 | It("should be a complete structure", func() { 52 | // Tests that we can get a mock relay. 53 | mockRelay := CreateRelay(config, logger) 54 | Expect(mockRelay).ShouldNot(Equal(nil)) 55 | Expect(reflect.TypeOf(mockRelay).String()).Should(Equal("*statsgod.MockRelay")) 56 | 57 | // Tests that we can get a carbon relay. 58 | config.Relay.Type = RelayTypeCarbon 59 | carbonRelay := CreateRelay(config, logger) 60 | Expect(carbonRelay).ShouldNot(Equal(nil)) 61 | Expect(reflect.TypeOf(carbonRelay).String()).Should(Equal("*statsgod.CarbonRelay")) 62 | 63 | // Tests that we panic if it cannot create a connection pool. 64 | config.Carbon.Host = "127.0.0.1" 65 | config.Carbon.Port = 0 66 | Expect(func() { CreateRelay(config, logger) }).Should(Panic()) 67 | 68 | // Tests that we can get a mock relay as the default value 69 | config.Relay.Type = "foo" 70 | fooRelay := CreateRelay(config, logger) 71 | Expect(fooRelay).ShouldNot(Equal(nil)) 72 | Expect(reflect.TypeOf(fooRelay).String()).Should(Equal("*statsgod.MockRelay")) 73 | }) 74 | }) 75 | 76 | Context("when creating a CarbonRelay", func() { 77 | It("should be a complete structure", func() { 78 | config.Relay.Type = RelayTypeCarbon 79 | backendRelay := CreateRelay(config, logger).(*CarbonRelay) 80 | backendRelay.Percentile = []int{50, 75, 90} 81 | backendRelay.SetPrefixesAndSuffixes(config) 82 | Expect(backendRelay.Prefixes[NamespaceTypeCounter]).Should(Equal("stats.counts.")) 83 | Expect(backendRelay.Prefixes[NamespaceTypeGauge]).Should(Equal("stats.gauges.")) 84 | Expect(backendRelay.Prefixes[NamespaceTypeRate]).Should(Equal("stats.rates.")) 85 | Expect(backendRelay.Prefixes[NamespaceTypeSet]).Should(Equal("stats.sets.")) 86 | Expect(backendRelay.Prefixes[NamespaceTypeTimer]).Should(Equal("stats.timers.")) 87 | Expect(backendRelay.Suffixes[NamespaceTypeCounter]).Should(Equal("")) 88 | Expect(backendRelay.Suffixes[NamespaceTypeGauge]).Should(Equal("")) 89 | Expect(backendRelay.Suffixes[NamespaceTypeRate]).Should(Equal("")) 90 | Expect(backendRelay.Suffixes[NamespaceTypeSet]).Should(Equal("")) 91 | Expect(backendRelay.Suffixes[NamespaceTypeTimer]).Should(Equal("")) 92 | Expect(backendRelay.FlushInterval).ShouldNot(Equal(nil)) 93 | Expect(backendRelay.Percentile).ShouldNot(Equal(nil)) 94 | // At this point the connection pool has not been established. 95 | Expect(backendRelay.ConnectionPool).ShouldNot(Equal(nil)) 96 | 97 | // Test prefixes and suffixes. 98 | config.Namespace.Prefix = "p" 99 | config.Namespace.Prefixes.Counters = "c" 100 | config.Namespace.Prefixes.Gauges = "g" 101 | config.Namespace.Prefixes.Rates = "r" 102 | config.Namespace.Prefixes.Sets = "s" 103 | config.Namespace.Prefixes.Timers = "t" 104 | config.Namespace.Suffix = "s" 105 | config.Namespace.Suffixes.Counters = "c" 106 | config.Namespace.Suffixes.Gauges = "g" 107 | config.Namespace.Suffixes.Rates = "r" 108 | config.Namespace.Suffixes.Sets = "s" 109 | config.Namespace.Suffixes.Timers = "t" 110 | backendRelay.SetPrefixesAndSuffixes(config) 111 | Expect(backendRelay.ApplyPrefixAndSuffix("m", NamespaceTypeCounter)).Should(Equal("p.c.m.c.s")) 112 | Expect(backendRelay.ApplyPrefixAndSuffix("m", NamespaceTypeGauge)).Should(Equal("p.g.m.g.s")) 113 | Expect(backendRelay.ApplyPrefixAndSuffix("m", NamespaceTypeRate)).Should(Equal("p.r.m.r.s")) 114 | Expect(backendRelay.ApplyPrefixAndSuffix("m", NamespaceTypeSet)).Should(Equal("p.s.m.s.s")) 115 | Expect(backendRelay.ApplyPrefixAndSuffix("m", NamespaceTypeTimer)).Should(Equal("p.t.m.t.s")) 116 | 117 | // Test a broken relay. 118 | metric, _ := ParseMetricString("metric.one") 119 | StopTemporaryListener() 120 | 121 | Expect(func() { backendRelay.Relay(*metric, logger) }).Should(Panic()) 122 | 123 | // Restart the broken relay to shutdown properly. 124 | tmpPort = StartTemporaryListener() 125 | }) 126 | }) 127 | 128 | Context("when relaying metrics", func() { 129 | // This is a full test of the goroutine that listens for parsed metrics 130 | // and then aggregates and relays to the designated backend. 131 | It("should aggregate and relay properly", func() { 132 | config, _ = CreateConfig("") 133 | config.Relay.Type = RelayTypeMock 134 | config.Relay.Flush = time.Microsecond 135 | config.Debug.Receipt = true 136 | relay := CreateRelay(config, logger) 137 | relayChannel := make(chan *Metric, 2) 138 | quit := false 139 | 140 | go RelayMetrics(relay, relayChannel, logger, &config, &quit) 141 | metricOne := CreateSimpleMetric("test.one", float64(123), MetricTypeGauge) 142 | metricTwo := CreateSimpleMetric("test.two", float64(234), MetricTypeGauge) 143 | relayChannel <- metricOne 144 | relayChannel <- metricTwo 145 | for len(relayChannel) > 0 { 146 | time.Sleep(10 * time.Microsecond) 147 | } 148 | quit = true 149 | Expect(len(relayChannel)).Should(Equal(0)) 150 | }) 151 | 152 | It("should delete metrics from the store after relaying", func() { 153 | config.Relay.Type = RelayTypeCarbon 154 | config.Carbon.Host = "127.0.0.1" 155 | config.Carbon.Port = tmpPort 156 | backendRelay := CreateRelay(config, logger) 157 | 158 | testMetrics := []string{ 159 | "test.one:3|c", 160 | "test.two:3|g", 161 | "test.three:3|ms", 162 | "test.four:3|s", 163 | } 164 | 165 | var metric *Metric 166 | metrics := make(map[string]Metric) 167 | for _, testMetric := range testMetrics { 168 | metric, _ = ParseMetricString(testMetric) 169 | metrics[metric.Key] = *metric 170 | } 171 | Expect(len(metrics)).Should(Equal(4)) 172 | RelayAllMetrics(backendRelay, metrics, logger) 173 | Expect(len(metrics)).Should(Equal(0)) 174 | }) 175 | 176 | It("should prepare internal stats", func() { 177 | flushStart := time.Now() 178 | flushStop := flushStart.Add(time.Second) 179 | flushCount := 10 180 | config.Service.Hostname = "test" 181 | config.Debug.Relay = true 182 | metrics := make(map[string]Metric) 183 | PrepareRuntimeMetrics(metrics, &config) 184 | PrepareFlushMetrics(metrics, &config, flushStart, flushStop, flushCount) 185 | // Runtime and flush information should be populated. 186 | Expect(metrics["statsgod.test.runtime.memory.heapalloc"].LastValue).ShouldNot(Equal(float64(0.0))) 187 | Expect(metrics["statsgod.test.runtime.memory.alloc"].LastValue).ShouldNot(Equal(float64(0.0))) 188 | Expect(metrics["statsgod.test.runtime.memory.sys"].LastValue).ShouldNot(Equal(float64(0.0))) 189 | Expect(metrics["statsgod.test.flush.duration"].LastValue).ShouldNot(Equal(float64(0.0))) 190 | Expect(metrics["statsgod.test.flush.count"].LastValue).ShouldNot(Equal(float64(0.0))) 191 | }) 192 | }) 193 | 194 | Context("when creating a MockRelay", func() { 195 | It("should be a complete structure", func() { 196 | backendRelay := CreateRelay(config, logger) 197 | metricOne, _ := ParseMetricString("test.one:3|c") 198 | backendRelay.Relay(*metricOne, logger) 199 | }) 200 | }) 201 | }) 202 | }) 203 | -------------------------------------------------------------------------------- /statsgod/signals.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Acquia, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Package statsgod - This library is responsible for parsing and defining what 18 | // a "metric" is that we are going to be aggregating. 19 | package statsgod 20 | 21 | import ( 22 | "os" 23 | "os/signal" 24 | "syscall" 25 | ) 26 | 27 | // ListenForSignals will listen for quit/reload signals and respond accordingly. 28 | func ListenForSignals(finishChannel chan int, config *ConfigValues, configFile *string, logger Logger) { 29 | // For signal handling we catch several signals. ABRT, INT, TERM and QUIT 30 | // are all used to clean up and stop the process. HUP is used to signal 31 | // a configuration reload without stopping the process. 32 | signalChannel := make(chan os.Signal, 1) 33 | signal.Notify(signalChannel, 34 | syscall.SIGABRT, 35 | syscall.SIGHUP, 36 | syscall.SIGINT, 37 | syscall.SIGTERM, 38 | syscall.SIGQUIT) 39 | go func() { 40 | for { 41 | s := <-signalChannel 42 | logger.Info.Printf("Processed signal %v", s) 43 | 44 | switch s { 45 | case syscall.SIGHUP: 46 | // Reload the configuration. 47 | logger.Info.Printf("Loading config changes from %s", *configFile) 48 | config.LoadFile(*configFile) 49 | default: 50 | // The other signals will kill the process. 51 | finishChannel <- 1 52 | break 53 | } 54 | } 55 | }() 56 | 57 | } 58 | -------------------------------------------------------------------------------- /statsgod/signals_test.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Acquia, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package statsgod_test 18 | 19 | import ( 20 | . "github.com/acquia/statsgod/statsgod" 21 | . "github.com/onsi/ginkgo" 22 | . "github.com/onsi/gomega" 23 | "io/ioutil" 24 | "syscall" 25 | "time" 26 | ) 27 | 28 | func catchSignal(finishChannel chan int) bool { 29 | select { 30 | case <-finishChannel: 31 | return true 32 | case <-time.After(time.Second): 33 | return false 34 | } 35 | } 36 | 37 | var _ = Describe("Signals", func() { 38 | 39 | Describe("Testing the signal handling", func() { 40 | configFile := "" 41 | config, _ := CreateConfig(configFile) 42 | finishChannel := make(chan int) 43 | logger := *CreateLogger(ioutil.Discard, ioutil.Discard, ioutil.Discard, ioutil.Discard) 44 | ListenForSignals(finishChannel, &config, &configFile, logger) 45 | 46 | config.Service.Name = "testing" 47 | syscall.Kill(syscall.Getpid(), syscall.SIGHUP) 48 | 49 | It("should stop the parser when a quit signal is sent", func() { 50 | ListenForSignals(finishChannel, &config, &configFile, logger) 51 | parseChannel := make(chan string, 2) 52 | relayChannel := make(chan *Metric, 2) 53 | auth := CreateAuth(config) 54 | quit := false 55 | go ParseMetrics(parseChannel, relayChannel, auth, logger, &quit) 56 | //syscall.Kill(syscall.Getpid(), syscall.SIGTERM) // @TODO can we test SIGTERM? 57 | quit = true 58 | }) 59 | 60 | It("should catch the 'reload' signal", func() { 61 | Expect(config.Service.Name).Should(Equal("statsgod")) 62 | }) 63 | 64 | It("should catch the 'quit' signals", func() { 65 | signals := []syscall.Signal{ 66 | syscall.SIGABRT, 67 | //syscall.SIGINT, // @TODO SIGINT kills the test. 68 | //syscall.SIGTERM, // @TODO SIGTERM kills the test. 69 | syscall.SIGQUIT, 70 | } 71 | for _, signal := range signals { 72 | syscall.Kill(syscall.Getpid(), signal) 73 | Expect(catchSignal(finishChannel)).Should(BeTrue()) 74 | ListenForSignals(finishChannel, &config, &configFile, logger) 75 | } 76 | }) 77 | 78 | }) 79 | }) 80 | -------------------------------------------------------------------------------- /statsgod/socket.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Acquia, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Package statsgod - this library manages the different socket listeners 18 | // that we use to collect metrics. 19 | package statsgod 20 | 21 | import ( 22 | "fmt" 23 | "net" 24 | "os" 25 | "strings" 26 | "syscall" 27 | "time" 28 | ) 29 | 30 | // Enumeration of the socket types. 31 | const ( 32 | SocketTypeUdp = iota 33 | SocketTypeTcp 34 | SocketTypeUnix 35 | ) 36 | 37 | const ( 38 | // MinimumLengthMessage is the shortest message (or fragment) we can parse. 39 | MinimumLengthMessage = 1 40 | ) 41 | 42 | // Socket is the interface for all of our socket types. 43 | type Socket interface { 44 | Listen(parseChannel chan string, logger Logger, config *ConfigValues) 45 | Close(logger Logger) 46 | GetAddr() string 47 | SocketIsActive() bool 48 | } 49 | 50 | // CreateSocket is a factory to create Socket structs. 51 | func CreateSocket(socketType int, addr string) Socket { 52 | switch socketType { 53 | case SocketTypeUdp: 54 | l := new(SocketUdp) 55 | l.Addr = addr 56 | return l 57 | case SocketTypeTcp: 58 | l := new(SocketTcp) 59 | l.Addr = addr 60 | return l 61 | case SocketTypeUnix: 62 | l := new(SocketUnix) 63 | l.Addr = addr 64 | return l 65 | default: 66 | panic("Unknown socket type requested.") 67 | } 68 | } 69 | 70 | // BlockForSocket blocks until the specified socket is active. 71 | func BlockForSocket(socket Socket, timeout time.Duration) { 72 | start := time.Now() 73 | for { 74 | if socket.SocketIsActive() == true { 75 | return 76 | } 77 | time.Sleep(time.Microsecond) 78 | if time.Since(start) > timeout { 79 | return 80 | } 81 | } 82 | } 83 | 84 | // SocketTcp contains the required fields to start a TCP socket. 85 | type SocketTcp struct { 86 | Addr string 87 | Listener net.Listener 88 | } 89 | 90 | // Listen listens on a socket and populates a channel with received messages. 91 | // Conforms to Socket.Listen(). 92 | func (l *SocketTcp) Listen(parseChannel chan string, logger Logger, config *ConfigValues) { 93 | if l.Addr == "" { 94 | panic("Could not establish a TCP socket. Address must be specified.") 95 | } 96 | listener, err := net.Listen("tcp", l.Addr) 97 | if err != nil { 98 | panic(fmt.Sprintf("Could not establish a TCP socket. %s", err)) 99 | } 100 | l.Listener = listener 101 | 102 | logger.Info.Printf("TCP socket opened on %s", l.Addr) 103 | for { 104 | conn, err := listener.Accept() 105 | if err != nil { 106 | logger.Error.Println("Could not accept connection", err) 107 | return 108 | } 109 | go readInput(conn, parseChannel, logger) 110 | } 111 | } 112 | 113 | // Close closes an open socket. Conforms to Socket.Close(). 114 | func (l *SocketTcp) Close(logger Logger) { 115 | logger.Info.Println("Closing TCP socket.") 116 | l.Listener.Close() 117 | } 118 | 119 | // SocketIsActive determines if the socket is listening. Conforms to Socket.SocketIsActive() 120 | func (l *SocketTcp) SocketIsActive() bool { 121 | return l.Listener != nil 122 | } 123 | 124 | // GetAddr retrieves a net compatible address string. Conforms to Socket.GetAddr(). 125 | func (l *SocketTcp) GetAddr() string { 126 | return l.Listener.Addr().String() 127 | } 128 | 129 | // SocketUdp contains the fields required to start a UDP socket. 130 | type SocketUdp struct { 131 | Addr string 132 | Listener *net.UDPConn 133 | } 134 | 135 | // Listen listens on a socket and populates a channel with received messages. 136 | // Conforms to Socket.Listen(). 137 | func (l *SocketUdp) Listen(parseChannel chan string, logger Logger, config *ConfigValues) { 138 | if l.Addr == "" { 139 | panic("Could not establish a UDP socket. Addr must be specified.") 140 | } 141 | addr, _ := net.ResolveUDPAddr("udp4", l.Addr) 142 | listener, err := net.ListenUDP("udp", addr) 143 | if err != nil { 144 | panic(fmt.Sprintf("Could not establish a UDP socket. %s", err)) 145 | } 146 | l.Listener = listener 147 | 148 | logger.Info.Printf("UDP socket opened on %s", l.Addr) 149 | for { 150 | readInputUdp(*listener, parseChannel, logger, config) 151 | } 152 | } 153 | 154 | // Close closes an open socket. Conforms to Socket.Close(). 155 | func (l *SocketUdp) Close(logger Logger) { 156 | logger.Info.Println("Closing UDP socket.") 157 | l.Listener.Close() 158 | } 159 | 160 | // SocketIsActive determines if the socket is listening. Conforms to Socket.SocketIsActive() 161 | func (l *SocketUdp) SocketIsActive() bool { 162 | return l.Listener != nil 163 | } 164 | 165 | // GetAddr retrieves a net compatible address string. Conforms to Socket.GetAddr(). 166 | func (l *SocketUdp) GetAddr() string { 167 | return l.Listener.LocalAddr().String() 168 | } 169 | 170 | // SocketUnix contains the fields required to start a Unix socket. 171 | type SocketUnix struct { 172 | Addr string 173 | Listener net.Listener 174 | } 175 | 176 | // Listen listens on a socket and populates a channel with received messages. 177 | // Conforms to Socket.Listen(). 178 | func (l *SocketUnix) Listen(parseChannel chan string, logger Logger, config *ConfigValues) { 179 | if l.Addr == "" { 180 | panic("Could not establish a Unix socket. No sock file specified.") 181 | } 182 | oldMask := syscall.Umask(0011) 183 | listener, err := net.Listen("unix", l.Addr) 184 | _ = syscall.Umask(oldMask) 185 | if err != nil { 186 | panic(fmt.Sprintf("Could not establish a Unix socket. %s", err)) 187 | } 188 | l.Listener = listener 189 | logger.Info.Printf("Unix socket opened at %s", l.Addr) 190 | for { 191 | conn, err := l.Listener.Accept() 192 | if err != nil { 193 | logger.Error.Println("Could not accept connection", err) 194 | return 195 | } 196 | go readInput(conn, parseChannel, logger) 197 | } 198 | } 199 | 200 | // Close closes an open socket. Conforms to Socket.Close(). 201 | func (l *SocketUnix) Close(logger Logger) { 202 | defer os.Remove(l.Addr) 203 | logger.Info.Println("Closing Unix socket.") 204 | l.Listener.Close() 205 | } 206 | 207 | // SocketIsActive determines if the socket is listening. Conforms to Socket.SocketIsActive() 208 | func (l *SocketUnix) SocketIsActive() bool { 209 | return l.Listener != nil 210 | } 211 | 212 | // GetAddr retrieves a net compatible address string. Conforms to Socket.GetAddr(). 213 | func (l *SocketUnix) GetAddr() string { 214 | return l.Addr 215 | } 216 | 217 | // readInput parses the buffer for TCP and Unix sockets. 218 | func readInput(conn net.Conn, parseChannel chan string, logger Logger) { 219 | defer conn.Close() 220 | // readLength is the length of our buffer. 221 | readLength := 512 222 | // metricCount is how many metrics to parse per read. 223 | metricCount := 0 224 | // overflow tracks any messages that span two buffers. 225 | overflow := "" 226 | // buf is a reusable buffer for reading from the connection. 227 | buf := make([]byte, readLength) 228 | 229 | // Read data from the connection until it is closed. 230 | for { 231 | length, err := conn.Read(buf) 232 | if err != nil { 233 | // EOF will present as an error, but it could just signal a hangup. 234 | break 235 | } 236 | conn.Write([]byte("")) 237 | if length > 0 { 238 | // Check for multiple metrics delimited by a newline character. 239 | metrics := strings.Split(overflow+string(buf[:length]), "\n") 240 | // If the buffer is full, the last metric likely has not fully been sent 241 | // yet. Try to parse it and if it throws an error, we'll prepend it to the 242 | // next connection read presuming that it will span to the next data read. 243 | metricCount = len(metrics) 244 | overflow = "" 245 | if length == readLength { 246 | // Attempt to parse the last metric. If that fails parse, we'll 247 | // reduce the size by one and save the overflow for the next read. 248 | _, err = ParseMetricString(metrics[len(metrics)-1]) 249 | if err != nil { 250 | overflow = metrics[len(metrics)-1] 251 | metricCount = len(metrics) - 1 252 | } 253 | } 254 | 255 | // Send the metrics to be parsed. 256 | for i := 0; i < metricCount; i++ { 257 | if len(metrics[i]) > MinimumLengthMessage { 258 | parseChannel <- metrics[i] 259 | } 260 | } 261 | } 262 | 263 | // Zero out the buffer for the next read. 264 | for b := range buf { 265 | buf[b] = 0 266 | } 267 | } 268 | } 269 | 270 | // readInputUdp parses the buffer for UDP sockets. 271 | func readInputUdp(conn net.UDPConn, parseChannel chan string, logger Logger, config *ConfigValues) { 272 | buf := make([]byte, config.Connection.Udp.Maxpacket) 273 | length, _, err := conn.ReadFromUDP(buf[0:]) 274 | if err == nil && length > MinimumLengthMessage { 275 | metrics := strings.Split(string(buf[:length]), "\n") 276 | for _, metric := range metrics { 277 | if len(metric) > MinimumLengthMessage { 278 | parseChannel <- metric 279 | } 280 | } 281 | } 282 | } 283 | -------------------------------------------------------------------------------- /statsgod/socket_test.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Acquia, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package statsgod_test 18 | 19 | import ( 20 | "fmt" 21 | . "github.com/acquia/statsgod/statsgod" 22 | . "github.com/onsi/ginkgo" 23 | . "github.com/onsi/gomega" 24 | "io/ioutil" 25 | "math/rand" 26 | "net" 27 | "strings" 28 | "time" 29 | ) 30 | 31 | var socketBenchmarkTimeLimit = 0.75 32 | 33 | var shortMetricString = generateMetricString(2, "short") 34 | var mediumMetricString = generateMetricString(10, "medium") 35 | var longMetricString = generateMetricString(100, "long") 36 | 37 | // Table for running the socket tests. 38 | var testSockets = []struct { 39 | socketType int 40 | socketDesc string 41 | socketAddr string 42 | badAddr string 43 | socketMessages []string 44 | }{ 45 | {SocketTypeTcp, "tcp", "127.0.0.1:0", "0.0.0.0", []string{"test.tcp:4|c", shortMetricString, mediumMetricString, longMetricString}}, 46 | {SocketTypeUdp, "udp", "127.0.0.1:0", "", []string{"test.udp:4|c", shortMetricString, mediumMetricString}}, 47 | {SocketTypeUnix, "unix", "/tmp/statsgod.sock", "/dev/null", []string{"test.unix:4|c", shortMetricString, mediumMetricString, longMetricString}}, 48 | } 49 | 50 | var sockets = make([]Socket, 3) 51 | var parseChannel = make(chan string) 52 | var logger = *CreateLogger(ioutil.Discard, ioutil.Discard, ioutil.Discard, ioutil.Discard) 53 | var config, _ = CreateConfig("") 54 | 55 | var _ = Describe("Sockets", func() { 56 | 57 | Describe("Testing the Socket interface", func() { 58 | It("should contain the required functions", func() { 59 | for i, _ := range testSockets { 60 | _, ok := sockets[i].(interface { 61 | Listen(parseChannel chan string, logger Logger, config *ConfigValues) 62 | Close(logger Logger) 63 | GetAddr() string 64 | SocketIsActive() bool 65 | }) 66 | Expect(ok).Should(Equal(true)) 67 | } 68 | 69 | }) 70 | 71 | It("should panic for unknown socket types", func() { 72 | defer GinkgoRecover() 73 | Expect(func() { CreateSocket(99, "null") }).Should(Panic()) 74 | }) 75 | }) 76 | 77 | Describe("Testing the Socket functionality", func() { 78 | It("should be able to receive messages", func() { 79 | for i, ts := range testSockets { 80 | for _, sm := range ts.socketMessages { 81 | sendSocketMessage(ts.socketDesc, sockets[i], sm) 82 | 83 | receivedMessages := strings.Split(sm, "\n") 84 | for _, rm := range receivedMessages { 85 | message := "" 86 | select { 87 | case message = <-parseChannel: 88 | case <-time.After(5 * time.Second): 89 | message = "" 90 | } 91 | Expect(strings.TrimSpace(message)).Should(Equal(strings.TrimSpace(rm))) 92 | } 93 | } 94 | } 95 | }) 96 | 97 | It("should ignore empty values", func() { 98 | message := "" 99 | for i, ts := range testSockets { 100 | sendSocketMessage(ts.socketDesc, sockets[i], "\x00") 101 | sendSocketMessage(ts.socketDesc, sockets[i], "\n") 102 | sendSocketMessage(ts.socketDesc, sockets[i], "") 103 | select { 104 | case message = <-parseChannel: 105 | case <-time.After(100 * time.Microsecond): 106 | message = "" 107 | } 108 | Expect(message).Should(Equal("")) 109 | } 110 | }) 111 | 112 | It("should panic if it has a bad address", func() { 113 | for _, ts := range testSockets { 114 | socket := CreateSocket(ts.socketType, "") 115 | Expect(func() { socket.Listen(parseChannel, logger, &config) }).Should(Panic()) 116 | socket = CreateSocket(ts.socketType, ts.badAddr) 117 | Expect(func() { socket.Listen(parseChannel, logger, &config) }).Should(Panic()) 118 | } 119 | }) 120 | 121 | It("should block while waiting for a socket to become active", func() { 122 | socket := CreateSocket(SocketTypeTcp, "127.0.0.1:0") 123 | start := time.Now() 124 | BlockForSocket(socket, time.Second) 125 | Expect(time.Now()).Should(BeTemporally(">=", start, time.Second)) 126 | }) 127 | 128 | It("should panic if it is already listening on an address.", func() { 129 | for i, ts := range testSockets { 130 | socket := CreateSocket(ts.socketType, sockets[i].GetAddr()) 131 | Expect(func() { socket.Listen(parseChannel, logger, &config) }).Should(Panic()) 132 | } 133 | }) 134 | 135 | Measure("it should receive metrics quickly.", func(b Benchmarker) { 136 | runtime := b.Time("runtime", func() { 137 | for i, ts := range testSockets { 138 | sendSocketMessage(ts.socketDesc, sockets[i], ts.socketMessages[0]) 139 | <-parseChannel 140 | } 141 | }) 142 | 143 | Expect(runtime.Seconds()).Should(BeNumerically("<", socketBenchmarkTimeLimit), "it should receive metrics quickly.") 144 | }, 1000) 145 | 146 | }) 147 | }) 148 | 149 | // sendSocketMessage will connect to a specified socket type send a message and close. 150 | func sendSocketMessage(socketType string, socket Socket, message string) { 151 | conn, err := net.Dial(socketType, socket.GetAddr()) 152 | if err != nil { 153 | panic(fmt.Sprintf("Dial failed: %v", err)) 154 | } 155 | defer conn.Close() 156 | _, err = conn.Write([]byte(message)) 157 | } 158 | 159 | // generateMetricString generates a string that can be used to send multiple metrics. 160 | func generateMetricString(count int, prefix string) (metricString string) { 161 | rand.Seed(time.Now().UnixNano()) 162 | 163 | for i := 0; i < count; i++ { 164 | metricString += fmt.Sprintf("test.%s:%d|c\n", prefix, rand.Intn(100)) 165 | } 166 | 167 | return strings.Trim(metricString, "\n") 168 | } 169 | -------------------------------------------------------------------------------- /statsgod/statistics.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Acquia, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Package statsgod - This library handles the statistics calculations. 18 | package statsgod 19 | 20 | import ( 21 | "errors" 22 | "math" 23 | "sort" 24 | ) 25 | 26 | // ValueSlice provides a storage for float64 values. 27 | type ValueSlice []float64 28 | 29 | // Get is a getter for internal values. 30 | func (values ValueSlice) Get(i int) float64 { return values[i] } 31 | 32 | // Len gets the length of the internal values. 33 | func (values ValueSlice) Len() int { return len(values) } 34 | 35 | // Less is used for the sorting interface. 36 | func (values ValueSlice) Less(i, j int) bool { return values[i] < values[j] } 37 | 38 | // Swap is used for the sorting interface. 39 | func (values ValueSlice) Swap(i, j int) { values[i], values[j] = values[j], values[i] } 40 | 41 | // UniqueCount provides the number of unique values sent during this period. 42 | func (values ValueSlice) UniqueCount() int { 43 | sort.Sort(values) 44 | count := 0 45 | var p float64 46 | for i, v := range values { 47 | if i > 0 && p != v { 48 | count++ 49 | } else if i == 0 { 50 | count++ 51 | } 52 | p = v 53 | } 54 | 55 | return count 56 | } 57 | 58 | // Minmax provides the minimum and maximum values for a ValueSlice structure. 59 | func (values ValueSlice) Minmax() (min float64, max float64, err error) { 60 | length := values.Len() 61 | min = values.Get(0) 62 | max = values.Get(0) 63 | 64 | for i := 0; i < length; i++ { 65 | xi := values.Get(i) 66 | 67 | if math.IsNaN(xi) { 68 | err = errors.New("NaN value") 69 | } else if xi < min { 70 | min = xi 71 | } else if xi > max { 72 | max = xi 73 | } 74 | } 75 | 76 | return 77 | } 78 | 79 | // Median finds the median value from a ValueSlice. 80 | func (values ValueSlice) Median() (median float64) { 81 | length := values.Len() 82 | leftBoundary := (length - 1) / 2 83 | rightBoundary := length / 2 84 | 85 | if length == 0 { 86 | return 0.0 87 | } 88 | 89 | if leftBoundary == rightBoundary { 90 | median = values.Get(leftBoundary) 91 | } else { 92 | median = (values.Get(leftBoundary) + values.Get(rightBoundary)) / 2.0 93 | } 94 | 95 | return 96 | } 97 | 98 | // Mean finds the mean value from a ValueSlice. 99 | func (values ValueSlice) Mean() (mean float64) { 100 | length := values.Len() 101 | for i := 0; i < length; i++ { 102 | mean += (values.Get(i) - mean) / float64(i+1) 103 | } 104 | return 105 | } 106 | 107 | // Quantile finds a specified quantile value from a ValueSlice. 108 | func (values ValueSlice) Quantile(quantile float64) float64 { 109 | length := values.Len() 110 | index := quantile * float64(length-1) 111 | boundary := int(index) 112 | delta := index - float64(boundary) 113 | 114 | if length == 0 { 115 | return 0.0 116 | } else if boundary == length-1 { 117 | return values.Get(boundary) 118 | } else { 119 | return (1-delta)*values.Get(boundary) + delta*values.Get(boundary+1) 120 | } 121 | } 122 | 123 | // Sum finds the total of all values. 124 | func (values ValueSlice) Sum() float64 { 125 | sum := float64(0) 126 | length := values.Len() 127 | for i := 0; i < length; i++ { 128 | sum += values.Get(i) 129 | } 130 | 131 | return sum 132 | } 133 | -------------------------------------------------------------------------------- /statsgod/statistics_test.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Acquia, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package statsgod_test 18 | 19 | import ( 20 | . "github.com/acquia/statsgod/statsgod" 21 | . "github.com/onsi/ginkgo" 22 | . "github.com/onsi/gomega" 23 | "math" 24 | "sort" 25 | ) 26 | 27 | var statsBenchmarkTimeLimit = 0.25 28 | 29 | var _ = Describe("Statistics", func() { 30 | 31 | Describe("Testing the ValueSlice struct", func() { 32 | values := ValueSlice{1, 5, 2, 4, 3} 33 | 34 | It("should contain values", func() { 35 | // Ensure we can get a correct length. 36 | Expect(values.Len()).Should(Equal(5)) 37 | // Ensure we can get a value at an index. 38 | Expect(values.Get(3)).Should(Equal(float64(4))) 39 | // Test the sort interface. 40 | sort.Sort(values) 41 | Expect(values.Get(0)).Should(Equal(float64(1))) 42 | Expect(values.Get(1)).Should(Equal(float64(2))) 43 | Expect(values.Get(2)).Should(Equal(float64(3))) 44 | Expect(values.Get(3)).Should(Equal(float64(4))) 45 | Expect(values.Get(4)).Should(Equal(float64(5))) 46 | 47 | }) 48 | }) 49 | 50 | Describe("Testing the statistics calculations", func() { 51 | Context("when the UniqueCount is applied", func() { 52 | values := ValueSlice{4, 5, 2, 3, 2, 4, 4, 5, 6, 7, 8, 9, 0, 4} 53 | count := values.UniqueCount() 54 | It("should find the correct number of unique values", func() { 55 | Expect(count).Should(Equal(9)) 56 | }) 57 | }) 58 | 59 | Context("when the Minmax is applied", func() { 60 | values := ValueSlice{5, math.NaN(), 2, 3, 4, 1} 61 | min, max, err := values.Minmax() 62 | It("should find the min/max values", func() { 63 | Expect(min).Should(Equal(float64(1))) 64 | Expect(max).Should(Equal(float64(5))) 65 | Expect(err).ShouldNot(BeNil()) 66 | }) 67 | 68 | Measure("it should find min/max quickly.", func(b Benchmarker) { 69 | runtime := b.Time("runtime", func() { 70 | min, max, _ = values.Minmax() 71 | }) 72 | Expect(runtime.Seconds()).Should(BeNumerically("<", statsBenchmarkTimeLimit), "it should find min/max quickly.") 73 | }, 100000) 74 | 75 | }) 76 | 77 | Context("when the Median is applied", func() { 78 | values := ValueSlice{123, 234, 345, 456, 567, 678, 789} 79 | median := values.Median() 80 | 81 | It("should find the median value", func() { 82 | Expect(median).Should(Equal(float64(456))) 83 | 84 | values = append(values, 890) 85 | median = values.Median() 86 | Expect(median).Should(Equal(float64(511.5))) 87 | 88 | valuesOriginal := values 89 | values = ValueSlice{} 90 | median = values.Median() 91 | Expect(median).Should(Equal(float64(0.0))) 92 | values = valuesOriginal 93 | }) 94 | 95 | Measure("it should find median quickly.", func(b Benchmarker) { 96 | runtime := b.Time("runtime", func() { 97 | median = values.Median() 98 | }) 99 | Expect(runtime.Seconds()).Should(BeNumerically("<", statsBenchmarkTimeLimit), "it should find median quickly.") 100 | }, 100000) 101 | 102 | }) 103 | 104 | Context("when the Mean is applied", func() { 105 | values := ValueSlice{123, 234, 345, 456, 567, 678, 789, 890} 106 | mean := values.Mean() 107 | It("should find the mean value", func() { 108 | Expect(mean).Should(Equal(float64(510.25))) 109 | }) 110 | 111 | Measure("it should find mean quickly.", func(b Benchmarker) { 112 | runtime := b.Time("runtime", func() { 113 | mean = values.Mean() 114 | }) 115 | Expect(runtime.Seconds()).Should(BeNumerically("<", statsBenchmarkTimeLimit), "it should find mean quickly.") 116 | }, 100000) 117 | 118 | }) 119 | 120 | Context("when the Quantile is applied", func() { 121 | values := ValueSlice{123, 234, 345, 456, 567, 678, 789, 890, 910, 1011} 122 | It("should find the requested quantile value", func() { 123 | q100 := values.Quantile(1) 124 | Expect(q100).Should(Equal(float64(1011))) 125 | q90 := values.Quantile(0.9) 126 | Expect(q90).Should(Equal(float64(920.1))) 127 | q80 := values.Quantile(0.8) 128 | Expect(q80).Should(Equal(float64(894))) 129 | q75 := values.Quantile(0.75) 130 | Expect(q75).Should(Equal(float64(864.75))) 131 | q50 := values.Quantile(0.5) 132 | Expect(q50).Should(Equal(float64(622.5))) 133 | q25 := values.Quantile(0.25) 134 | Expect(q25).Should(Equal(float64(372.75))) 135 | 136 | valuesOriginal := values 137 | values = ValueSlice{} 138 | q0 := values.Quantile(1) 139 | Expect(q0).Should(Equal(float64(0.0))) 140 | values = valuesOriginal 141 | }) 142 | 143 | testQuantile := float64(0.0) 144 | Measure("it should find quantile quickly.", func(b Benchmarker) { 145 | runtime := b.Time("runtime", func() { 146 | testQuantile = values.Quantile(0.9) 147 | }) 148 | Expect(runtime.Seconds()).Should(BeNumerically("<", statsBenchmarkTimeLimit), "it should find quantile quickly.") 149 | }, 100000) 150 | 151 | }) 152 | 153 | Context("when the sum is applied", func() { 154 | values := ValueSlice{123, 234, 345, 456, 567, 678, 789, 890, 910, 1011} 155 | It("should find the correct sum value", func() { 156 | sum := values.Sum() 157 | Expect(sum).Should(Equal(float64(6003.0))) 158 | }) 159 | }) 160 | }) 161 | }) 162 | -------------------------------------------------------------------------------- /statsgod/statsgod_suite_test.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Acquia, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package statsgod_test 18 | 19 | import ( 20 | . "github.com/acquia/statsgod/statsgod" 21 | . "github.com/onsi/ginkgo" 22 | . "github.com/onsi/gomega" 23 | "testing" 24 | "time" 25 | ) 26 | 27 | func TestStatsgod(t *testing.T) { 28 | RegisterFailHandler(Fail) 29 | RunSpecs(t, "Statsgod Suite") 30 | } 31 | 32 | var _ = BeforeSuite(func() { 33 | for i, ts := range testSockets { 34 | socket := CreateSocket(ts.socketType, ts.socketAddr) 35 | go socket.Listen(parseChannel, logger, &config) 36 | BlockForSocket(socket, time.Second) 37 | sockets[i] = socket 38 | } 39 | }) 40 | 41 | var _ = AfterSuite(func() { 42 | for i, _ := range testSockets { 43 | sockets[i].Close(logger) 44 | } 45 | }) 46 | -------------------------------------------------------------------------------- /statsgod_test.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Acquia, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package main 18 | --------------------------------------------------------------------------------