├── .gitignore ├── .travis.yml ├── LICENSE ├── Makefile.am ├── README.md ├── VERSION ├── autogen.sh ├── configure.ac ├── py ├── __init__.py ├── carbonsink.py └── test_carbonsink.py ├── src ├── Makefile.am ├── buffer.c ├── buffer.h ├── hashlib.c ├── hashlib.h ├── hashring.c ├── hashring.h ├── list.c ├── list.h ├── log.c ├── log.h ├── main.c ├── protocol.c ├── protocol.h ├── server.c ├── server.h ├── stathasher.c ├── stats.c ├── stats.h ├── stresstest.c ├── tcpclient.c ├── tcpclient.h ├── tcpserver.c ├── tcpserver.h ├── tests │ ├── empty.yaml │ ├── hashring1.txt │ ├── hashring2.txt │ ├── stathasher.yaml │ ├── stathasher_just_carbon.yaml │ ├── stathasher_just_statsd.yaml │ ├── statsrelay.yaml │ ├── statsrelay_udp.yaml │ ├── test_endtoend.py │ ├── test_hashlib.c │ └── test_hashring.c ├── udpserver.c ├── udpserver.h ├── validate.c ├── validate.h ├── yaml_config.c └── yaml_config.h ├── tag-release.sh └── test ├── devnull.c ├── ketama.servers ├── listener.py ├── poll_stats.py ├── spam.py └── test.servers /.gitignore: -------------------------------------------------------------------------------- 1 | *.log 2 | *.o 3 | *.py[co] 4 | *.swp 5 | *.trs 6 | .deps 7 | .dirstamp 8 | Makefile 9 | Makefile.in 10 | __pycache__ 11 | aclocal.m4 12 | autom4te.cache/ 13 | compile 14 | config.* 15 | configure 16 | configure.scan 17 | core.* 18 | depcomp 19 | install-sh 20 | massif.out.* 21 | missing 22 | perf.data 23 | src/stathasher 24 | src/statsrelay 25 | stamp-* 26 | stresstest 27 | test-driver 28 | test/devnull 29 | test_hashlib 30 | test_hashring 31 | test_stats 32 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: c 2 | compiler: gcc 3 | before_install: 4 | - sudo apt-get update -qq 5 | - sudo apt-get install -qq automake libev-dev libyaml-dev pkg-config cppcheck 6 | script: "autoreconf --install && ./configure && make && make check && cppcheck --enable=all --verbose --suppress=variableScope src/" 7 | notifications: 8 | email: 9 | - statsrelay-dev@googlegroups.com 10 | 11 | env: 12 | global: 13 | # The next declaration is the encrypted COVERITY_SCAN_TOKEN, created 14 | # via the "travis encrypt" command using the project repo's public key 15 | - secure: "SlDD9j89v8VAPzvE4MQ/f8mjOq9Ws6YjXL7v0cBZRBmYQIRBJn4u5yLWNfdrBkwHE06O+t+XVkM5ZV38U1OFbFdYQ27fN6AiAV1MGpwQqd75iIulbGilvwTev5L8ZO5WCrDYcWwwxrOS68JKIRcQdORAxE1eOaiG8Mtqw1nOMOU=" 16 | 17 | addons: 18 | coverity_scan: 19 | project: 20 | name: "uber/statsrelay" 21 | description: "A consistent-hashing relay for statsd metrics" 22 | notification_email: statsrelay-dev@googlegroups.com 23 | build_command_prepend: "autoreconf --install && ./configure" 24 | build_command: "make" 25 | branch_pattern: master 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | statsrelay 2 | Copyright (c) 2016 Uber Technologies, Inc. 3 | The MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to 7 | deal in the Software without restriction, including without limitation the 8 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 9 | sell copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 | IN THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile.am: -------------------------------------------------------------------------------- 1 | SUBDIRS=src 2 | EXTRA_DIST = autogen.sh 3 | 4 | dist_doc_DATA=README.md LICENSE* 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Statsrelay 2 | 3 | (This project is deprecated and not maintained.) 4 | 5 | Statsrelay is a consistent-hashing relay for statsd and carbon metrics. 6 | 7 | [![Build Status](https://travis-ci.org/uber/statsrelay.svg?branch=master)](https://travis-ci.org/uber/statsrelay) 8 | [![Coverity Status](https://scan.coverity.com/projects/2789/badge.svg)](https://scan.coverity.com/projects/2789) 9 | [![Mailing List](https://groups.google.com/forum/#!forum/statsrelay-dev)](https://groups.google.com/forum/#!forum/statsrelay-dev) 10 | 11 | ## License 12 | MIT License 13 | Copyright (c) 2016 Uber Technologies, Inc. 14 | 15 | ## Build 16 | 17 | Dependencies: 18 | - automake 19 | - pkg-config 20 | - libev (>= 4.11) 21 | - libyaml 22 | 23 | On Debian/Ubuntu: 24 | 25 | ``` 26 | apt-get install automake pkg-config libev-dev libyaml-devel 27 | 28 | ./autogen.sh 29 | ./configure 30 | make clean 31 | make 32 | make check 33 | make install 34 | ``` 35 | 36 | ## Use 37 | 38 | ``` 39 | Usage: statsrelay [options] 40 | -h, --help Display this message 41 | -v, --verbose Write log messages to stderr in addition to syslog 42 | syslog 43 | -l, --log-level Set the logging level to DEBUG, INFO, WARN, or ERROR 44 | (default: INFO) 45 | -c, --config=filename Use the given hashring config file 46 | (default: /etc/statsrelay.yaml) 47 | -t, --check-config=filename Check the config syntax 48 | (default: /etc/statsrelay.yaml) 49 | --version Print the version 50 | ``` 51 | 52 | ``` 53 | statsrelay --config=/path/to/statsrelay.yaml 54 | ``` 55 | 56 | This process will run in the foreground. If you need to daemonize, use 57 | start-stop-script, daemontools, supervisord, upstart, systemd, or your 58 | preferred service watchdog. 59 | 60 | By default statsrelay binds to 127.0.0.1:8125 for statsd proxying, and 61 | it binds to 127.0.0.1:2003 for carbon proxying. 62 | 63 | For each line that statsrelay receives in the statsd format 64 | "statname.foo.bar:1|c\n", the key will be hashed to determine which 65 | backend server the stat will be relayed to. If no connection to that 66 | backend is open, the line is queued and a connection attempt is 67 | started. Once a connection is established, all queued metrics are 68 | relayed to the backend and the queue is emptied. If the backend 69 | connection fails, the queue persists in memory and the connection will 70 | be retried after one second. Any stats received for that backend during 71 | the retry window are added to the queue. 72 | 73 | Each backend has its own send queue. If a send queue reaches 74 | `max-send-queue` bytes (default: 128MB) in size, new incoming stats 75 | are dropped until a backend connection is successful and the queue 76 | begins to drain. 77 | 78 | All log messages are sent to syslog with the INFO priority. 79 | 80 | Upon SIGHUP, the config file will be reloaded and all backend 81 | connections closed. Note that any stats in the send queue at the time 82 | of SIGHUP will be dropped. 83 | 84 | If SIGINT or SIGTERM are caught, all connections are killed, send 85 | queues are dropped, and memory freed. statsrelay exits with return 86 | code 0 if all went well. 87 | 88 | To retrieve server statistics, connect to TCP port 8125 and send the 89 | string "status" followed by a newline '\n' character. The end of the 90 | status output is denoted by two consecutive newlines "\n\n" 91 | 92 | stats example: 93 | ``` 94 | $ echo status | nc localhost 8125 95 | 96 | global bytes_recv_udp gauge 0 97 | global bytes_recv_tcp gauge 41 98 | global total_connections gauge 1 99 | global last_reload timestamp 0 100 | global malformed_lines gauge 0 101 | backend:127.0.0.2:8127:tcp bytes_queued gauge 27 102 | backend:127.0.0.2:8127:tcp bytes_sent gauge 27 103 | backend:127.0.0.2:8127:tcp relayed_lines gauge 3 104 | backend:127.0.0.2:8127:tcp dropped_lines gauge 0 105 | ``` 106 | 107 | ## Config Options 108 | 109 | There are a few options you can use to control the behavior of statsrelay, which 110 | can be set in `/etc/statsrelay.yaml`. Here is a minimal config: 111 | 112 | ```yaml 113 | carbon: 114 | bind: 127.0.0.1:9085 115 | tcp_cork: true 116 | validate: true 117 | shard_map: 118 | 0: 10.10.10.10:9085 119 | ``` 120 | 121 | Besides the `bind` and `shard_map` settings (explained elsewhere), here is what 122 | the boolean options do: 123 | 124 | * `tcp_cork` enabled the `TCP_CORK` option on TCP sockets; it's enabled by 125 | default and in many cases will significantly decrease the number of small TCP 126 | sockets that statsrelay emits (at a small penalty of up to 200ms latency in 127 | some cases). 128 | * `validate` tries to validate incoming data before forwarding it to statsd or 129 | carbon; it's on by default 130 | 131 | ## Scaling With Virtual Shards 132 | 133 | Statsrelay implements a virtual sharding scheme, which allows you to 134 | easily scale your statsd and carbon backends by reassigning virtual 135 | shards to actual statsd/carbon instance or servers. This technique 136 | also applies to alternative statsd implementations like statsite. 137 | 138 | Consider the following simplified example with this config file: 139 | 140 | ```yaml 141 | statsd: 142 | bind: 127.0.0.1:8125 143 | validate: true 144 | shard_map: 145 | 0: 10.0.0.1:9000 146 | 1: 10.0.0.1:9000 147 | 2: 10.0.0.1:9001 148 | 3: 10.0.0.1:9001 149 | 4: 10.0.0.2:9000 150 | 5: 10.0.0.2:9000 151 | 6: 10.0.0.2:9001 152 | 7: 10.0.0.2:9001 153 | carbon: 154 | ... 155 | ``` 156 | 157 | In this file we've defined two actual backend hosts (10.0.0.1 and 158 | 10.0.0.2). Each of these hosts is running two statsd instances, one on 159 | port 9000 and one on port 9001 (this is a good way to scale statsd, 160 | since statsd and alternative implementations like statsite are 161 | typically single threaded). In a real setup, you'd likely be running 162 | more statsd instances on each server, and you'd likely have more 163 | repeated lines to assign more virtual shards to each statsd 164 | instance. At Uber we use 4096 virtual shards, with a much smaller 165 | number of actual backend instances. 166 | 167 | Internally statsrelay assigns a zero-indexed virtual shard to each 168 | line in the file; so 10.0.0.1:9000 has virtual shards 0 and 1, 169 | 10.0.0.1:9001 has virtual shards 2 and 3, and so on. 170 | 171 | Let's say that the backend server 10.0.0.1 has become overloaded, and 172 | we want to add a new server to the configuration. We might do that 173 | like this: 174 | 175 | ```yaml 176 | statsd: 177 | bind: 127.0.0.1:8125 178 | validate: true 179 | shard_map: 180 | 0: 10.0.0.1:9000 181 | 1: 10.0.0.3:9000 182 | 2: 10.0.0.1:9001 183 | 3: 10.0.0.3:9001 184 | 4: 10.0.0.2:9000 185 | 5: 10.0.0.2:9000 186 | 6: 10.0.0.2:9001 187 | 7: 10.0.0.2:9001 188 | carbon: 189 | ... 190 | ``` 191 | 192 | In the new configuration we've moved one of the two virtual shards for 193 | 10.0.0.1:9000 to 10.0.0.3:9000, and we've moved one of the two virtual 194 | shards for 10.0.0.1:9001 to 10.0.0.3:9001. In other words, we've 195 | reassigned the mapping for virtual shard 1 and virtual shard 3. Note 196 | that when you do this, you want to maintain the same number of virtual 197 | shards always, so you probably want to pick a large number of virtual 198 | shards to start (say, 1024 virtual shards, meaning the configuration 199 | file should have 1024 lines). You should have many duplicated lines in 200 | the config file when you do this. 201 | 202 | To do optimal shard assignment, you'll want to write a program that 203 | looks at the CPU usage of your shards and figures out the optimal 204 | distribution of shards. How you do that is up to you, but a good 205 | technique is to start by generating a statsrelay config that has many 206 | virtual shards evenly assigned, and then periodically have a script 207 | that finds which actual backends are overloaded and reassigns some of 208 | the virtual shards on those hosts to less loaded hosts (or to new 209 | hosts). 210 | 211 | If you don't initially assign enough virtual shards and then later 212 | expand to more, everything will work, but data migration for carbon 213 | will be a bit trickier; see below. 214 | 215 | ## A Note On Carbon Scaling 216 | 217 | Statsrelay can do relaying for carbon lines just like statsd. The 218 | strategy for scaling carbon using virtual shards is exactly the 219 | same. One important difference, however, is that when you move a 220 | carbon shard you'll want to move the associated whisper files as 221 | well. You can do this using the `stathasher` binary that is built by 222 | statsrelay. By pointing that command at your statsrelay config, you 223 | can send it key names on stdin and have the virtual shard ids printed 224 | to stdout. 225 | 226 | Using this technique you can script the reassignment of whisper 227 | files. The general idea is to walk the filesystem and gather all of 228 | the unique keys stored in carbon backends on a host. You can then get 229 | an idea for how expensive each virtual shard is based on the storage 230 | space, number of whisper files, and possibly I/O metrics for each 231 | virtual shard. By gathering the weights for each virtual shard on a 232 | host, you can figure out the optimal way to redistribute the mapping 233 | of virtual shards to actual carbon backends. 234 | 235 | Note that when you move carbon instances, you also probably want to 236 | migrate the whisper files as well. This ensures that you retain 237 | historical data, and that graphite will get the right answer if it 238 | queries multiple carbon backends. You can migrate the whisper files by 239 | rsyncing the files you've identified as belonging to a moved virtual 240 | shard using the `stathasher` binary described above. Remember to take 241 | care to ensure that the old whisper files are deleted on the old host. 242 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 1.6.8 2 | -------------------------------------------------------------------------------- /autogen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | autoreconf --install 3 | -------------------------------------------------------------------------------- /configure.ac: -------------------------------------------------------------------------------- 1 | AC_PREREQ([2.68]) 2 | AC_INIT([statsrelay], m4_esyscmd([tr -d '\n' < VERSION])) 3 | AM_INIT_AUTOMAKE([-Wall -Werror foreign subdir-objects]) 4 | AC_USE_SYSTEM_EXTENSIONS 5 | AC_CONFIG_HEADERS([config.h]) 6 | 7 | # Checks for programs. 8 | AC_PROG_CC 9 | AM_PROG_CC_C_O 10 | 11 | # Checks for libraries. 12 | 13 | # Checks for header files. 14 | AC_CHECK_HEADERS([arpa/inet.h fcntl.h inttypes.h netdb.h netinet/in.h stddef.h stdint.h stdlib.h string.h sys/socket.h sys/time.h syslog.h unistd.h]) 15 | AC_CHECK_HEADERS([ev.h], [], [AC_MSG_ERROR([unable to find header ev.h])]) 16 | AC_CHECK_HEADERS([yaml.h], [], [AC_MSG_ERROR([unable to find header yaml.h])]) 17 | 18 | # Checks for typedefs, structures, and compiler characteristics. 19 | AC_TYPE_PID_T 20 | AC_TYPE_SIZE_T 21 | AC_TYPE_SSIZE_T 22 | AC_TYPE_UINT16_T 23 | AC_TYPE_UINT32_T 24 | AC_TYPE_UINT64_T 25 | AC_TYPE_UINT8_T 26 | 27 | # Checks for library functions. 28 | AC_FUNC_ERROR_AT_LINE 29 | AC_FUNC_FORK 30 | AC_FUNC_MALLOC 31 | AC_FUNC_REALLOC 32 | AC_FUNC_STRTOD 33 | AC_CHECK_FUNCS([gettimeofday memchr memmove memset socket strchr strdup strerror strndup strrchr strtol]) 34 | 35 | AC_CONFIG_FILES([Makefile 36 | src/Makefile]) 37 | AC_CHECK_LIB([ev], [ev_run]) 38 | AC_CHECK_LIB([yaml], [yaml_parser_initialize]) 39 | AC_REVISION([m4_esyscmd_s([git describe --always])]) 40 | AC_OUTPUT 41 | -------------------------------------------------------------------------------- /py/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/statsrelay/068a8257de82b7686164f82acfe0cc7f42da96da/py/__init__.py -------------------------------------------------------------------------------- /py/carbonsink.py: -------------------------------------------------------------------------------- 1 | """A sink for statsite to send metrics to carbon. 2 | 3 | This supports a few interesting features: 4 | 5 | * There's support for a "file monitoring" sink which works by 6 | detecting specially formatted metrics, and updating the mtime of a 7 | file on the local filesystem; this facilitates monitoring. 8 | 9 | * Support for buffering metrics, which is useful when moving them 10 | with statsrelay. 11 | """ 12 | 13 | import argparse 14 | import contextlib 15 | import logging 16 | import os 17 | import socket 18 | import subprocess 19 | import sys 20 | import time 21 | 22 | 23 | log = logging.getLogger('carbonsink') 24 | 25 | 26 | def get_hash(hasher, key): 27 | hasher.stdin.write(key + '\n') 28 | line = hasher.stdout.readline() 29 | out_dict = {} 30 | for part in line.split(): 31 | k, v = part.split('=', 1) 32 | if k.endswith('_shard'): 33 | v = int(v, 10) 34 | out_dict[k] = v 35 | return out_dict 36 | 37 | 38 | class CarbonMetric(object): 39 | """Representation of a metric in carbon.""" 40 | 41 | __slots__ = ['key', 'value', 'timestamp'] 42 | 43 | def __init__(self, key, value, timestamp): 44 | self.key = key.replace(' ', '_') 45 | self.value = value 46 | self.timestamp = int(timestamp, 10) 47 | 48 | @classmethod 49 | def from_statsite(cls, line): 50 | key, val, timestamp = line.split('|', 2) 51 | return cls(key, val, timestamp) 52 | 53 | def as_carbon_line(self): 54 | return '%s %s %d\n' % (self.key, self.value, self.timestamp) 55 | 56 | 57 | class Sink(object): 58 | """Abstraction for sinks.""" 59 | 60 | def write(self, metric): 61 | raise NotImplemented 62 | 63 | 64 | class CarbonSink(Sink): 65 | """A sink which writes to a carbon-cache or carbon-relay daemon.""" 66 | 67 | def __init__(self, host, port, retries=3): 68 | self.host = host 69 | self.port = int(port) 70 | self.retries = retries 71 | self.sock = None 72 | 73 | def connect(self): 74 | if self.sock is None: 75 | log.info('Connecting to %s:%s', self.host, self.port) 76 | self.sock = socket.create_connection( 77 | (self.host, self.port), timeout=10) 78 | 79 | def write(self, metric): 80 | """Write the metric, retrying if necessary.""" 81 | for attempt in xrange(self.retries): 82 | carbon_line = metric.as_carbon_line() 83 | try: 84 | self.connect() 85 | self.sock.sendall(carbon_line) 86 | return True 87 | except: 88 | log.exception('Failed to send metrics to %s:%s on attempt %d', 89 | self.host, self.port, attempt) 90 | self.close() 91 | 92 | # Only sleep if we are going to retry 93 | if self.retries > 1: 94 | time.sleep(0.1) 95 | log.error('Dropping metric after %d retries', self.retries) 96 | return False 97 | 98 | def close(self): 99 | """Ensure the underlying socket is closed.""" 100 | if self.sock is not None: 101 | try: 102 | self.sock.close() 103 | except: 104 | pass 105 | self.sock = None 106 | 107 | 108 | class FileMonitoringSink(Sink): 109 | """A sink that writes timestamps for specially formatted stats. 110 | 111 | The purpose of this sink is to facilitate monitoring (e.g. from 112 | nagios checks). 113 | """ 114 | 115 | def __init__(self, filename, monitoring_stat): 116 | self.filename = filename 117 | self.monitoring_stat = monitoring_stat 118 | 119 | def write(self, metric): 120 | if metric.key == self.monitoring_stat: 121 | try: 122 | with open(self.filename, 'w') as f: 123 | f.write('%s\n' % (metric.value,)) 124 | except IOError: 125 | log.exception('Failed to write to filename %s', self.filename) 126 | return False 127 | return True 128 | 129 | 130 | class MetricHandler(object): 131 | """Handler for metrics. 132 | 133 | This class encapsulates the logic for knowing when to buffer 134 | stats, how to write to multiple sinks, and so forth. 135 | """ 136 | 137 | STATHASHER_PATH = '/usr/bin/stathasher' 138 | 139 | def __init__(self, servers, prefix, retries=3): 140 | servers = [x.rsplit(':', 1) for x in servers] 141 | self.sinks = [CarbonSink(*x, retries=retries) for x in servers] 142 | self.prefix = prefix 143 | self.buffer_shards = set() 144 | self.buffer_dir = None 145 | self.buffer_cache = {} 146 | self.stathasher = None 147 | 148 | def add_monitoring_sink(self, filename, metric_to_monitor): 149 | metric_to_monitor = '%s.%s' % (self.prefix, metric_to_monitor) 150 | self.sinks.append(FileMonitoringSink(filename, metric_to_monitor)) 151 | 152 | def detect_buffer_shards(self, fileobj, buffer_dir, 153 | stathasher_config=None): 154 | """Populate self.buffer_shards from the buffer file.""" 155 | self.buffer_dir = buffer_dir 156 | try: 157 | for line in fileobj: 158 | try: 159 | shard_num = int(line, 10) 160 | except ValueError: 161 | pass 162 | self.buffer_shards.add(shard_num) 163 | except IOError: 164 | # if we can't open the file (e.g. it doesn't exist), then 165 | # there are no buffer shards to exclude 166 | pass 167 | 168 | if self.buffer_shards: 169 | args = [self.STATHASHER_PATH] 170 | if stathasher_config is not None: 171 | args.extend(['-c', stathasher_config]) 172 | self.stathasher = subprocess.Popen( 173 | args, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 174 | 175 | def lookup_buffer(self, shard_num): 176 | """Get the buffer file for a shard num, for appending to.""" 177 | try: 178 | return self.buffer_cache[shard_num] 179 | except KeyError: 180 | filename = os.path.join( 181 | self.buffer_dir, 'shard_%s.txt' % (shard_num,)) 182 | file_obj = open(filename, 'a') 183 | self.buffer_cache[shard_num] = file_obj 184 | return file_obj 185 | 186 | def close(self): 187 | """Ensure the stathasher process terminates.""" 188 | if self.stathasher and self.stathasher.returncode is None: 189 | self.stathasher.kill() 190 | for v in self.buffer_cache.itervalues(): 191 | v.close() 192 | 193 | def buffer_line(self, line, metric): 194 | """Buffer the line if buffering is enabled for this metric. 195 | 196 | Returns True if the line is buffered, False otherwise. 197 | """ 198 | if not self.buffer_shards: 199 | return False 200 | hashval = get_hash(self.stathasher, metric.key) 201 | shard = hashval['carbon_shard'] 202 | if shard in self.buffer_shards: 203 | file_obj = self.lookup_buffer(shard) 204 | file_obj.write(metric.as_carbon_line()) 205 | file_obj.flush() 206 | return True 207 | else: 208 | return False 209 | 210 | def handle_metric(self, line): 211 | key, val, timestamp = line.split('|', 2) 212 | metric = CarbonMetric(self.prefix + '.' + key, val, timestamp) 213 | if not self.buffer_line(line, metric): 214 | # The above call will buffer the line if buffering is 215 | # enabled and suitable for the metric; in the typical 216 | # case, no buffering will happen, and instead we write the 217 | # metric to all of the sinks. 218 | for sink in self.sinks: 219 | if not sink.write(metric): 220 | log.error( 221 | 'Hard failure sending to %r, giving up on it', sink) 222 | continue 223 | 224 | 225 | @contextlib.contextmanager 226 | def metric_handler(*args, **kwargs): 227 | with contextlib.closing(MetricHandler(*args, **kwargs)) as handler: 228 | yield handler 229 | 230 | 231 | def main(): 232 | parser = argparse.ArgumentParser() 233 | parser.add_argument('--attempts', type=int, default=3) 234 | parser.add_argument('--logfile', default=None) 235 | parser.add_argument('--statsite-instance', default=None) 236 | parser.add_argument('--monitoring-stat', default=None) 237 | parser.add_argument('-c', '--cache-directory', 238 | default='/var/cache/statsite', 239 | help='directory for cache files') 240 | parser.add_argument('-b', '--buffer-shard-file', 241 | default='/etc/statsrelay_buffer_shards.txt', 242 | help='buffer metrics for shards from this file') 243 | parser.add_argument('--statsrelay-config', 244 | default=None, 245 | help='statsrelay config to use when hashing') 246 | parser.add_argument('prefix') 247 | parser.add_argument('servers', nargs='+') 248 | args = parser.parse_args() 249 | 250 | logging.basicConfig() 251 | if args.logfile: 252 | fh = logging.FileHandler(args.logfile) 253 | fh.setFormatter(logging.Formatter( 254 | fmt='%(asctime)s\t%(levelname)s\t%(message)s')) 255 | log.addHandler(fh) 256 | 257 | with metric_handler(args.servers, args.prefix, 258 | retries=args.attempts) as handler: 259 | if args.statsite_instance and args.monitoring_stat: 260 | filename = os.path.join( 261 | args.cache_directory, args.statsite_instance) 262 | handler.add_monitoring_sink(filename, args.monitoring_stat) 263 | 264 | try: 265 | with open(args.buffer_shard_file) as buffer_shard_file: 266 | handler.detect_buffer_shards( 267 | buffer_shard_file, 268 | args.cache_directory, 269 | args.statsrelay_config) 270 | except (OSError, IOError): 271 | pass 272 | 273 | # N.B. we want to force stdin to be read iteratively, to avoid 274 | # buffering when statsite is sending a large amount of 275 | # data. This means that we must explicitly get lines using 276 | # stdin.readline(), and we can't use a map or iterator over 277 | # stdin. 278 | while True: 279 | line = sys.stdin.readline().rstrip() 280 | if not line: 281 | break 282 | handler.handle_metric(line.rstrip()) 283 | 284 | 285 | if __name__ == '__main__': 286 | main() 287 | -------------------------------------------------------------------------------- /py/test_carbonsink.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import random 4 | import shutil 5 | import socket 6 | import subprocess 7 | import tempfile 8 | import unittest 9 | import StringIO 10 | 11 | import carbonsink 12 | 13 | 14 | class TestCarbonMetric(unittest.TestCase): 15 | 16 | def test_simple(self): 17 | metric = carbonsink.CarbonMetric.from_statsite('foo|bar|1') 18 | self.assertEqual(metric.key, 'foo') 19 | self.assertEqual(metric.value, 'bar') 20 | self.assertEqual(metric.timestamp, 1) 21 | self.assertEqual(metric.as_carbon_line(), 'foo bar 1\n') 22 | 23 | def test_spaces(self): 24 | metric = carbonsink.CarbonMetric.from_statsite('foo quux|bar|1') 25 | self.assertEqual(metric.key, 'foo_quux') # space is transformed to _ 26 | self.assertEqual(metric.value, 'bar') 27 | self.assertEqual(metric.timestamp, 1) 28 | 29 | 30 | class TestFileMonitoringSink(unittest.TestCase): 31 | 32 | def setUp(self): 33 | self.tempfile = tempfile.NamedTemporaryFile() 34 | 35 | def check_contents(self, expected): 36 | with open(self.tempfile.name) as tf: 37 | contents = tf.read() 38 | self.assertEqual(contents, expected) 39 | 40 | def test_file_monitoring_sink(self): 41 | self.check_contents('') 42 | sink = carbonsink.FileMonitoringSink(self.tempfile.name, 'foo') 43 | sink.write(carbonsink.CarbonMetric.from_statsite('wrong|0|1')) 44 | self.check_contents('') 45 | sink.write(carbonsink.CarbonMetric.from_statsite('foo|2|3')) 46 | self.check_contents('2\n') 47 | sink.write(carbonsink.CarbonMetric.from_statsite('foo|4|5')) 48 | self.check_contents('4\n') 49 | 50 | 51 | class ListenerTestCase(unittest.TestCase): 52 | """Allow tests to run that listen on a port. 53 | 54 | This is kind of a hack. The listener is created using nc with the 55 | subset of options that seem to be portable across the different nc 56 | implementations. 57 | """ 58 | 59 | def setUp(self): 60 | super(ListenerTestCase, self).setUp() 61 | sock = socket.socket() 62 | sock.bind(('127.0.0.1', 0)) 63 | _, port = sock.getsockname() 64 | sock.close() 65 | self.proc = subprocess.Popen( 66 | ['nc', '-l', '-p', str(port)], 67 | stdout=subprocess.PIPE) 68 | self.port = port 69 | 70 | def tearDown(self): 71 | self.proc.kill() 72 | super(ListenerTestCase, self).tearDown() 73 | 74 | 75 | class TestCarbonSink(ListenerTestCase): 76 | 77 | def test_carbon_sink(self): 78 | sink = carbonsink.CarbonSink('127.0.0.1', self.port) 79 | metric = carbonsink.CarbonMetric.from_statsite('foo|bar|1') 80 | sink.write(metric) 81 | self.assertEqual(self.proc.stdout.readline(), 'foo bar 1\n') 82 | 83 | 84 | class TestMetricHandler(ListenerTestCase): 85 | 86 | def setUp(self): 87 | super(TestMetricHandler, self).setUp() 88 | self.hasher = None 89 | self.addr = '127.0.0.1:%d' % (self.port,) 90 | self.tempfile = tempfile.NamedTemporaryFile() 91 | self.tempd = None 92 | 93 | def tearDown(self): 94 | if self.hasher is not None: 95 | self.hasher.kill() 96 | if self.tempd is not None: 97 | shutil.rmtree(self.tempd) 98 | super(TestMetricHandler, self).tearDown() 99 | 100 | def check_monitoring_contents(self, expected): 101 | with open(self.tempfile.name) as tf: 102 | contents = tf.read() 103 | self.assertEqual(contents, expected) 104 | 105 | def check_line(self, expected): 106 | data = self.proc.stdout.readline() 107 | self.assertEqual(data, expected + '\n') 108 | 109 | def test_no_buffering(self): 110 | with carbonsink.metric_handler([self.addr], 'prefix') as handler: 111 | self.check_monitoring_contents('') 112 | handler.add_monitoring_sink(self.tempfile.name, 'mon') 113 | self.check_monitoring_contents('') 114 | 115 | # handle a simple metric 116 | handler.handle_metric('foo|bar|1') 117 | self.check_monitoring_contents('') 118 | self.check_line('prefix.foo bar 1') 119 | 120 | # test the monitoring metric 121 | handler.handle_metric('mon|foo|1') 122 | self.check_monitoring_contents('foo\n') 123 | self.check_line('prefix.mon foo 1') 124 | 125 | # test again without the monitoring metric 126 | handler.handle_metric('quux|baz|1') 127 | self.check_monitoring_contents('foo\n') 128 | self.check_line('prefix.quux baz 1') 129 | 130 | def test_with_buffering(self): 131 | fruits = ['apple', 'banana', 'cherry', 'durian', 'guava', 'kiwi', 132 | 'lemon', 'orange', 'peach', 'pear', 'quince', 'strawberry'] 133 | 134 | stathasher_file = tempfile.NamedTemporaryFile() 135 | stathasher_file.write("carbon:\n bind: 127.0.0.1:2004\n " 136 | "shard_map: {0: '127.0.0.1:2000', " 137 | "1: '127.0.0.1:2000', " 138 | "2: '127.0.0.1:2000', " 139 | "3: '127.0.0.1:2000'}\n") 140 | stathasher_file.flush() 141 | 142 | self.hasher = subprocess.Popen( 143 | [carbonsink.MetricHandler.STATHASHER_PATH, 144 | '-c', stathasher_file.name], 145 | stdin=subprocess.PIPE, 146 | stdout=subprocess.PIPE) 147 | 148 | fruit_hash = {} 149 | for fruit in fruits: 150 | fruit_key = '%s.%s' % ('prefix', fruit) 151 | fruit_hash[fruit] = carbonsink.get_hash( 152 | self.hasher, fruit_key)['carbon_shard'] 153 | 154 | # ensure that all shards are accounted for 155 | self.assertEqual(set(fruit_hash.values()), set([0, 1, 2, 3])) 156 | 157 | # pick two random shards to buffer 158 | buffer_shards = random.sample([0, 1, 2, 3], 2) 159 | buffer_file = StringIO.StringIO('\n'.join(map(str, buffer_shards))) 160 | 161 | self.tempd = tempfile.mkdtemp() 162 | 163 | with carbonsink.metric_handler([self.addr], 'prefix') as handler: 164 | handler.detect_buffer_shards( 165 | buffer_file, self.tempd, stathasher_file.name) 166 | for fruit in fruits: 167 | carbon_line = 'prefix.%s foo 1' % (fruit,) 168 | shard_num = fruit_hash[fruit] 169 | handler.handle_metric('%s|foo|1' % (fruit,)) 170 | if shard_num not in buffer_shards: 171 | # check that the nc process received the data 172 | self.check_line(carbon_line) 173 | else: 174 | # open the buffer file and get the last line 175 | filename = os.path.join( 176 | self.tempd, 'shard_%d.txt' % (shard_num,)) 177 | with open(filename) as log_file: 178 | for line in log_file: 179 | pass 180 | # ensure the final line is the buffered line 181 | self.assertEqual(carbon_line + '\n', line) 182 | 183 | 184 | if __name__ == '__main__': 185 | logging.basicConfig() 186 | logging.root.setLevel(logging.ERROR + 1) 187 | unittest.main() 188 | -------------------------------------------------------------------------------- /src/Makefile.am: -------------------------------------------------------------------------------- 1 | AM_CFLAGS=-O2 -g -std=c99 -pedantic -Wall -fno-strict-aliasing -D_XOPEN_SOURCE=700 -D_DEFAULT_SOURCE -D_BSD_SOURCE 2 | bin_PROGRAMS=statsrelay stathasher stresstest 3 | BASE_SOURCES=buffer.c hashlib.c hashring.c list.c log.c protocol.c tcpclient.c tcpserver.c udpserver.c server.c stats.c validate.c yaml_config.c 4 | statsrelay_SOURCES=$(BASE_SOURCES) main.c 5 | stathasher_SOURCES=hashlib.c hashring.c list.c log.c yaml_config.c stathasher.c 6 | stresstest_SOURCES=stresstest.c 7 | 8 | check_PROGRAMS=test_hashlib test_hashring 9 | TESTS=$(check_PROGRAMS) tests/test_endtoend.py 10 | test_hashlib_SOURCES=tests/test_hashlib.c hashlib.c 11 | test_hashring_SOURCES=tests/test_hashring.c hashlib.c hashring.c list.c log.c 12 | -------------------------------------------------------------------------------- /src/buffer.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "buffer.h" 5 | 6 | #define INITIAL_BUFFER_SIZE 4096 7 | 8 | 9 | /* 10 | ptr head tail 11 | | |XXXXXXXXXXXXXXXXXXXXXX| | 12 | [ ---------------- size ---------------------------] 13 | [ spacecount ] 14 | [ datacount ] 15 | 16 | */ 17 | 18 | int buffer_allocate(buffer_t *b, size_t size) 19 | { 20 | b->size = size; 21 | b->ptr = (char *)malloc(b->size); 22 | if (!b->ptr) return -1; 23 | #ifdef SANITIZE_BUFFERS 24 | memset(b->ptr, 0, size); 25 | #endif 26 | b->head = b->ptr; 27 | b->tail = b->ptr; 28 | return 0; 29 | } 30 | 31 | int buffer_init(buffer_t *b) 32 | { 33 | return buffer_allocate(b, INITIAL_BUFFER_SIZE); 34 | } 35 | 36 | int buffer_init_contents(buffer_t *b, const char *data, size_t size) 37 | { 38 | buffer_allocate(b, size); 39 | return buffer_set(b, data, size); 40 | } 41 | 42 | buffer_t *create_buffer(size_t size) 43 | { 44 | buffer_t *ret = (buffer_t *)malloc(sizeof(*ret)); 45 | if (0 != buffer_allocate(ret, size)) { 46 | free(ret); 47 | ret = NULL; 48 | } 49 | return ret; 50 | } 51 | 52 | size_t buffer_datacount(buffer_t *b) 53 | { 54 | return b->tail - b->head; 55 | } 56 | 57 | size_t buffer_spacecount(buffer_t *b) 58 | { 59 | return b->size - (b->tail - b->ptr); 60 | } 61 | 62 | char *buffer_head(buffer_t *b) 63 | { 64 | return b->head; 65 | } 66 | 67 | char *buffer_tail(buffer_t *b) 68 | { 69 | return b->tail; 70 | } 71 | 72 | /* Assumes we are always making it bigger */ 73 | char *myrealloc(char *p, size_t old, size_t new) 74 | { 75 | char *pnew = malloc(new); 76 | if (pnew == NULL) { 77 | return NULL; 78 | } 79 | memcpy(pnew, p, old); 80 | free(p); 81 | return pnew; 82 | } 83 | 84 | int buffer_newsize(buffer_t *b, size_t newsize) 85 | { 86 | char *pnew = myrealloc(b->ptr, b->size, newsize); 87 | if (!pnew) 88 | return -1; 89 | b->head = pnew + (b->head - b->ptr); 90 | b->tail = pnew + (b->tail - b->ptr); 91 | b->ptr = pnew; 92 | b->size = newsize; 93 | return 0; 94 | } 95 | 96 | int buffer_expand(buffer_t *b) 97 | { 98 | return buffer_newsize(b, b->size * 2); 99 | } 100 | 101 | int buffer_consume(buffer_t *b, size_t amt) 102 | { 103 | if (b->head + amt > b->tail) 104 | return -1; 105 | 106 | b->head += amt; 107 | return 0; 108 | } 109 | 110 | int buffer_produced(buffer_t *b, size_t amt) 111 | { 112 | if ((b->tail + amt) - b->ptr > b->size) 113 | return -1; 114 | 115 | b->tail += amt; 116 | return 0; 117 | } 118 | 119 | int buffer_set(buffer_t *b, const char *data, size_t size) 120 | { 121 | if (b->size < size) { 122 | if (0 != buffer_newsize(b, size)) 123 | return -1; 124 | } 125 | memcpy(b->ptr, data, size); 126 | return buffer_produced(b, size); 127 | } 128 | 129 | int buffer_realign(buffer_t *b) 130 | { 131 | if (b->tail != b->head) { 132 | memmove(b->ptr, b->head, b->tail - b->head); 133 | } 134 | /* do not switch the order of the following two statements */ 135 | b->tail = b->ptr + (b->tail - b->head); 136 | b->head = b->ptr; 137 | return 0; 138 | } 139 | 140 | void buffer_destroy(buffer_t *b) 141 | { 142 | free(b->ptr); 143 | b->head = NULL; 144 | b->tail = NULL; 145 | b->size = 0; 146 | } 147 | 148 | void delete_buffer(buffer_t *b) 149 | { 150 | buffer_destroy(b); 151 | free(b); 152 | } 153 | 154 | void buffer_wrap(buffer_t *b, const char *data, size_t size) 155 | { 156 | /* Promise not to modify */ 157 | b->ptr = (char *)data; 158 | b->head = b->ptr; 159 | b->tail = b->head + size; 160 | b->size = 0; 161 | } 162 | -------------------------------------------------------------------------------- /src/buffer.h: -------------------------------------------------------------------------------- 1 | #ifndef BUFFER_H 2 | #define BUFFER_H 3 | 4 | #include 5 | 6 | struct buffer { 7 | char *ptr; 8 | char *head; 9 | char *tail; 10 | size_t size; 11 | }; 12 | 13 | typedef struct buffer buffer_t; 14 | 15 | // Init a buffer to default size 16 | int buffer_init(buffer_t *); 17 | int buffer_init_contents(buffer_t *, const char *, size_t); 18 | // Create a new buffer at specified size 19 | buffer_t *create_buffer(size_t size); 20 | 21 | // Returns the size of the consumed space in buffer 22 | size_t buffer_datacount(buffer_t *); 23 | 24 | // Returns the size of available space in buffer 25 | size_t buffer_spacecount(buffer_t *); 26 | 27 | // Returns a pointer to the beginning of used space 28 | char *buffer_head(buffer_t *); 29 | 30 | // Returns a pointer to the end of used space 31 | char *buffer_tail(buffer_t *); 32 | 33 | // Doubles the size of the buffer 34 | int buffer_expand(buffer_t *); 35 | 36 | // Expands the buffer to the given size 37 | int buffer_newsize(buffer_t *b, size_t newsize); 38 | 39 | // Advances head 40 | int buffer_consume(buffer_t *, size_t); 41 | 42 | // Advances tail 43 | int buffer_produced(buffer_t *, size_t); 44 | 45 | // Sets to the new contents, expanding if necessary 46 | int buffer_set(buffer_t *, const char *data, size_t size); 47 | 48 | // Copy data from head to the beginning of the buffer 49 | int buffer_realign(buffer_t *); 50 | 51 | // Frees all memory associated with the buffer 52 | void buffer_destroy(buffer_t *); 53 | // Delete the buffer object 54 | void delete_buffer(buffer_t *); 55 | 56 | /* Take this piece of memory and wrap it. Don't copy it, don't touch it, 57 | just wrap it. This is entirely for interface compatibilty with things 58 | that take buffers */ 59 | void buffer_wrap(buffer_t *, const char *, size_t); 60 | 61 | #endif 62 | -------------------------------------------------------------------------------- /src/hashlib.c: -------------------------------------------------------------------------------- 1 | #include "hashlib.h" 2 | 3 | // This has to be a constant value, so that things don't get hashed 4 | // differently when we restart statsrelay. 5 | static const uint32_t HASHLIB_SEED = 0xaccd3d34; 6 | 7 | // From Wikipedia 8 | static uint32_t murmur3_32(const char *key, uint32_t len, uint32_t seed) { 9 | static const uint32_t c1 = 0xcc9e2d51; 10 | static const uint32_t c2 = 0x1b873593; 11 | static const uint32_t r1 = 15; 12 | static const uint32_t r2 = 13; 13 | static const uint32_t m = 5; 14 | static const uint32_t n = 0xe6546b64; 15 | 16 | uint32_t hash = seed; 17 | 18 | const int nblocks = len / 4; 19 | const uint32_t *blocks = (const uint32_t *) key; 20 | int i; 21 | for (i = 0; i < nblocks; i++) { 22 | uint32_t k = blocks[i]; 23 | k *= c1; 24 | k = (k << r1) | (k >> (32 - r1)); 25 | k *= c2; 26 | 27 | hash ^= k; 28 | hash = ((hash << r2) | (hash >> (32 - r2))) * m + n; 29 | } 30 | 31 | const uint8_t *tail = (const uint8_t *) (key + nblocks * 4); 32 | uint32_t k1 = 0; 33 | 34 | switch (len & 3) { 35 | case 3: 36 | k1 ^= tail[2] << 16; 37 | case 2: 38 | k1 ^= tail[1] << 8; 39 | case 1: 40 | k1 ^= tail[0]; 41 | 42 | k1 *= c1; 43 | k1 = (k1 << r1) | (k1 >> (32 - r1)); 44 | k1 *= c2; 45 | hash ^= k1; 46 | } 47 | 48 | hash ^= len; 49 | hash ^= (hash >> 16); 50 | hash *= 0x85ebca6b; 51 | hash ^= (hash >> 13); 52 | hash *= 0xc2b2ae35; 53 | hash ^= (hash >> 16); 54 | 55 | return hash; 56 | } 57 | 58 | 59 | uint32_t stats_hash(const char *key, 60 | uint32_t keylen, 61 | uint32_t output_domain) { 62 | return murmur3_32(key, keylen, HASHLIB_SEED) % output_domain; 63 | } 64 | -------------------------------------------------------------------------------- /src/hashlib.h: -------------------------------------------------------------------------------- 1 | #ifndef STATSRELAY_HASHLIB_H 2 | #define STATSRELAY_HASHLIB_H 3 | 4 | #include 5 | 6 | // hash a key to get a value in the range [0, output_domain) 7 | uint32_t stats_hash(const char *key, 8 | uint32_t keylen, 9 | uint32_t output_domain); 10 | 11 | #endif // STATSRELAY_HASHLIB_H 12 | -------------------------------------------------------------------------------- /src/hashring.c: -------------------------------------------------------------------------------- 1 | #include "./hashring.h" 2 | 3 | #include "./hashlib.h" 4 | #include "./list.h" 5 | #include "./log.h" 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | struct hashring { 14 | list_t backends; 15 | void *alloc_data; 16 | hashring_alloc_func alloc; 17 | hashring_dealloc_func dealloc; 18 | }; 19 | 20 | hashring_t hashring_init(void *alloc_data, 21 | hashring_alloc_func alloc, 22 | hashring_dealloc_func dealloc) { 23 | struct hashring *ring = malloc(sizeof(struct hashring)); 24 | if (ring == NULL) { 25 | stats_error_log("failure to malloc() in hashring_init"); 26 | return NULL; 27 | } 28 | ring->backends = statsrelay_list_new(); 29 | ring->alloc_data = alloc_data; 30 | ring->alloc = alloc; 31 | ring->dealloc = dealloc; 32 | return ring; 33 | } 34 | 35 | hashring_t hashring_load_from_config(struct proto_config *pc, 36 | void *alloc_data, 37 | hashring_alloc_func alloc_func, 38 | hashring_dealloc_func dealloc_func) { 39 | hashring_t ring = hashring_init(alloc_data, alloc_func, dealloc_func); 40 | if (ring == NULL) { 41 | stats_error_log("failed to hashring_init"); 42 | return NULL; 43 | } 44 | for (size_t i = 0; i < pc->ring->size; i++) { 45 | if (!hashring_add(ring, pc->ring->data[i])) { 46 | hashring_dealloc(ring); 47 | return NULL; 48 | } 49 | } 50 | return ring; 51 | } 52 | 53 | bool hashring_add(hashring_t ring, const char *line) { 54 | if (line == NULL) { 55 | stats_error_log("cowardly refusing to alloc NULL pointer"); 56 | goto add_err; 57 | } 58 | // allocate an object 59 | void *obj = ring->alloc(line, ring->alloc_data); 60 | if (obj == NULL) { 61 | stats_error_log("hashring: failed to alloc line \"%s\"", line); 62 | goto add_err; 63 | } 64 | 65 | // grow the list 66 | if (statsrelay_list_expand(ring->backends) == NULL) { 67 | stats_error_log("hashring: failed to expand list"); 68 | ring->dealloc(obj); 69 | goto add_err; 70 | } 71 | 72 | ring->backends->data[ring->backends->size - 1] = obj; 73 | return true; 74 | 75 | add_err: 76 | return false; 77 | } 78 | 79 | size_t hashring_size(hashring_t ring) { 80 | if (ring == NULL) { 81 | return 0; 82 | } 83 | return ring->backends->size; 84 | } 85 | 86 | void* hashring_choose(struct hashring *ring, 87 | const char *key, 88 | uint32_t *shard_num) { 89 | if (ring == NULL || ring->backends == NULL) { 90 | return NULL; 91 | } 92 | const size_t ring_size = ring->backends->size; 93 | if (ring_size == 0) { 94 | return NULL; 95 | } 96 | const uint32_t index = stats_hash(key, strlen(key), ring_size); 97 | if (shard_num != NULL) { 98 | *shard_num = index; 99 | } 100 | return ring->backends->data[index]; 101 | } 102 | 103 | void hashring_dealloc(struct hashring *ring) { 104 | if (ring == NULL) { 105 | return; 106 | } 107 | if (ring->backends == NULL) { 108 | return; 109 | } 110 | const size_t ring_size = ring->backends->size; 111 | for (size_t i = 0; i < ring_size; i++) { 112 | bool need_dealloc = true; 113 | for (size_t j = 0; j < i; j++) { 114 | if (ring->backends->data[i] == ring->backends->data[j]) { 115 | need_dealloc = false; 116 | break; 117 | } 118 | } 119 | if (need_dealloc) { 120 | ring->dealloc(ring->backends->data[i]); 121 | } 122 | } 123 | statsrelay_list_destroy(ring->backends); 124 | free(ring); 125 | } 126 | -------------------------------------------------------------------------------- /src/hashring.h: -------------------------------------------------------------------------------- 1 | #ifndef STATSRELAY_HASHRING_H 2 | #define STATSRELAY_HASHRING_H 3 | 4 | #include 5 | #include 6 | 7 | #include "./yaml_config.h" 8 | 9 | typedef void* (*hashring_alloc_func)(const char *, void *data); 10 | typedef void (*hashring_dealloc_func)(void *); 11 | 12 | // opaque hashring type 13 | struct hashring; 14 | typedef struct hashring* hashring_t; 15 | 16 | // Initialize the hashring with the list of backends. 17 | hashring_t hashring_init(void *alloc_data, 18 | hashring_alloc_func alloc_func, 19 | hashring_dealloc_func dealloc_func); 20 | 21 | 22 | hashring_t hashring_load_from_config(struct proto_config *pc, 23 | void *alloc_data, 24 | hashring_alloc_func alloc_func, 25 | hashring_dealloc_func dealloc_func); 26 | 27 | // Add an item to the hashring; returns true on success, false on 28 | // failure. 29 | bool hashring_add(hashring_t ring, const char *line); 30 | 31 | // The size of the hashring 32 | size_t hashring_size(hashring_t ring); 33 | 34 | // Choose a backend; if shard_num is not NULL, the shard number that 35 | // was used will be placed into the return value. 36 | void *hashring_choose(hashring_t ring, 37 | const char *key, 38 | uint32_t *shard_num); 39 | 40 | // Release allocated memory 41 | void hashring_dealloc(hashring_t ring); 42 | 43 | #endif // STATSRELAY_HASHRING_H 44 | -------------------------------------------------------------------------------- /src/list.c: -------------------------------------------------------------------------------- 1 | #include "list.h" 2 | 3 | #include 4 | 5 | 6 | list_t statsrelay_list_new() { 7 | list_t list = malloc(sizeof(struct statsrelay_list)); 8 | if (list == NULL) { 9 | return list; 10 | } 11 | list->data = NULL; 12 | list->allocated_size = 0; 13 | list->size = 0; 14 | return list; 15 | } 16 | 17 | void* statsrelay_list_expand(list_t list) { 18 | size_t index = list->size; 19 | list->size++; 20 | 21 | if (list->allocated_size < list->size) { 22 | if (list->allocated_size == 0) { 23 | list->allocated_size = 1; 24 | list->data = malloc(sizeof(void *)); 25 | if (list->data == NULL) { 26 | return NULL; 27 | } 28 | } else { 29 | list->allocated_size <<= 1; 30 | void *newdata = realloc( 31 | list->data, sizeof(void *) * list->allocated_size); 32 | if (newdata == NULL) { 33 | perror("realloc()"); 34 | return NULL; 35 | } 36 | list->data = newdata; 37 | } 38 | } 39 | return list->data + index; 40 | } 41 | 42 | void statsrelay_list_destroy(list_t list) { 43 | free(list->data); 44 | free(list); 45 | } 46 | 47 | void statsrelay_list_destroy_full(list_t list) { 48 | for (size_t i = 0; i < list->size; i++) { 49 | free(list->data[i]); 50 | } 51 | statsrelay_list_destroy(list); 52 | } 53 | -------------------------------------------------------------------------------- /src/list.h: -------------------------------------------------------------------------------- 1 | #ifndef STATSRELAY_LIST_H 2 | #define STATSRELAY_LIST_H 3 | 4 | #include 5 | 6 | struct statsrelay_list { 7 | size_t allocated_size; 8 | size_t size; 9 | void **data; 10 | }; 11 | 12 | typedef struct statsrelay_list* list_t; 13 | 14 | // create a new list 15 | list_t statsrelay_list_new(); 16 | 17 | // get the address for a new item in the list, and ensure its size is 18 | // expanded 19 | void *statsrelay_list_expand(list_t list); 20 | 21 | // deallocate the list 22 | void statsrelay_list_destroy(list_t list); 23 | 24 | // deallocate the list and its contents 25 | void statsrelay_list_destroy_full(list_t list); 26 | 27 | 28 | #endif // STATSRELAY_LIST_H 29 | -------------------------------------------------------------------------------- /src/log.c: -------------------------------------------------------------------------------- 1 | #include "log.h" 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #define STATSRELAY_LOG_BUF_SIZE 256 9 | 10 | static bool g_verbose = 0; 11 | static enum statsrelay_log_level g_level; 12 | static int fmt_buf_size = 0; 13 | static char *fmt_buf = NULL; 14 | 15 | void stats_log_verbose(bool verbose) { 16 | g_verbose = verbose; 17 | } 18 | 19 | void stats_set_log_level(enum statsrelay_log_level level) { 20 | g_level = level; 21 | } 22 | 23 | void stats_vlog(const char *prefix, 24 | const char *format, 25 | va_list ap) { 26 | int fmt_len; 27 | char *np; 28 | size_t total_written, bw; 29 | // Allocate the format buffer on the first log call 30 | if (fmt_buf == NULL) { 31 | if ((fmt_buf = malloc(STATSRELAY_LOG_BUF_SIZE)) == NULL) { 32 | goto alloc_failure; 33 | } 34 | fmt_buf_size = STATSRELAY_LOG_BUF_SIZE; 35 | } 36 | 37 | // Keep trying to vsnprintf until we have a sufficiently sized buffer 38 | // allocated. 39 | while (1) { 40 | fmt_len = vsnprintf(fmt_buf, fmt_buf_size, format, ap); 41 | 42 | if (fmt_len < 0) { 43 | return; // output error (shouldn't happen for vs* functions) 44 | } else if (fmt_len < fmt_buf_size) { 45 | break; // vsnprintf() didn't truncate 46 | } 47 | 48 | fmt_buf_size <<= 1; // double size 49 | 50 | if ((np = realloc(fmt_buf, fmt_buf_size)) == NULL) { 51 | goto alloc_failure; 52 | } 53 | fmt_buf = np; 54 | } 55 | 56 | if (g_verbose == 1) { 57 | if (prefix != NULL) { 58 | fprintf(stderr, prefix); 59 | } 60 | total_written = 0; 61 | while (total_written < fmt_len) { 62 | // try to write to stderr, but if there are any 63 | // failures (e.g. parent had closed stderr) then just 64 | // proceed to the syslog call 65 | bw = fwrite(fmt_buf + total_written, sizeof(char), fmt_len - total_written, stderr); 66 | if (bw == 0) { 67 | break; 68 | } 69 | total_written += bw; 70 | } 71 | if (total_written >= fmt_len) { 72 | fputc('\n', stderr); 73 | } 74 | } 75 | 76 | syslog(LOG_INFO, fmt_buf, fmt_len); 77 | 78 | if (fmt_buf_size > STATSRELAY_LOG_BUF_SIZE) { 79 | if ((np = realloc(fmt_buf, STATSRELAY_LOG_BUF_SIZE)) == NULL) { 80 | goto alloc_failure; 81 | } 82 | fmt_buf = np; 83 | fmt_buf_size = STATSRELAY_LOG_BUF_SIZE; 84 | } 85 | return; 86 | 87 | alloc_failure: 88 | stats_log_end(); // reset everything 89 | return; 90 | } 91 | 92 | void stats_debug_log(const char *format, ...) { 93 | if (g_level <= STATSRELAY_LOG_DEBUG) { 94 | va_list args; 95 | va_start(args, format); 96 | stats_vlog("DEBUG: ", format, args); 97 | va_end(args); 98 | } 99 | } 100 | 101 | void stats_log(const char *format, ...) { 102 | if (g_level <= STATSRELAY_LOG_INFO) { 103 | va_list args; 104 | va_start(args, format); 105 | stats_vlog(NULL, format, args); 106 | va_end(args); 107 | } 108 | } 109 | 110 | void stats_error_log(const char *format, ...) { 111 | if (g_level <= STATSRELAY_LOG_ERROR) { 112 | bool orig_verbose = g_verbose; 113 | g_verbose = true; 114 | va_list args; 115 | va_start(args, format); 116 | stats_vlog("ERROR: ", format, args); 117 | va_end(args); 118 | g_verbose = orig_verbose; 119 | } 120 | } 121 | 122 | void stats_log_end(void) { 123 | free(fmt_buf); 124 | fmt_buf = NULL; 125 | fmt_buf_size = 0; 126 | } 127 | -------------------------------------------------------------------------------- /src/log.h: -------------------------------------------------------------------------------- 1 | #ifndef STATSRELAY_LOG_H 2 | #define STATSRELAY_LOG_H 3 | 4 | #include 5 | #include 6 | 7 | enum statsrelay_log_level { 8 | STATSRELAY_LOG_DEBUG = 10, 9 | STATSRELAY_LOG_INFO = 20, 10 | STATSRELAY_LOG_WARN = 30, 11 | STATSRELAY_LOG_ERROR = 40 12 | }; 13 | 14 | // set verbose logging, i.e. send logs to stderr 15 | void stats_log_verbose(bool verbose); 16 | 17 | void stats_set_log_level(enum statsrelay_log_level level); 18 | 19 | // variadic log function 20 | void stats_vlog(const char *prefix, const char *format, va_list ap); 21 | 22 | // log a message 23 | void stats_log(const char *format, ...); 24 | 25 | // log a debug message 26 | void stats_debug_log(const char *format, ...); 27 | 28 | // log an error message 29 | void stats_error_log(const char *format, ...); 30 | 31 | // finish logging; this ensures that the internally allocated buffer is freed; 32 | // it can safely be called multiple times 33 | void stats_log_end(void); 34 | 35 | #endif 36 | -------------------------------------------------------------------------------- /src/main.c: -------------------------------------------------------------------------------- 1 | #include "config.h" 2 | #include "protocol.h" 3 | #include "tcpserver.h" 4 | #include "udpserver.h" 5 | #include "server.h" 6 | #include "stats.h" 7 | #include "log.h" 8 | #include "validate.h" 9 | #include "yaml_config.h" 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | struct server_collection servers; 22 | 23 | static struct option long_options[] = { 24 | {"config", required_argument, NULL, 'c'}, 25 | {"check-config", required_argument, NULL, 't'}, 26 | {"verbose", no_argument, NULL, 'v'}, 27 | {"version", no_argument, NULL, 'V'}, 28 | {"log-level", required_argument, NULL, 'l'}, 29 | {"help", no_argument, NULL, 'h'}, 30 | }; 31 | 32 | static void graceful_shutdown(struct ev_loop *loop, ev_signal *w, int revents) { 33 | stats_log("Received signal, shutting down."); 34 | destroy_server_collection(&servers); 35 | ev_break(loop, EVBREAK_ALL); 36 | } 37 | 38 | static void reload_config(struct ev_loop *loop, ev_signal *w, int revents) { 39 | stats_log("Received SIGHUP, reloading."); 40 | if (server != NULL) { 41 | stats_server_reload(server); 42 | } 43 | } 44 | 45 | static char* to_lower(const char *input) { 46 | char *output = strdup(input); 47 | for (int i = 0; output[i] != '\0'; i++) { 48 | output[i] = tolower(output[i]); 49 | } 50 | return output; 51 | } 52 | 53 | static struct config *load_config(const char *filename) { 54 | FILE *file_handle = fopen(filename, "r"); 55 | if (file_handle == NULL) { 56 | stats_error_log("failed to open file %s", servers.config_file); 57 | return NULL; 58 | } 59 | struct config *cfg = parse_config(file_handle); 60 | fclose(file_handle); 61 | return cfg; 62 | } 63 | 64 | 65 | static void print_help(const char *argv0) { 66 | printf("Usage: %s [options]\n" 67 | " -h, --help Display this message\n" 68 | " -v, --verbose Write log messages to stderr in addition to syslog\n" 69 | " syslog\n" 70 | " -l, --log-level Set the logging level to DEBUG, INFO, WARN, or ERROR\n" 71 | " (default: INFO)\n" 72 | " -c, --config=filename Use the given hashring config file\n" 73 | " (default: %s)\n" 74 | " -t, --check-config=filename Check the config syntax\n" 75 | " (default: %s)\n" 76 | " --version Print the version\n", 77 | argv0, 78 | default_config, 79 | default_config); 80 | } 81 | 82 | int main(int argc, char **argv) { 83 | ev_signal sigint_watcher, sigterm_watcher, sighup_watcher; 84 | char *lower; 85 | int8_t c = 0; 86 | bool just_check_config = false; 87 | struct config *cfg = NULL; 88 | servers.initialized = false; 89 | 90 | stats_set_log_level(STATSRELAY_LOG_INFO); // set default value 91 | while (c != -1) { 92 | c = (int8_t)getopt_long(argc, argv, "t:c:l:vh", long_options, NULL); 93 | switch (c) { 94 | case -1: 95 | break; 96 | case 0: 97 | case 'h': 98 | print_help(argv[0]); 99 | return 1; 100 | case 'v': 101 | stats_log_verbose(true); 102 | break; 103 | case 'V': 104 | puts(PACKAGE_STRING); 105 | return 0; 106 | case 'l': 107 | lower = to_lower(optarg); 108 | if (lower == NULL) { 109 | fprintf(stderr, "main: unable to allocate memory\n"); 110 | goto err; 111 | } 112 | if (strcmp(lower, "debug") == 0) { 113 | stats_set_log_level(STATSRELAY_LOG_DEBUG); 114 | stats_log_verbose(true); 115 | } else if (strcmp(lower, "warn") == 0) { 116 | stats_set_log_level(STATSRELAY_LOG_WARN); 117 | } else if (strcmp(lower, "error") == 0) { 118 | stats_set_log_level(STATSRELAY_LOG_ERROR); 119 | } 120 | free(lower); 121 | break; 122 | case 'c': 123 | init_server_collection(&servers, optarg); 124 | break; 125 | case 't': 126 | init_server_collection(&servers, optarg); 127 | just_check_config = true; 128 | break; 129 | default: 130 | fprintf(stderr, "%s: Unknown argument %c\n", argv[0], c); 131 | goto err; 132 | } 133 | } 134 | stats_log(PACKAGE_STRING); 135 | 136 | if (!servers.initialized) { 137 | init_server_collection(&servers, default_config); 138 | } 139 | 140 | cfg = load_config(servers.config_file); 141 | if (cfg == NULL) { 142 | stats_error_log("failed to parse config"); 143 | goto err; 144 | } 145 | if (just_check_config) { 146 | goto success; 147 | } 148 | bool worked = connect_server_collection(&servers, cfg); 149 | if (!worked) { 150 | goto err; 151 | } 152 | 153 | struct ev_loop *loop = ev_default_loop(0); 154 | ev_signal_init(&sigint_watcher, graceful_shutdown, SIGINT); 155 | ev_signal_start(loop, &sigint_watcher); 156 | 157 | ev_signal_init(&sigterm_watcher, graceful_shutdown, SIGTERM); 158 | ev_signal_start(loop, &sigterm_watcher); 159 | 160 | ev_signal_init(&sighup_watcher, reload_config, SIGHUP); 161 | ev_signal_start(loop, &sighup_watcher); 162 | 163 | stats_log("main: Starting event loop"); 164 | ev_run(loop, 0); 165 | 166 | success: 167 | destroy_server_collection(&servers); 168 | destroy_config(cfg); 169 | stats_log_end(); 170 | return 0; 171 | 172 | err: 173 | destroy_server_collection(&servers); 174 | destroy_config(cfg); 175 | stats_log_end(); 176 | return 1; 177 | } 178 | -------------------------------------------------------------------------------- /src/protocol.c: -------------------------------------------------------------------------------- 1 | #include "protocol.h" 2 | 3 | #include 4 | 5 | static size_t simple_parse(const char *instr, size_t inlen, const char needle) { 6 | if (instr == 0 || inlen == 0) { 7 | return 0; 8 | } 9 | const char *p = memchr(instr, needle, inlen); 10 | if (p == NULL) { 11 | return 0; 12 | } 13 | return p - instr; 14 | } 15 | 16 | size_t protocol_parser_carbon(const char *instr, size_t inlen) { 17 | return simple_parse(instr, inlen, ' '); 18 | } 19 | 20 | size_t protocol_parser_statsd(const char *instr, size_t inlen) { 21 | return simple_parse(instr, inlen, ':'); 22 | } 23 | -------------------------------------------------------------------------------- /src/protocol.h: -------------------------------------------------------------------------------- 1 | #ifndef STATSRELAY_PROTOCOL_H 2 | #define STATSRELAY_PROTOCOL_H 3 | 4 | #include 5 | 6 | // This header file abstracts the protocol parsing logic. The signature for a 7 | // protocol parser is like: 8 | // 9 | // size_t parser(const char *instr, const size_t inlen); 10 | // 11 | // The first two paramaters, instr and inlen specify a const pointer to a 12 | // character buffer, and that buffer's size. The function parser will then 13 | // parse this string and find the part of the string that refers to the hash 14 | // key. 15 | // 16 | // Since all of the protocols supported (statsd and carbon) have the key at the 17 | // start of the string, a simplified interace is supported. The return value of 18 | // the parser is the number of bytes that represent the key. 19 | // 20 | // On failure, 0 is returned; 21 | 22 | typedef size_t (*protocol_parser_t)(const char *, size_t); 23 | 24 | size_t protocol_parser_carbon(const char *, size_t); 25 | size_t protocol_parser_statsd(const char *, size_t); 26 | 27 | #endif // STATSRELAY_PROTOCOL_H 28 | -------------------------------------------------------------------------------- /src/server.c: -------------------------------------------------------------------------------- 1 | #include "./server.h" 2 | 3 | #include "./log.h" 4 | 5 | #include 6 | #include 7 | 8 | static void init_server(struct server *server) { 9 | server->enabled = false; 10 | server->server = NULL; 11 | server->ts = NULL; 12 | server->us = NULL; 13 | } 14 | 15 | static bool connect_server(struct server *server, 16 | struct proto_config *config, 17 | protocol_parser_t parser, 18 | validate_line_validator_t validator, 19 | const char *name) { 20 | if (config->ring->size == 0) { 21 | stats_log("%s has no backends, skipping", name); 22 | return false; 23 | } 24 | 25 | struct ev_loop *loop = ev_default_loop(0); 26 | 27 | server->server = stats_server_create( 28 | loop, config, parser, validator); 29 | 30 | server->enabled = true; 31 | 32 | if (server->server == NULL) { 33 | stats_error_log("main: Unable to create stats_server"); 34 | return false; 35 | } 36 | server->ts = tcpserver_create(loop, server->server); 37 | if (server->ts == NULL) { 38 | stats_error_log("failed to create tcpserver"); 39 | return false; 40 | } 41 | 42 | server->us = udpserver_create(loop, server->server); 43 | if (server->us == NULL) { 44 | stats_error_log("failed to create udpserver"); 45 | return false; 46 | } 47 | 48 | if (tcpserver_bind(server->ts, config->bind, stats_connection, stats_recv) != 0) { 49 | stats_error_log("unable to bind tcp %s", config->bind); 50 | return false; 51 | } 52 | if (udpserver_bind(server->us, config->bind, stats_udp_recv) != 0) { 53 | stats_error_log("unable to bind udp %s", config->bind); 54 | return false; 55 | } 56 | return true; 57 | } 58 | 59 | static void destroy_server(struct server *server) { 60 | if (!server->enabled) { 61 | return; 62 | } 63 | if (server->ts != NULL) { 64 | tcpserver_destroy(server->ts); 65 | } 66 | if (server->us != NULL) { 67 | udpserver_destroy(server->us); 68 | } 69 | if (server->server != NULL) { 70 | stats_server_destroy(server->server); 71 | } 72 | } 73 | 74 | void init_server_collection(struct server_collection *server_collection, 75 | const char *filename) { 76 | server_collection->initialized = true; 77 | server_collection->config_file = strdup(filename); 78 | init_server(&server_collection->carbon_server); 79 | init_server(&server_collection->statsd_server); 80 | } 81 | 82 | bool connect_server_collection(struct server_collection *server_collection, 83 | struct config *config) { 84 | bool enabled_any = false; 85 | enabled_any |= connect_server(&server_collection->carbon_server, 86 | &config->carbon_config, 87 | protocol_parser_carbon, 88 | validate_carbon, 89 | "carbon"); 90 | enabled_any |= connect_server(&server_collection->statsd_server, 91 | &config->statsd_config, 92 | protocol_parser_statsd, 93 | validate_statsd, 94 | "statsd"); 95 | if (!enabled_any) { 96 | stats_error_log("failed to enable any backends"); 97 | } 98 | return enabled_any; 99 | } 100 | 101 | void destroy_server_collection(struct server_collection *server_collection) { 102 | if (server_collection->initialized) { 103 | free(server_collection->config_file); 104 | destroy_server(&server_collection->carbon_server); 105 | destroy_server(&server_collection->statsd_server); 106 | server_collection->initialized = false; 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /src/server.h: -------------------------------------------------------------------------------- 1 | #ifndef STATSRELAY_SERVER_H 2 | #define STATSRELAY_SERVER_H 3 | 4 | #include "./stats.h" 5 | #include "./tcpserver.h" 6 | #include "./udpserver.h" 7 | 8 | #include 9 | 10 | struct server { 11 | bool enabled; 12 | stats_server_t *server; 13 | tcpserver_t *ts; 14 | udpserver_t *us; 15 | struct ev_loop *loop; 16 | }; 17 | 18 | struct server_collection { 19 | bool initialized; 20 | char *config_file; 21 | struct server statsd_server; 22 | struct server carbon_server; 23 | }; 24 | 25 | void init_server_collection(struct server_collection *server_collection, 26 | const char *filename); 27 | 28 | bool connect_server_collection(struct server_collection *server_collection, 29 | struct config *config); 30 | 31 | void destroy_server_collection(struct server_collection *server_collection); 32 | 33 | #endif // STATSRELAY_SERVER_H 34 | -------------------------------------------------------------------------------- /src/stathasher.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include "./hashring.h" 8 | #include "./yaml_config.h" 9 | 10 | static struct option long_options[] = { 11 | {"config", required_argument, NULL, 'c'}, 12 | {"help", no_argument, NULL, 'h'}, 13 | }; 14 | 15 | static void* my_strdup(const char *str, void *unused_data) { 16 | return strdup(str); 17 | } 18 | 19 | static void print_help(const char *argv0) { 20 | printf("Usage: %s [-h] [-c /path/to/config.yaml]", argv0); 21 | } 22 | 23 | int main(int argc, char **argv) { 24 | char *config_name = (char *) default_config; 25 | int8_t c = 0; 26 | while (c != -1) { 27 | c = (int8_t)getopt_long(argc, argv, "c:h", long_options, NULL); 28 | switch (c) { 29 | case -1: 30 | break; 31 | case 0: 32 | case 'h': 33 | print_help(argv[0]); 34 | return 0; 35 | case 'c': 36 | config_name = optarg; 37 | break; 38 | default: 39 | printf("%s: Unknown argument %c\n", argv[0], c); 40 | return 1; 41 | } 42 | } 43 | if (optind != 1 && optind != 3) { 44 | printf("%s: unexpected command optoins\n", argv[0]); 45 | return 1; 46 | } 47 | 48 | FILE *config_file = fopen(config_name, "r"); 49 | if (config_file == NULL) { 50 | fprintf(stderr, "failed to open %s\n", config_name); 51 | return 1; 52 | } 53 | struct config *app_cfg = parse_config(config_file); 54 | 55 | fclose(config_file); 56 | if (app_cfg == NULL) { 57 | fprintf(stderr, "failed to parse config %s\n", config_name); 58 | return 1; 59 | } 60 | 61 | hashring_t carbon_ring = NULL, statsd_ring = NULL; 62 | 63 | if (app_cfg->carbon_config.initialized) { 64 | carbon_ring = hashring_load_from_config( 65 | &app_cfg->carbon_config, NULL, my_strdup, free); 66 | } 67 | if (app_cfg->statsd_config.initialized) { 68 | statsd_ring = hashring_load_from_config( 69 | &app_cfg->statsd_config, NULL, my_strdup, free); 70 | } 71 | destroy_config(app_cfg); 72 | 73 | uint32_t shard; 74 | char *choice = NULL; 75 | char *line = NULL; 76 | size_t len; 77 | ssize_t bytes_read; 78 | while ((bytes_read = getline(&line, &len, stdin)) != -1) { 79 | // trim whitespace 80 | for (ssize_t i = 0; i < bytes_read; i++) { 81 | if (isspace(line[i])) { 82 | line[i] = '\0'; 83 | break; 84 | } 85 | } 86 | printf("key=%s", line); 87 | if (carbon_ring != NULL) { 88 | choice = hashring_choose(carbon_ring, line, &shard); 89 | if (choice != NULL) { 90 | printf(" carbon=%s carbon_shard=%d", choice, shard); 91 | } 92 | } 93 | if (statsd_ring != NULL) { 94 | choice = hashring_choose(statsd_ring, line, &shard); 95 | if (choice != NULL) { 96 | printf(" statsd=%s statsd_shard=%d", choice, shard); 97 | } 98 | } 99 | putchar('\n'); 100 | fflush(stdout); 101 | } 102 | free(line); 103 | hashring_dealloc(carbon_ring); 104 | hashring_dealloc(statsd_ring); 105 | return 0; 106 | } 107 | -------------------------------------------------------------------------------- /src/stats.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include "./hashring.h" 16 | #include "./buffer.h" 17 | #include "./log.h" 18 | #include "./stats.h" 19 | #include "./tcpclient.h" 20 | #include "./validate.h" 21 | 22 | #define MAX_UDP_LENGTH 65536 23 | 24 | typedef struct { 25 | tcpclient_t client; 26 | char *key; 27 | uint64_t bytes_queued; 28 | uint64_t bytes_sent; 29 | uint64_t relayed_lines; 30 | uint64_t dropped_lines; 31 | int failing; 32 | } stats_backend_t; 33 | 34 | struct stats_server_t { 35 | struct ev_loop *loop; 36 | 37 | uint64_t bytes_recv_udp; 38 | uint64_t bytes_recv_tcp; 39 | uint64_t total_connections; 40 | uint64_t malformed_lines; 41 | time_t last_reload; 42 | 43 | struct proto_config *config; 44 | size_t num_backends; 45 | stats_backend_t **backend_list; 46 | 47 | hashring_t ring; 48 | protocol_parser_t parser; 49 | validate_line_validator_t validator; 50 | }; 51 | 52 | typedef struct { 53 | stats_server_t *server; 54 | buffer_t buffer; 55 | int sd; 56 | } stats_session_t; 57 | 58 | // callback after bytes are sent 59 | static int stats_sent(void *tcpclient, 60 | enum tcpclient_event event, 61 | void *context, 62 | char *data, 63 | size_t len) { 64 | stats_backend_t *backend = (stats_backend_t *) context; 65 | backend->bytes_sent += len; 66 | return 0; 67 | } 68 | 69 | // Add a backend to the backend list. 70 | static int add_backend(stats_server_t *server, stats_backend_t *backend) { 71 | stats_backend_t **new_backends = realloc( 72 | server->backend_list, sizeof(stats_backend_t *) * (server->num_backends + 1)); 73 | if (new_backends == NULL) { 74 | stats_log("add_backend: failed to realloc backends list"); 75 | return 1; 76 | } 77 | server->backend_list = new_backends; 78 | server->backend_list[server->num_backends++] = backend; 79 | return 0; 80 | } 81 | 82 | // Find a backend in the backend list; this is used so we don't create 83 | // duplicate backends. This is linear with the number of actual 84 | // backends in the file which should be fine for any reasonable 85 | // configuration (say, less than 10,000 backend statsite or carbon 86 | // servers). Also note that while this is linear, it only happens 87 | // during statsrelay initialization, not when running. 88 | static stats_backend_t *find_backend(stats_server_t *server, const char *key) { 89 | for (size_t i = 0; i < server->num_backends; i++) { 90 | stats_backend_t *backend = server->backend_list[i]; 91 | if (strcmp(backend->key, key) == 0) { 92 | return backend; 93 | } 94 | } 95 | return NULL; 96 | } 97 | 98 | // Make a backend, returning it from the backend list if it's already 99 | // been created. 100 | static void* make_backend(const char *host_and_port, void *data) { 101 | stats_backend_t *backend = NULL; 102 | char *full_key = NULL; 103 | 104 | // First we normalize so that the key is in the format 105 | // host:port:protocol 106 | char *host = NULL; 107 | char *port = NULL; 108 | char *protocol = NULL; 109 | 110 | const char *colon1 = strchr(host_and_port, ':'); 111 | if (colon1 == NULL) { 112 | stats_log("failed to parse host/port in \"%s\"", host_and_port); 113 | goto make_err; 114 | } 115 | host = strndup(host_and_port, colon1 - host_and_port); 116 | if (host == NULL) { 117 | stats_log("stats: alloc error in host"); 118 | goto make_err; 119 | } 120 | const char *colon2 = strchr(colon1 + 1, ':'); 121 | if (colon2 == NULL) { 122 | port = strdup(colon1 + 1); 123 | protocol = strdup("tcp"); // use TCP by default 124 | } else { 125 | port = strndup(colon1 + 1, colon2 - colon1 - 1); 126 | protocol = strdup(colon2 + 1); 127 | } 128 | if (port == NULL || protocol == NULL) { 129 | stats_log("stats: alloc error in port/protocol"); 130 | goto make_err; 131 | } 132 | 133 | if (colon2 == NULL) { 134 | const size_t hp_len = strlen(host_and_port); 135 | const size_t space_needed = hp_len + strlen(protocol) + 2; 136 | full_key = malloc(space_needed); 137 | if (full_key != NULL && snprintf(full_key, space_needed, "%s:%s", host_and_port, protocol) < 0) { 138 | stats_error_log("failed to snprintf"); 139 | goto make_err; 140 | } 141 | } else { 142 | full_key = strdup(host_and_port); 143 | } 144 | if (full_key == NULL) { 145 | stats_error_log("failed to create backend key"); 146 | goto make_err; 147 | } 148 | 149 | // Find the key in our list of backends 150 | stats_server_t *server = (stats_server_t *) data; 151 | backend = find_backend(server, full_key); 152 | if (backend != NULL) { 153 | free(host); 154 | free(port); 155 | free(protocol); 156 | free(full_key); 157 | return backend; 158 | } 159 | backend = malloc(sizeof(stats_backend_t)); 160 | if (backend == NULL) { 161 | stats_log("stats: alloc error creating backend"); 162 | goto make_err; 163 | } 164 | 165 | if (tcpclient_init(&backend->client, 166 | server->loop, 167 | backend, 168 | server->config, 169 | host, 170 | port, 171 | protocol)) { 172 | stats_log("stats: failed to tcpclient_init"); 173 | goto make_err; 174 | } 175 | 176 | if (tcpclient_connect(&backend->client)) { 177 | stats_log("stats: failed to connect tcpclient"); 178 | goto make_err; 179 | } 180 | backend->bytes_queued = 0; 181 | backend->bytes_sent = 0; 182 | backend->relayed_lines = 0; 183 | backend->dropped_lines = 0; 184 | backend->failing = 0; 185 | backend->key = full_key; 186 | tcpclient_set_sent_callback(&backend->client, stats_sent); 187 | add_backend(server, backend); 188 | stats_debug_log("initialized new backend %s", backend->key); 189 | 190 | free(host); 191 | free(port); 192 | free(protocol); 193 | return backend; 194 | 195 | make_err: 196 | free(host); 197 | free(port); 198 | free(protocol); 199 | free(full_key); 200 | return NULL; 201 | } 202 | 203 | 204 | static void kill_backend(void *data) { 205 | stats_backend_t *backend = (stats_backend_t *) data; 206 | if (backend->key != NULL) { 207 | stats_debug_log("killing backend %s", backend->key); 208 | free(backend->key); 209 | } 210 | tcpclient_destroy(&backend->client, 1); 211 | free(backend); 212 | } 213 | 214 | stats_server_t *stats_server_create(struct ev_loop *loop, 215 | struct proto_config *config, 216 | protocol_parser_t parser, 217 | validate_line_validator_t validator) { 218 | stats_server_t *server; 219 | server = malloc(sizeof(stats_server_t)); 220 | if (server == NULL) { 221 | stats_log("stats: Unable to allocate memory"); 222 | return NULL; 223 | } 224 | 225 | server->loop = loop; 226 | server->num_backends = 0; 227 | server->backend_list = NULL; 228 | server->config = config; 229 | server->ring = hashring_load_from_config( 230 | config, server, make_backend, kill_backend); 231 | if (server->ring == NULL) { 232 | stats_error_log("hashring_load_from_config failed"); 233 | goto server_create_err; 234 | } 235 | 236 | server->bytes_recv_udp = 0; 237 | server->bytes_recv_tcp = 0; 238 | server->malformed_lines = 0; 239 | server->total_connections = 0; 240 | server->last_reload = 0; 241 | 242 | server->parser = parser; 243 | server->validator = validator; 244 | 245 | stats_debug_log("initialized server with %d backends, hashring size = %d", 246 | server->num_backends, hashring_size(server->ring)); 247 | 248 | return server; 249 | 250 | server_create_err: 251 | if (server != NULL) { 252 | hashring_dealloc(server->ring); 253 | free(server); 254 | } 255 | return NULL; 256 | } 257 | 258 | size_t stats_num_backends(stats_server_t *server) { 259 | return server->num_backends; 260 | } 261 | 262 | void stats_server_reload(stats_server_t *server) { 263 | hashring_dealloc(server->ring); 264 | 265 | free(server->backend_list); 266 | server->num_backends = 0; 267 | server->backend_list = NULL; 268 | 269 | server->last_reload = time(NULL); 270 | 271 | // FIXME 272 | } 273 | 274 | void *stats_connection(int sd, void *ctx) { 275 | stats_session_t *session; 276 | 277 | stats_debug_log("stats: accepted client connection on fd %d", sd); 278 | session = (stats_session_t *) malloc(sizeof(stats_session_t)); 279 | if (session == NULL) { 280 | stats_log("stats: Unable to allocate memory"); 281 | return NULL; 282 | } 283 | 284 | if (buffer_init(&session->buffer) != 0) { 285 | stats_log("stats: Unable to initialize buffer"); 286 | free(session); 287 | return NULL; 288 | } 289 | 290 | session->server = (stats_server_t *) ctx; 291 | session->server->total_connections++; 292 | session->sd = sd; 293 | return (void *) session; 294 | } 295 | 296 | static int stats_relay_line(const char *line, size_t len, stats_server_t *ss) { 297 | if (ss->config->enable_validation && ss->validator != NULL) { 298 | if (ss->validator(line, len) != 0) { 299 | return 1; 300 | } 301 | } 302 | 303 | static char key_buffer[8192]; 304 | size_t key_len = ss->parser(line, len); 305 | if (key_len == 0) { 306 | ss->malformed_lines++; 307 | stats_log("stats: failed to find key: \"%s\"", line); 308 | return 1; 309 | } 310 | memcpy(key_buffer, line, key_len); 311 | key_buffer[key_len] = '\0'; 312 | 313 | stats_backend_t *backend = hashring_choose(ss->ring, key_buffer, NULL); 314 | 315 | if (backend == NULL) { 316 | return 1; 317 | } 318 | 319 | if (tcpclient_sendall(&backend->client, line, len + 1) != 0) { 320 | backend->dropped_lines++; 321 | if (backend->failing == 0) { 322 | stats_log("stats: Error sending to backend %s", backend->key); 323 | backend->failing = 1; 324 | } 325 | return 2; 326 | } else { 327 | backend->failing = 0; 328 | } 329 | 330 | backend->bytes_queued += len + 1; 331 | backend->relayed_lines++; 332 | 333 | return 0; 334 | } 335 | 336 | void stats_send_statistics(stats_session_t *session) { 337 | stats_backend_t *backend; 338 | ssize_t bytes_sent; 339 | 340 | // TODO: this only needs to be allocated once, not every time we send 341 | // statistics 342 | buffer_t *response = create_buffer(MAX_UDP_LENGTH); 343 | if (response == NULL) { 344 | stats_log("failed to allocate send_statistics buffer"); 345 | return; 346 | } 347 | 348 | buffer_produced(response, 349 | snprintf((char *)buffer_tail(response), buffer_spacecount(response), 350 | "global bytes_recv_udp gauge %" PRIu64 "\n", 351 | session->server->bytes_recv_udp)); 352 | 353 | buffer_produced(response, 354 | snprintf((char *)buffer_tail(response), buffer_spacecount(response), 355 | "global bytes_recv_tcp gauge %" PRIu64 "\n", 356 | session->server->bytes_recv_tcp)); 357 | 358 | buffer_produced(response, 359 | snprintf((char *)buffer_tail(response), buffer_spacecount(response), 360 | "global total_connections gauge %" PRIu64 "\n", 361 | session->server->total_connections)); 362 | 363 | buffer_produced(response, 364 | snprintf((char *)buffer_tail(response), buffer_spacecount(response), 365 | "global last_reload timestamp %" PRIu64 "\n", 366 | session->server->last_reload)); 367 | 368 | buffer_produced(response, 369 | snprintf((char *)buffer_tail(response), buffer_spacecount(response), 370 | "global malformed_lines gauge %" PRIu64 "\n", 371 | session->server->malformed_lines)); 372 | 373 | for (size_t i = 0; i < session->server->num_backends; i++) { 374 | backend = session->server->backend_list[i]; 375 | 376 | buffer_produced(response, 377 | snprintf((char *)buffer_tail(response), buffer_spacecount(response), 378 | "backend:%s bytes_queued gauge %" PRIu64 "\n", 379 | backend->key, backend->bytes_queued)); 380 | 381 | buffer_produced(response, 382 | snprintf((char *)buffer_tail(response), buffer_spacecount(response), 383 | "backend:%s bytes_sent gauge %" PRIu64 "\n", 384 | backend->key, backend->bytes_sent)); 385 | 386 | buffer_produced(response, 387 | snprintf((char *)buffer_tail(response), buffer_spacecount(response), 388 | "backend:%s relayed_lines gauge %" PRIu64 "\n", 389 | backend->key, backend->relayed_lines)); 390 | 391 | buffer_produced(response, 392 | snprintf((char *)buffer_tail(response), buffer_spacecount(response), 393 | "backend:%s dropped_lines gauge %" PRIu64 "\n", 394 | backend->key, backend->dropped_lines)); 395 | 396 | buffer_produced(response, 397 | snprintf((char *)buffer_tail(response), buffer_spacecount(response), 398 | "backend:%s failing boolean %i\n", 399 | backend->key, backend->failing)); 400 | } 401 | 402 | buffer_produced(response, 403 | snprintf((char *)buffer_tail(response), buffer_spacecount(response), "\n")); 404 | 405 | while (buffer_datacount(response) > 0) { 406 | bytes_sent = send(session->sd, buffer_head(response), buffer_datacount(response), 0); 407 | if (bytes_sent < 0) { 408 | stats_log("stats: Error sending status response: %s", strerror(errno)); 409 | break; 410 | } 411 | 412 | if (bytes_sent == 0) { 413 | stats_log("stats: Error sending status response: Client closed connection"); 414 | break; 415 | } 416 | 417 | buffer_consume(response, bytes_sent); 418 | } 419 | delete_buffer(response); 420 | } 421 | 422 | static int stats_process_lines(stats_session_t *session) { 423 | char *head, *tail; 424 | size_t len; 425 | 426 | static char line_buffer[MAX_UDP_LENGTH + 2]; 427 | 428 | while (1) { 429 | size_t datasize = buffer_datacount(&session->buffer); 430 | if (datasize == 0) { 431 | break; 432 | } 433 | head = (char *)buffer_head(&session->buffer); 434 | tail = memchr(head, '\n', datasize); 435 | if (tail == NULL) { 436 | break; 437 | } 438 | len = tail - head; 439 | memcpy(line_buffer, head, len); 440 | memcpy(line_buffer + len, "\n\0", 2); 441 | 442 | if (len == 6 && strcmp(line_buffer, "status\n") == 0) { 443 | stats_send_statistics(session); 444 | } else if (stats_relay_line(line_buffer, len, session->server) != 0) { 445 | return 1; 446 | } 447 | buffer_consume(&session->buffer, len + 1); // Add 1 to include the '\n' 448 | } 449 | 450 | return 0; 451 | } 452 | 453 | void stats_session_destroy(stats_session_t *session) { 454 | buffer_destroy(&session->buffer); 455 | free(session); 456 | } 457 | 458 | int stats_recv(int sd, void *data, void *ctx) { 459 | stats_session_t *session = (stats_session_t *)ctx; 460 | 461 | ssize_t bytes_read; 462 | size_t space; 463 | 464 | // First we try to realign the buffer (memmove so that head 465 | // and ptr match) If that fails, we double the size of the 466 | // buffer 467 | space = buffer_spacecount(&session->buffer); 468 | if (space == 0) { 469 | buffer_realign(&session->buffer); 470 | space = buffer_spacecount(&session->buffer); 471 | if (space == 0) { 472 | if (buffer_expand(&session->buffer) != 0) { 473 | stats_log("stats: Unable to expand buffer, aborting"); 474 | goto stats_recv_err; 475 | } 476 | space = buffer_spacecount(&session->buffer); 477 | } 478 | } 479 | 480 | bytes_read = recv(sd, buffer_tail(&session->buffer), space, 0); 481 | if (bytes_read < 0) { 482 | stats_log("stats: Error receiving from socket: %s", strerror(errno)); 483 | goto stats_recv_err; 484 | } else if (bytes_read == 0) { 485 | stats_debug_log("stats: client from fd %d closed connection", sd); 486 | goto stats_recv_err; 487 | } else { 488 | stats_debug_log("stats: received %zd bytes from tcp client fd %d", bytes_read, sd); 489 | } 490 | 491 | session->server->bytes_recv_tcp += bytes_read; 492 | 493 | if (buffer_produced(&session->buffer, bytes_read) != 0) { 494 | stats_log("stats: Unable to produce buffer by %i bytes, aborting", bytes_read); 495 | goto stats_recv_err; 496 | } 497 | 498 | if (stats_process_lines(session) != 0) { 499 | stats_log("stats: Invalid line processed, closing connection"); 500 | goto stats_recv_err; 501 | } 502 | 503 | return 0; 504 | 505 | stats_recv_err: 506 | stats_session_destroy(session); 507 | return 1; 508 | } 509 | 510 | // TODO: refactor this whole method to share more code with the tcp receiver: 511 | // * this shouldn't have to allocate a new buffer -- it should be on the ss 512 | // * the line processing stuff should use stats_process_lines() 513 | int stats_udp_recv(int sd, void *data) { 514 | stats_server_t *ss = (stats_server_t *)data; 515 | ssize_t bytes_read; 516 | char *head, *tail; 517 | 518 | static char buffer[MAX_UDP_LENGTH]; 519 | static char line_buffer[MAX_UDP_LENGTH + 2]; 520 | 521 | bytes_read = read(sd, buffer, MAX_UDP_LENGTH); 522 | 523 | if (bytes_read == 0) { 524 | stats_error_log("stats: Unexpectedly received zero-length UDP payload."); 525 | goto udp_recv_err; 526 | } else if (bytes_read < 0) { 527 | if (errno == EAGAIN) { 528 | stats_error_log("stats: interrupted during recvfrom"); 529 | goto udp_recv_err; 530 | } else { 531 | stats_error_log("stats: Error calling recvfrom: %s", strerror(errno)); 532 | goto udp_recv_err; 533 | } 534 | } else { 535 | stats_debug_log("stats: received %zd bytes from udp fd %d", bytes_read, sd); 536 | } 537 | 538 | ss->bytes_recv_udp += bytes_read; 539 | 540 | size_t line_len; 541 | size_t offset = 0; 542 | while (offset < bytes_read) { 543 | head = (char *) buffer + offset; 544 | if ((tail = memchr(head, '\n', bytes_read - offset)) == NULL) { 545 | tail = buffer + bytes_read; 546 | } 547 | 548 | line_len = tail - head; 549 | memcpy(line_buffer, head, line_len); 550 | memcpy(line_buffer + line_len, "\n\0", 2); 551 | 552 | if (stats_relay_line(line_buffer, line_len, ss) != 0) { 553 | goto udp_recv_err; 554 | } 555 | offset += line_len + 1; 556 | } 557 | return 0; 558 | 559 | udp_recv_err: 560 | return 1; 561 | } 562 | 563 | void stats_server_destroy(stats_server_t *server) { 564 | hashring_dealloc(server->ring); 565 | free(server->backend_list); 566 | server->num_backends = 0; 567 | free(server); 568 | } 569 | -------------------------------------------------------------------------------- /src/stats.h: -------------------------------------------------------------------------------- 1 | #ifndef STATSRELAY_STATS_H 2 | #define STATSRELAY_STATS_H 3 | 4 | #include 5 | #include 6 | 7 | #include "protocol.h" 8 | #include "validate.h" 9 | #include "yaml_config.h" 10 | 11 | typedef struct stats_server_t stats_server_t; 12 | 13 | stats_server_t *stats_server_create( 14 | struct ev_loop *loop, 15 | struct proto_config *config, 16 | protocol_parser_t parser, 17 | validate_line_validator_t validator); 18 | stats_server_t *server; 19 | 20 | size_t stats_num_backends(stats_server_t *server); 21 | 22 | void stats_server_reload(stats_server_t *server); 23 | 24 | void stats_server_destroy(stats_server_t *server); 25 | 26 | // ctx is a (void *) cast of the stats_server_t instance. 27 | void *stats_connection(int sd, void *ctx); 28 | 29 | int stats_recv(int sd, void *data, void *ctx); 30 | 31 | int stats_udp_recv(int sd, void *data); 32 | 33 | #endif // STATSRELAY_STATS_H 34 | -------------------------------------------------------------------------------- /src/stresstest.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | static struct option long_options[] = { 13 | {"port", required_argument, NULL, 'p'}, 14 | {"stat-names", required_argument, NULL, 's'}, 15 | {"help", no_argument, NULL, 'h'}, 16 | }; 17 | 18 | static void print_help(const char *progname) { 19 | printf("help\n"); 20 | } 21 | 22 | 23 | int main(int argc, char **argv) { 24 | uint16_t port = 0; 25 | const char *stat_names = NULL; 26 | int option_index = 0; 27 | int8_t c; 28 | while ((c = (int8_t)getopt_long(argc, argv, "p:s:h", long_options, &option_index)) != -1) { 29 | switch (c) { 30 | case 0: 31 | case 'h': 32 | print_help(argv[0]); 33 | return 1; 34 | case 'p': 35 | port = (uint16_t) strtol(optarg, NULL, 10); 36 | break; 37 | case 's': 38 | stat_names = strdup(optarg); 39 | if (stat_names == NULL) { 40 | perror("failed to strdup()"); 41 | goto err; 42 | } 43 | break; 44 | default: 45 | fprintf(stderr, "%s: Unknown argument %c", argv[0], c); 46 | goto err; 47 | } 48 | } 49 | if (stat_names == NULL) { 50 | fprintf(stderr, "missing -s option\n"); 51 | goto err; 52 | } else if (port == 0) { 53 | fprintf(stderr, "missing or invalid -p option\n"); 54 | goto err; 55 | } 56 | 57 | int sock = socket(AF_INET, SOCK_STREAM, 0); 58 | if (sock == -1) { 59 | perror("failed to socket()"); 60 | goto err; 61 | } 62 | struct sockaddr_in serv_addr; 63 | memset(&serv_addr, 0, sizeof(serv_addr)); 64 | serv_addr.sin_family = AF_INET; 65 | serv_addr.sin_port = htons(port); 66 | 67 | printf("connecting to port %hu\n", port); 68 | if (connect(sock, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { 69 | perror("failed to connect()"); 70 | goto err; 71 | } 72 | 73 | FILE *fp = fopen(stat_names, "r"); 74 | if (fp == NULL) { 75 | goto err; 76 | } 77 | 78 | 79 | struct timeval t0, t1, total; 80 | if (gettimeofday(&t0, NULL) == -1) { 81 | perror("gettimeofday()"); 82 | goto err; 83 | } 84 | char *line = NULL; 85 | size_t len = 0; 86 | ssize_t read; 87 | size_t lines_sent = 0; 88 | 89 | while ((read = getline(&line, &len, fp)) != -1) { 90 | bool need_realloc = false; 91 | while (read + 6 > len) { 92 | need_realloc = true; 93 | len <<= 1; 94 | } 95 | if (need_realloc) { 96 | void *np = realloc(line, len); 97 | if (np == NULL) { 98 | perror("failed to realloc()"); 99 | goto err; 100 | } 101 | line = np; 102 | } 103 | while (line[read] == '\n' || line[read] == '\0') { 104 | read--; 105 | } 106 | memcpy(line + read + 1, ":1|c\n\0", 6); 107 | size_t goal = strlen(line); 108 | size_t total_sent = 0; 109 | while (total_sent < goal) { 110 | ssize_t bytes_sent = send( 111 | sock, line + total_sent, goal - total_sent, 0); 112 | if (bytes_sent <= 0) { 113 | perror("failed to send()"); 114 | goto err; 115 | } 116 | total_sent += bytes_sent; 117 | } 118 | lines_sent++; 119 | } 120 | 121 | if (lines_sent == 0) { 122 | goto err; 123 | } else if (gettimeofday(&t1, NULL) == -1) { 124 | perror("gettimeofday()"); 125 | goto err; 126 | } 127 | 128 | timersub(&t1, &t0, &total); 129 | double total_micros = total.tv_sec * 1000000 + total.tv_usec; 130 | printf("sent %zd lines in %lu microseconds = %6.3f microseconds per line\n", 131 | lines_sent, 132 | (unsigned long) total_micros, 133 | total_micros / lines_sent); 134 | 135 | return 0; 136 | 137 | err: 138 | return 1; 139 | } 140 | -------------------------------------------------------------------------------- /src/tcpclient.c: -------------------------------------------------------------------------------- 1 | #include "tcpclient.h" 2 | #include "buffer.h" 3 | #include "log.h" 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | #include 20 | 21 | #define DEFAULT_BUFFER_SIZE (1<<16) 22 | 23 | static const char *tcpclient_state_name[] = { 24 | "INIT", "CONNECTING", "BACKOFF", "CONNECTED", "TERMINATED" 25 | }; 26 | 27 | static int tcpclient_default_callback(void *tc, enum tcpclient_event event, void *context, char *data, size_t len) { 28 | // default is to do nothing 29 | if (event == EVENT_RECV) { 30 | free(data); 31 | } 32 | return 0; 33 | } 34 | 35 | static void tcpclient_set_state(tcpclient_t *client, enum tcpclient_state state) { 36 | stats_log("tcpclient[%s]: State transition %s -> %s", 37 | client->name, 38 | tcpclient_state_name[client->state], 39 | tcpclient_state_name[state]); 40 | client->state = state; 41 | } 42 | 43 | static void tcpclient_connect_timeout(struct ev_loop *loop, struct ev_timer *watcher, int events) { 44 | tcpclient_t *client = (tcpclient_t *)watcher->data; 45 | if (client->connect_watcher.started) { 46 | ev_io_stop(loop, &client->connect_watcher.watcher); 47 | client->connect_watcher.started = false; 48 | } 49 | 50 | close(client->sd); 51 | stats_error_log("tcpclient[%s]: Connection timeout", client->name); 52 | client->last_error = time(NULL); 53 | tcpclient_set_state(client, STATE_BACKOFF); 54 | client->callback_error(client, EVENT_ERROR, client->callback_context, NULL, 0); 55 | } 56 | 57 | int tcpclient_init(tcpclient_t *client, 58 | struct ev_loop *loop, 59 | void *callback_context, 60 | struct proto_config *config, 61 | char *host, 62 | char *port, 63 | char *protocol) { 64 | size_t len; 65 | 66 | client->state = STATE_INIT; 67 | client->loop = loop; 68 | client->sd = -1; 69 | client->addr = NULL; 70 | client->last_error = 0; 71 | client->failing = 0; 72 | client->config = config; 73 | client->socktype = SOCK_DGRAM; 74 | 75 | if(host == NULL) { 76 | stats_error_log("tcpclient_init: host is NULL\n"); 77 | return 1; 78 | } 79 | len = strlen(host); 80 | if((client->host = calloc(1, len + 1)) == NULL) { 81 | stats_error_log("tcpclient[%s]: unable to allocate memory for host\n", client->host); 82 | return 1; 83 | } 84 | strncpy(client->host, host, len); 85 | 86 | if(port == NULL) { 87 | stats_error_log("tcpclient_init: port is NULL\n"); 88 | free(client->host); 89 | return 1; 90 | } 91 | len = strlen(port); 92 | if((client->port = calloc(1, len + 1)) == NULL) { 93 | stats_error_log("tcpclient[%s]: unable to allocate memory for port\n", client->host); 94 | free(client->host); 95 | return 1; 96 | } 97 | strncpy(client->port, port, len); 98 | 99 | if(protocol == NULL) { 100 | // Default to TCP if protocol is not set 101 | if((client->protocol = calloc(1, 4)) == NULL) { 102 | stats_error_log("tcpclient[%s]: unable to allocate memory for protocol\n", client->host); 103 | free(client->host); 104 | free(client->port); 105 | return 1; 106 | } 107 | strcpy(client->protocol, "tcp"); 108 | }else{ 109 | len = strlen(protocol); 110 | if((client->protocol = calloc(1, len + 1)) == NULL) { 111 | stats_error_log("tcpclient[%s]: unable to allocate memory for protocol\n", client->host); 112 | free(client->host); 113 | free(client->port); 114 | return 1; 115 | } 116 | strncpy(client->protocol, protocol, len); 117 | } 118 | 119 | strncpy(client->name, "UNRESOLVED", TCPCLIENT_NAME_LEN); 120 | 121 | client->callback_connect = &tcpclient_default_callback; 122 | client->callback_sent = &tcpclient_default_callback; 123 | client->callback_recv = &tcpclient_default_callback; 124 | client->callback_error = &tcpclient_default_callback; 125 | client->callback_context = callback_context; 126 | buffer_init(&client->send_queue); 127 | buffer_newsize(&client->send_queue, DEFAULT_BUFFER_SIZE); 128 | ev_timer_init(&client->timeout_watcher, 129 | tcpclient_connect_timeout, 130 | TCPCLIENT_CONNECT_TIMEOUT, 131 | 0); 132 | 133 | client->connect_watcher.started = false; 134 | client->read_watcher.started = false; 135 | client->write_watcher.started = false; 136 | return 0; 137 | } 138 | 139 | void tcpclient_set_sent_callback(tcpclient_t *client, tcpclient_callback callback) { 140 | client->callback_sent = callback; 141 | } 142 | 143 | static void tcpclient_read_event(struct ev_loop *loop, struct ev_io *watcher, int events) { 144 | tcpclient_t *client = (tcpclient_t *)watcher->data; 145 | ssize_t len; 146 | char *buf; 147 | 148 | if (!(events & EV_READ)) { 149 | return; 150 | } 151 | 152 | buf = malloc(TCPCLIENT_RECV_BUFFER); 153 | if (buf == NULL) { 154 | stats_error_log("tcpclient[%s]: Unable to allocate memory for receive buffer", client->name); 155 | return; 156 | } 157 | len = recv(client->sd, buf, TCPCLIENT_RECV_BUFFER, 0); 158 | if (len < 0) { 159 | stats_error_log("tcpclient[%s]: Error from recv: %s", client->name, strerror(errno)); 160 | if (client->read_watcher.started) { 161 | ev_io_stop(client->loop, &client->read_watcher.watcher); 162 | client->read_watcher.started = false; 163 | } 164 | if (client->write_watcher.started) { 165 | ev_io_stop(client->loop, &client->write_watcher.watcher); 166 | client->write_watcher.started = false; 167 | } 168 | close(client->sd); 169 | free(buf); 170 | tcpclient_set_state(client, STATE_BACKOFF); 171 | client->last_error = time(NULL); 172 | client->callback_error(client, EVENT_ERROR, client->callback_context, NULL, 0); 173 | return; 174 | } 175 | 176 | if (len == 0) { 177 | stats_error_log("tcpclient[%s]: Server closed connection", client->name); 178 | ev_io_stop(client->loop, &client->read_watcher.watcher); 179 | ev_io_stop(client->loop, &client->write_watcher.watcher); 180 | close(client->sd); 181 | free(buf); 182 | tcpclient_set_state(client, STATE_INIT); 183 | client->last_error = time(NULL); 184 | client->callback_error(client, EVENT_ERROR, client->callback_context, NULL, 0); 185 | return; 186 | } 187 | client->callback_recv(client, EVENT_RECV, client->callback_context, buf, len); 188 | 189 | } 190 | 191 | 192 | static void tcpclient_write_event(struct ev_loop *loop, struct ev_io *watcher, int events) { 193 | tcpclient_t *client = (tcpclient_t *)watcher->data; 194 | buffer_t *sendq; 195 | 196 | if (!(events & EV_WRITE)) { 197 | return; 198 | } 199 | 200 | sendq = &client->send_queue; 201 | ssize_t buf_len = buffer_datacount(sendq); 202 | if (buf_len > 0) { 203 | ssize_t send_len = send(client->sd, sendq->head, buf_len, 0); 204 | stats_debug_log("tcpclient: sent %zd of %zd bytes to backend client %s via fd %d", 205 | send_len, buf_len, client->name, client->sd); 206 | if (send_len < 0) { 207 | stats_error_log("tcpclient[%s]: Error from send: %s", client->name, strerror(errno)); 208 | ev_io_stop(client->loop, &client->write_watcher.watcher); 209 | ev_io_stop(client->loop, &client->read_watcher.watcher); 210 | client->last_error = time(NULL); 211 | tcpclient_set_state(client, STATE_BACKOFF); 212 | close(client->sd); 213 | client->callback_error(client, EVENT_ERROR, client->callback_context, NULL, 0); 214 | return; 215 | } else { 216 | client->callback_sent(client, EVENT_SENT, client->callback_context, sendq->head, (size_t) send_len); 217 | if (buffer_consume(sendq, send_len) != 0) { 218 | stats_error_log("tcpclient[%s]: Unable to consume send queue", client->name); 219 | return; 220 | } 221 | size_t qsize = buffer_datacount(&client->send_queue); 222 | if (client->failing && qsize < client->config->max_send_queue) { 223 | stats_log("tcpclient[%s]: client recovered from full queue, send queue is now %zd bytes", 224 | client->name, 225 | qsize); 226 | client->failing = 0; 227 | } 228 | if (qsize == 0) { 229 | ev_io_stop(client->loop, &client->write_watcher.watcher); 230 | client->write_watcher.started = false; 231 | } 232 | } 233 | } else { 234 | // No data left in the client's buffer, stop waiting 235 | // for write events. 236 | ev_io_stop(client->loop, &client->write_watcher.watcher); 237 | client->write_watcher.started = false; 238 | } 239 | } 240 | 241 | static void tcpclient_connected(struct ev_loop *loop, struct ev_io *watcher, int events) { 242 | tcpclient_t *client = (tcpclient_t *)watcher->data; 243 | int err; 244 | socklen_t len = sizeof(err); 245 | 246 | // Cancel timeout timer 247 | ev_timer_stop(loop, &client->timeout_watcher); 248 | ev_io_stop(loop, &client->connect_watcher.watcher); 249 | 250 | if (getsockopt(client->sd, SOL_SOCKET, SO_ERROR, &err, &len) != 0) { 251 | stats_error_log("tcpclient[%s]: Unable to get socket error state: %s", client->name, strerror(errno)); 252 | return; 253 | } 254 | 255 | if ((events & EV_ERROR) || err) { 256 | stats_error_log("tcpclient[%s]: Connect failed: %s", client->name, strerror(err)); 257 | close(client->sd); 258 | client->last_error = time(NULL); 259 | tcpclient_set_state(client, STATE_BACKOFF); 260 | return; 261 | } 262 | 263 | tcpclient_set_state(client, STATE_CONNECTED); 264 | 265 | // Setup events for recv 266 | client->read_watcher.started = true; 267 | client->read_watcher.watcher.data = client; 268 | ev_io_init(&client->read_watcher.watcher, tcpclient_read_event, client->sd, EV_READ); 269 | ev_io_start(client->loop, &client->read_watcher.watcher); 270 | 271 | client->write_watcher.started = true; 272 | client->write_watcher.watcher.data = client; 273 | ev_io_init(&client->write_watcher.watcher, tcpclient_write_event, client->sd, EV_WRITE); 274 | ev_io_start(client->loop, &client->write_watcher.watcher); 275 | 276 | client->callback_connect(client, EVENT_CONNECTED, client->callback_context, NULL, 0); 277 | } 278 | 279 | int tcpclient_connect(tcpclient_t *client) { 280 | struct addrinfo hints; 281 | struct addrinfo *addr; 282 | int sd; 283 | 284 | if (client->state == STATE_CONNECTED || client->state == STATE_CONNECTING) { 285 | // Already connected, do nothing 286 | return 1; 287 | } 288 | 289 | if (client->state == STATE_BACKOFF) { 290 | // If backoff timer has expired, change to STATE_INIT and call recursively 291 | if ((time(NULL) - client->last_error) > TCPCLIENT_RETRY_TIMEOUT) { 292 | tcpclient_set_state(client, STATE_INIT); 293 | return tcpclient_connect(client); 294 | } else { 295 | return 2; 296 | } 297 | } 298 | 299 | if (client->state == STATE_INIT) { 300 | // Resolve address, create socket, set nonblocking, setup callbacks, fire connect 301 | if (client->config->always_resolve_dns == true && client->addr != NULL) { 302 | freeaddrinfo(client->addr); 303 | client->addr = NULL; 304 | } 305 | 306 | if (client->addr == NULL) { 307 | // We only know about tcp and udp, so if we get something unexpected just 308 | // default to tcp 309 | if (strncmp(client->protocol, "udp", 3) == 0) { 310 | client->socktype = SOCK_DGRAM; 311 | } else { 312 | client->socktype = SOCK_STREAM; 313 | } 314 | memset(&hints, 0, sizeof(hints)); 315 | hints.ai_family = AF_UNSPEC; 316 | hints.ai_socktype = client->socktype; 317 | hints.ai_flags = AI_PASSIVE; 318 | 319 | if (getaddrinfo(client->host, client->port, &hints, &addr) != 0) { 320 | stats_error_log("tcpclient: Error resolving backend address %s: %s", client->host, gai_strerror(errno)); 321 | client->last_error = time(NULL); 322 | tcpclient_set_state(client, STATE_BACKOFF); 323 | client->callback_error(client, EVENT_ERROR, client->callback_context, NULL, 0); 324 | return 3; 325 | } 326 | client->addr = addr; 327 | snprintf(client->name, TCPCLIENT_NAME_LEN, "%s/%s/%s", client->host, client->port, client->protocol); 328 | } else { 329 | addr = client->addr; 330 | } 331 | 332 | if ((sd = socket(addr->ai_family, addr->ai_socktype, addr->ai_protocol)) < 0) { 333 | stats_error_log("tcpclient[%s]: Unable to create socket: %s", client->name, strerror(errno)); 334 | client->last_error = time(NULL); 335 | tcpclient_set_state(client, STATE_BACKOFF); 336 | client->callback_error(client, EVENT_ERROR, client->callback_context, NULL, 0); 337 | return 4; 338 | } 339 | #ifdef TCP_CORK 340 | if (client->config->enable_tcp_cork && 341 | addr->ai_family == AF_INET && 342 | addr->ai_socktype == SOCK_STREAM && 343 | addr->ai_protocol == IPPROTO_TCP) { 344 | int state = 1; 345 | if (setsockopt(sd, IPPROTO_TCP, TCP_CORK, &state, sizeof(state))) { 346 | stats_error_log("failed to set TCP_CORK"); 347 | } 348 | } 349 | #endif 350 | client->sd = sd; 351 | 352 | if (fcntl(sd, F_SETFL, (fcntl(sd, F_GETFL) | O_NONBLOCK)) != 0) { 353 | stats_error_log("tcpclient[%s]: Unable to set socket to non-blocking: %s", client->name, strerror(errno)); 354 | client->last_error = time(NULL); 355 | tcpclient_set_state(client, STATE_BACKOFF); 356 | close(sd); 357 | client->callback_error(client, EVENT_ERROR, client->callback_context, NULL, 0); 358 | return 5; 359 | } 360 | 361 | client->connect_watcher.started = true; 362 | client->connect_watcher.watcher.data = client; 363 | client->timeout_watcher.data = client; 364 | ev_io_init(&client->connect_watcher.watcher, tcpclient_connected, sd, EV_WRITE); 365 | ev_io_start(client->loop, &client->connect_watcher.watcher); 366 | ev_timer_set(&client->timeout_watcher, TCPCLIENT_CONNECT_TIMEOUT, 0); 367 | ev_timer_start(client->loop, &client->timeout_watcher); 368 | 369 | if (connect(sd, addr->ai_addr, addr->ai_addrlen) != 0 && errno != EINPROGRESS) { 370 | stats_error_log("tcpclient[%s]: Unable to connect: %s", client->name, strerror(errno)); 371 | client->last_error = time(NULL); 372 | tcpclient_set_state(client, STATE_BACKOFF); 373 | ev_timer_stop(client->loop, &client->timeout_watcher); 374 | ev_io_stop(client->loop, &client->connect_watcher.watcher); 375 | close(sd); 376 | client->callback_error(client, EVENT_ERROR, client->callback_context, NULL, 0); 377 | return 6; 378 | } 379 | 380 | tcpclient_set_state(client, STATE_CONNECTING); 381 | return 0; 382 | } 383 | 384 | stats_error_log("tcpclient[%s]: Connect with unknown state %i", client->name, client->state); 385 | return 7; 386 | } 387 | 388 | int tcpclient_sendall(tcpclient_t *client, const char *buf, size_t len) { 389 | buffer_t *sendq = &client->send_queue; 390 | 391 | if (client->addr == NULL) { 392 | stats_error_log("tcpclient[%s]: Cannot send before connect!", client->name); 393 | return 1; 394 | } else { 395 | // Does nothing if we're already connected, triggers a 396 | // reconnect if backoff has expired. 397 | tcpclient_connect(client); 398 | } 399 | 400 | if (buffer_datacount(&client->send_queue) >= client->config->max_send_queue) { 401 | if (client->failing == 0) { 402 | stats_error_log("tcpclient[%s]: send queue for %s client is full (at %zd bytes, max is %" PRIu64 " bytes), dropping data", 403 | client->name, 404 | tcpclient_state_name[client->state], 405 | buffer_datacount(&client->send_queue), 406 | client->config->max_send_queue); 407 | client->failing = 1; 408 | } 409 | return 2; 410 | } 411 | if (buffer_spacecount(sendq) < len) { 412 | if (buffer_realign(sendq) != 0) { 413 | stats_error_log("tcpclient[%s]: Unable to realign send queue", client->name); 414 | return 3; 415 | } 416 | } 417 | while (buffer_spacecount(sendq) < len) { 418 | if (buffer_expand(sendq) != 0) { 419 | stats_error_log("tcpclient[%s]: Unable to allocate additional memory for send queue, dropping data", client->name); 420 | return 4; 421 | } 422 | } 423 | memcpy(buffer_tail(sendq), buf, len); 424 | buffer_produced(sendq, len); 425 | 426 | if (client->state == STATE_CONNECTED) { 427 | client->write_watcher.started = true; 428 | ev_io_start(client->loop, &client->write_watcher.watcher); 429 | } 430 | return 0; 431 | } 432 | 433 | void tcpclient_destroy(tcpclient_t *client, int drop_queue) { 434 | if (client == NULL) { 435 | return; 436 | } 437 | ev_timer_stop(client->loop, &client->timeout_watcher); 438 | if (client->connect_watcher.started) { 439 | stats_debug_log("tcpclient_destroy: stopping connect watcher"); 440 | ev_io_stop(client->loop, &client->connect_watcher.watcher); 441 | client->connect_watcher.started = false; 442 | } 443 | if (client->read_watcher.started) { 444 | stats_debug_log("tcpclient_destroy: stopping read watcher"); 445 | ev_io_stop(client->loop, &client->read_watcher.watcher); 446 | client->read_watcher.started = false; 447 | } 448 | if (client->write_watcher.started) { 449 | stats_debug_log("tcpclient_destroy: stopping write watcher"); 450 | ev_io_stop(client->loop, &client->write_watcher.watcher); 451 | client->write_watcher.started = false; 452 | } 453 | stats_debug_log("closing client->sd %d", client->sd); 454 | close(client->sd); 455 | if (client->addr != NULL) { 456 | freeaddrinfo(client->addr); 457 | } 458 | buffer_destroy(&client->send_queue); 459 | 460 | free(client->host); 461 | free(client->port); 462 | free(client->protocol); 463 | } 464 | -------------------------------------------------------------------------------- /src/tcpclient.h: -------------------------------------------------------------------------------- 1 | // This module handles all outbound network communication. Despite it's name, 2 | // it is also capable of connecting to UDP endpoints. 3 | 4 | #ifndef STATSRELAY_TCPCLIENT_H 5 | #define STATSRELAY_TCPCLIENT_H 6 | 7 | #include "config.h" 8 | #include "buffer.h" 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include 15 | 16 | #include "yaml_config.h" 17 | 18 | #define TCPCLIENT_CONNECT_TIMEOUT 2.0 19 | #define TCPCLIENT_RETRY_TIMEOUT 1 20 | #define TCPCLIENT_RECV_BUFFER 65536 21 | #define TCPCLIENT_SEND_QUEUE 134217728 // 128MB 22 | #define TCPCLIENT_NAME_LEN 256 23 | 24 | enum tcpclient_event { 25 | EVENT_CONNECTED, 26 | EVENT_SENT, 27 | EVENT_RECV, 28 | EVENT_ERROR 29 | }; 30 | 31 | enum tcpclient_state { 32 | STATE_INIT = 0, 33 | STATE_CONNECTING, 34 | STATE_BACKOFF, 35 | STATE_CONNECTED, 36 | STATE_TERMINATED 37 | }; 38 | 39 | // data has different meaning depending on the event... 40 | // EVENT_CONNECTED data = NULL 41 | // EVENT_SENT data = pointer to buffer passed to tcpclient_sendall 42 | // EVENT_RECV data = received data (must be free'd manually) 43 | // EVENT_ERROR data = string describing the error 44 | typedef int (*tcpclient_callback)(void *, enum tcpclient_event, void *, char *, size_t); 45 | 46 | typedef struct io_watcher_t { 47 | ev_io watcher; 48 | bool started; 49 | } io_watcher_t; 50 | 51 | typedef struct tcpclient_t { 52 | tcpclient_callback callback_connect; 53 | tcpclient_callback callback_sent; 54 | tcpclient_callback callback_recv; 55 | tcpclient_callback callback_error; 56 | void *callback_context; 57 | 58 | struct ev_loop *loop; 59 | ev_timer timeout_watcher; 60 | io_watcher_t connect_watcher; 61 | io_watcher_t read_watcher; 62 | io_watcher_t write_watcher; 63 | 64 | char name[TCPCLIENT_NAME_LEN]; 65 | struct addrinfo *addr; 66 | buffer_t send_queue; 67 | enum tcpclient_state state; 68 | time_t last_error; 69 | int retry_count; 70 | int failing; 71 | int sd; 72 | int socktype; 73 | 74 | char *host; 75 | char *port; 76 | char *protocol; 77 | 78 | struct proto_config *config; 79 | } tcpclient_t; 80 | 81 | int tcpclient_init(tcpclient_t *client, 82 | struct ev_loop *loop, 83 | void *callback_connect, 84 | struct proto_config *config, 85 | char *host, 86 | char *port, 87 | char *protocol); 88 | 89 | void tcpclient_set_sent_callback(tcpclient_t *client, 90 | tcpclient_callback callback); 91 | 92 | int tcpclient_connect(tcpclient_t *client); 93 | 94 | int tcpclient_sendall(tcpclient_t *client, 95 | const char *buf, 96 | size_t len); 97 | 98 | void tcpclient_destroy(tcpclient_t *client, 99 | int drop_queued); 100 | 101 | #endif // STATSRELAY_TCPCLIENT_H 102 | -------------------------------------------------------------------------------- /src/tcpserver.c: -------------------------------------------------------------------------------- 1 | #include "tcpserver.h" 2 | #include "log.h" 3 | 4 | #include 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include 18 | 19 | #define MAX_TCP_HANDLERS 32 20 | #define LISTEN_BACKLOG 128 21 | 22 | typedef struct tcplistener_t tcplistener_t; 23 | typedef struct tcpsession_t tcpsession_t; 24 | 25 | // tcpserver_t represents an event loop bound to multiple sockets 26 | struct tcpserver_t { 27 | struct ev_loop *loop; 28 | tcplistener_t *listeners[MAX_TCP_HANDLERS]; 29 | int listeners_len; 30 | void *data; 31 | }; 32 | 33 | // tcplistener_t represents a socket listening on a port 34 | struct tcplistener_t { 35 | struct ev_loop *loop; 36 | int sd; 37 | struct ev_io *watcher; 38 | void *data; 39 | void *(*cb_conn)(int, void *); 40 | int (*cb_recv)(int, void *, void *); 41 | }; 42 | 43 | // tcpsession_t represents a client connection to the server 44 | struct tcpsession_t { 45 | struct ev_loop *loop; 46 | int sd; 47 | struct ev_io *watcher; 48 | void *data; 49 | int (*cb_recv)(int, void *, void *); 50 | struct sockaddr_storage client_addr; 51 | void *ctx; 52 | void (*ctx_dealloc)(void *); 53 | }; 54 | 55 | static tcpsession_t *tcpsession_create(tcplistener_t *listener) { 56 | tcpsession_t *session; 57 | 58 | session = malloc(sizeof(tcpsession_t)); 59 | if (session == NULL) { 60 | return NULL; 61 | } 62 | 63 | session->watcher = malloc(sizeof(struct ev_io)); 64 | if (session->watcher == NULL) { 65 | free(session); 66 | return NULL; 67 | } 68 | 69 | session->loop = listener->loop; 70 | session->data = listener->data; 71 | session->sd = -1; 72 | session->cb_recv = listener->cb_recv; 73 | session->watcher->data = (void *)session; 74 | session->ctx = NULL; 75 | session->ctx_dealloc = NULL; 76 | return session; 77 | } 78 | 79 | static void tcpsession_destroy(tcpsession_t *session) { 80 | if (session->sd > 0) { 81 | close(session->sd); 82 | } 83 | ev_io_stop(session->loop, session->watcher); 84 | free(session->watcher); 85 | free(session); 86 | } 87 | 88 | // Called every time the session socket is readable (data available) 89 | // if you don't consume it, it'll get called again very quickly 90 | static void tcpsession_recv_callback(struct ev_loop *loop, 91 | struct ev_io *watcher, 92 | int revents) { 93 | tcpsession_t *session; 94 | 95 | if (revents & EV_ERROR) { 96 | // ev(3) says this is an error of "unspecified" type, so 97 | // that's bloody useful. 98 | stats_error_log("tcpsession: libev server socket error"); 99 | return; 100 | } 101 | 102 | session = (tcpsession_t *)watcher->data; 103 | if (session == NULL) { 104 | stats_error_log("tcpsession: Unable to determine tcpsession, not calling recv callback"); 105 | return; 106 | } 107 | 108 | if (session->cb_recv == NULL) { 109 | stats_error_log("tcpsession: No recv callback registered for session, ignoring event"); 110 | return; 111 | } 112 | 113 | if (session->cb_recv(session->sd, session->data, session->ctx) != 0) { 114 | //stats_error_log("tcpsession: recv callback returned non-zero, closing connection"); 115 | tcpsession_destroy(session); 116 | return; 117 | } 118 | } 119 | 120 | // Called every time the server socket is readable (new connection to be accepted) 121 | // if you don't consume it, it'll get called again very quickly 122 | static void tcplistener_accept_callback(struct ev_loop *loop, 123 | struct ev_io *watcher, 124 | int revents) { 125 | stats_debug_log("in tcplistener_accept_callback"); 126 | socklen_t sin_size; 127 | tcplistener_t *listener; 128 | tcpsession_t *session; 129 | int err; 130 | 131 | if (revents & EV_ERROR) { 132 | // ev(3) says this is an error of "unspecified" type, so 133 | // that's bloody useful. 134 | stats_error_log("tcplistener: libev server socket error"); 135 | return; 136 | } 137 | 138 | listener = (tcplistener_t *) watcher->data; 139 | session = tcpsession_create(listener); 140 | if (session == NULL) { 141 | stats_error_log("tcplistener: Unable to allocate tcpsession, not calling accept()"); 142 | return; 143 | } 144 | 145 | sin_size = sizeof(session->client_addr); 146 | session->sd = accept(watcher->fd, (struct sockaddr *)&session->client_addr, &sin_size); 147 | stats_debug_log("tcpserver: accepted new tcp client connection, client fd = %d, tcp server fd = %d", session->sd, watcher->fd); 148 | if (session->sd < 0) { 149 | stats_error_log("tcplistener: Error accepting connection: %s", strerror(errno)); 150 | return; 151 | } 152 | 153 | err = fcntl(session->sd, F_SETFL, (fcntl(session->sd, F_GETFL) | O_NONBLOCK)); 154 | if (err != 0) { 155 | stats_error_log("tcplistener: Error setting socket to non-blocking: %s", strerror(errno)); 156 | return; 157 | } 158 | 159 | session->ctx = listener->cb_conn(session->sd, session->data); 160 | 161 | ev_io_init(session->watcher, tcpsession_recv_callback, session->sd, EV_READ); 162 | ev_io_start(loop, session->watcher); 163 | } 164 | 165 | 166 | tcpserver_t *tcpserver_create(struct ev_loop *loop, void *data) { 167 | tcpserver_t *server; 168 | server = (tcpserver_t *) malloc(sizeof(tcpserver_t)); 169 | server->loop = ev_default_loop(0); 170 | server->listeners_len = 0; 171 | server->data = data; 172 | return server; 173 | } 174 | 175 | 176 | static tcplistener_t *tcplistener_create(tcpserver_t *server, 177 | struct addrinfo *addr, 178 | void *(*cb_conn)(int, void *), 179 | int (*cb_recv)(int, void *, void *)) { 180 | tcplistener_t *listener; 181 | char addr_string[INET6_ADDRSTRLEN]; 182 | void *ip; 183 | int port; 184 | int yes = 1; 185 | int err; 186 | 187 | listener = malloc(sizeof(tcplistener_t)); 188 | listener->loop = server->loop; 189 | listener->data = server->data; 190 | listener->cb_conn = cb_conn; 191 | listener->cb_recv = cb_recv; 192 | listener->sd = socket( 193 | addr->ai_family, 194 | addr->ai_socktype, 195 | addr->ai_protocol); 196 | 197 | memset(addr_string, 0, INET6_ADDRSTRLEN); 198 | if (addr->ai_family == AF_INET) { 199 | struct sockaddr_in *ipv4 = (struct sockaddr_in *)addr->ai_addr; 200 | ip = &(ipv4->sin_addr); 201 | port = ntohs(ipv4->sin_port); 202 | } else { 203 | struct sockaddr_in6 *ipv6 = (struct sockaddr_in6 *)addr->ai_addr; 204 | ip = &(ipv6->sin6_addr); 205 | port = ntohs(ipv6->sin6_port); 206 | } 207 | if (inet_ntop(addr->ai_family, ip, addr_string, addr->ai_addrlen) == NULL) { 208 | stats_error_log("tcplistener: Unable to format network address string"); 209 | free(listener); 210 | return NULL; 211 | } 212 | 213 | if (listener->sd < 0) { 214 | stats_error_log("tcplistener: Error creating socket %s[:%i]: %s", addr_string, port, strerror(errno)); 215 | free(listener); 216 | return NULL; 217 | } 218 | 219 | err = setsockopt(listener->sd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(int)); 220 | if (err != 0) { 221 | stats_error_log("tcplistener: Error setting SO_REUSEADDR on %s[:%i]: %s", addr_string, port, strerror(errno)); 222 | free(listener); 223 | return NULL; 224 | } 225 | 226 | err = fcntl(listener->sd, F_SETFL, (fcntl(listener->sd, F_GETFL) | O_NONBLOCK)); 227 | if (err != 0) { 228 | stats_error_log("tcplistener: Error setting socket to non-blocking for %s[:%i]: %s", addr_string, port, strerror(errno)); 229 | free(listener); 230 | return NULL; 231 | } 232 | 233 | err = bind(listener->sd, addr->ai_addr, addr->ai_addrlen); 234 | if (err != 0) { 235 | stats_error_log("tcplistener: Error binding socket for %s[:%i]: %s", addr_string, port, strerror(errno)); 236 | free(listener); 237 | return NULL; 238 | } 239 | 240 | err = listen(listener->sd, LISTEN_BACKLOG); 241 | if (err != 0) { 242 | stats_error_log("tcplistener: Error listening to socket %s[:%i]: %s", addr_string, port, strerror(errno)); 243 | free(listener); 244 | return NULL; 245 | } 246 | 247 | listener->watcher = malloc(sizeof(struct ev_io)); 248 | listener->watcher->data = (void *) listener; 249 | 250 | ev_io_init(listener->watcher, tcplistener_accept_callback, listener->sd, EV_READ); 251 | stats_log("tcpserver: Listening on frontend %s[:%i], fd = %d", 252 | addr_string, port, listener->sd); 253 | 254 | return listener; 255 | } 256 | 257 | 258 | static void tcplistener_destroy(tcpserver_t *server, tcplistener_t *listener) { 259 | if (listener->watcher != NULL) { 260 | ev_io_stop(server->loop, listener->watcher); 261 | free(listener->watcher); 262 | } 263 | free(listener); 264 | } 265 | 266 | 267 | int tcpserver_bind(tcpserver_t *server, 268 | const char *address_and_port, 269 | void *(*cb_conn)(int, void *), 270 | int (*cb_recv)(int, void *, void *)) { 271 | tcplistener_t *listener; 272 | struct addrinfo hints; 273 | struct addrinfo *addrs, *p; 274 | int err; 275 | 276 | char *address = strdup(address_and_port); 277 | if (address == NULL) { 278 | stats_error_log("tcpserver: strdup(3) failed"); 279 | return 1; 280 | } 281 | 282 | char *ptr = strrchr(address_and_port, ':'); 283 | if (ptr == NULL) { 284 | free(address); 285 | stats_error_log("tcpserver: missing port"); 286 | return 1; 287 | } 288 | const char *port = ptr + 1; 289 | address[ptr - address_and_port] = '\0'; 290 | 291 | memset(&hints, 0, sizeof(struct addrinfo)); 292 | hints.ai_family = AF_UNSPEC; 293 | hints.ai_socktype = SOCK_STREAM; 294 | hints.ai_flags = AI_PASSIVE; 295 | 296 | err = getaddrinfo(address, port, &hints, &addrs); 297 | if (err != 0) { 298 | free(address); 299 | stats_error_log("tcpserver: getaddrinfo error: %s", gai_strerror(err)); 300 | return 1; 301 | } 302 | 303 | for (p = addrs; p != NULL; p = p->ai_next) { 304 | if (server->listeners_len >= MAX_TCP_HANDLERS) { 305 | stats_error_log("tcpserver: Unable to create more than %i TCP listeners", MAX_TCP_HANDLERS); 306 | free(address); 307 | freeaddrinfo(addrs); 308 | return 1; 309 | } 310 | listener = tcplistener_create(server, p, cb_conn, cb_recv); 311 | if (listener == NULL) { 312 | continue; 313 | } 314 | server->listeners[server->listeners_len] = listener; 315 | server->listeners_len++; 316 | ev_io_start(server->loop, listener->watcher); 317 | } 318 | 319 | free(address); 320 | freeaddrinfo(addrs); 321 | return 0; 322 | } 323 | 324 | void tcpserver_destroy(tcpserver_t *server) { 325 | for (int i = 0; i < server->listeners_len; i++) { 326 | tcplistener_destroy(server, server->listeners[i]); 327 | } 328 | free(server); 329 | } 330 | -------------------------------------------------------------------------------- /src/tcpserver.h: -------------------------------------------------------------------------------- 1 | #ifndef TCPSERVER_H 2 | #define TCPSERVER_H 3 | 4 | #include "config.h" 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | typedef struct tcpserver_t tcpserver_t; 11 | 12 | tcpserver_t *tcpserver_create(struct ev_loop *loop, void *data); 13 | int tcpserver_bind(tcpserver_t *server, 14 | const char *address_and_port, 15 | void *(*cb_conn)(int, void *), 16 | int (*cb_recv)(int, void *, void *)); 17 | void tcpserver_destroy(tcpserver_t *server); 18 | 19 | 20 | #endif 21 | -------------------------------------------------------------------------------- /src/tests/empty.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/statsrelay/068a8257de82b7686164f82acfe0cc7f42da96da/src/tests/empty.yaml -------------------------------------------------------------------------------- /src/tests/hashring1.txt: -------------------------------------------------------------------------------- 1 | 127.0.0.1:9000 2 | 127.0.0.1:9000 3 | 127.0.0.1:9001 4 | 127.0.0.1:9001 5 | -------------------------------------------------------------------------------- /src/tests/hashring2.txt: -------------------------------------------------------------------------------- 1 | 127.0.0.1:9000 2 | 127.0.0.1:9002 3 | 127.0.0.1:9001 4 | 127.0.0.1:9003 5 | -------------------------------------------------------------------------------- /src/tests/stathasher.yaml: -------------------------------------------------------------------------------- 1 | carbon: 2 | bind: 127.0.0.1:2004 3 | shard_map: 4 | 0: 127.0.0.1:2000 5 | 1: 127.0.0.1:2001 6 | 2: 127.0.0.1:2002 7 | 3: 127.0.0.1:2003 8 | statsd: 9 | bind: 127.0.0.1:3004 10 | shard_map: 11 | 0: 127.0.0.1:3000 12 | 1: 127.0.0.1:3001 13 | 2: 127.0.0.1:3002 14 | 3: 127.0.0.1:3003 15 | -------------------------------------------------------------------------------- /src/tests/stathasher_just_carbon.yaml: -------------------------------------------------------------------------------- 1 | carbon: 2 | bind: 127.0.0.1:2004 3 | shard_map: 4 | 0: 127.0.0.1:2000 5 | 1: 127.0.0.1:2001 6 | 2: 127.0.0.1:2002 7 | 3: 127.0.0.1:2003 8 | -------------------------------------------------------------------------------- /src/tests/stathasher_just_statsd.yaml: -------------------------------------------------------------------------------- 1 | statsd: 2 | bind: 127.0.0.1:3004 3 | shard_map: 4 | 0: 127.0.0.1:3000 5 | 1: 127.0.0.1:3001 6 | 2: 127.0.0.1:3002 7 | 3: 127.0.0.1:3003 8 | -------------------------------------------------------------------------------- /src/tests/statsrelay.yaml: -------------------------------------------------------------------------------- 1 | carbon: 2 | bind: 127.0.0.1:BIND_CARBON_PORT 3 | tcp_cork: TCP_CORK 4 | validate: true 5 | always_resolve_dns: false 6 | shard_map: 7 | 0: 127.0.0.1:SEND_CARBON_PORT 8 | 1: 127.0.0.1:SEND_CARBON_PORT 9 | 2: 127.0.0.1:SEND_CARBON_PORT 10 | 3: 127.0.0.1:SEND_CARBON_PORT 11 | 4: 127.0.0.1:SEND_CARBON_PORT 12 | 5: 127.0.0.1:SEND_CARBON_PORT 13 | 6: 127.0.0.1:SEND_CARBON_PORT 14 | 7: 127.0.0.1:SEND_CARBON_PORT 15 | statsd: 16 | bind: 127.0.0.1:BIND_STATSD_PORT 17 | tcp_cork: TCP_CORK 18 | validate: true 19 | always_resolve_dns: false 20 | shard_map: 21 | 0: 127.0.0.1:SEND_STATSD_PORT 22 | 1: 127.0.0.1:SEND_STATSD_PORT 23 | 2: 127.0.0.1:SEND_STATSD_PORT 24 | 3: 127.0.0.1:SEND_STATSD_PORT 25 | 4: 127.0.0.1:SEND_STATSD_PORT 26 | 5: 127.0.0.1:SEND_STATSD_PORT 27 | 6: 127.0.0.1:SEND_STATSD_PORT 28 | 7: 127.0.0.1:SEND_STATSD_PORT 29 | -------------------------------------------------------------------------------- /src/tests/statsrelay_udp.yaml: -------------------------------------------------------------------------------- 1 | carbon: 2 | bind: 127.0.0.1:BIND_CARBON_PORT 3 | validate: true 4 | always_resolve_dns: false 5 | shard_map: 6 | 0: 127.0.0.1:SEND_CARBON_PORT:udp 7 | 1: 127.0.0.1:SEND_CARBON_PORT:udp 8 | 2: 127.0.0.1:SEND_CARBON_PORT:udp 9 | 3: 127.0.0.1:SEND_CARBON_PORT:udp 10 | 4: 127.0.0.1:SEND_CARBON_PORT:udp 11 | 5: 127.0.0.1:SEND_CARBON_PORT:udp 12 | 6: 127.0.0.1:SEND_CARBON_PORT:udp 13 | 7: 127.0.0.1:SEND_CARBON_PORT:udp 14 | statsd: 15 | bind: 127.0.0.1:BIND_STATSD_PORT 16 | validate: true 17 | always_resolve_dns: false 18 | shard_map: 19 | 0: 127.0.0.1:SEND_STATSD_PORT:udp 20 | 1: 127.0.0.1:SEND_STATSD_PORT:udp 21 | 2: 127.0.0.1:SEND_STATSD_PORT:udp 22 | 3: 127.0.0.1:SEND_STATSD_PORT:udp 23 | 4: 127.0.0.1:SEND_STATSD_PORT:udp 24 | 5: 127.0.0.1:SEND_STATSD_PORT:udp 25 | 6: 127.0.0.1:SEND_STATSD_PORT:udp 26 | 7: 127.0.0.1:SEND_STATSD_PORT:udp 27 | -------------------------------------------------------------------------------- /src/tests/test_endtoend.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import contextlib 4 | import signal 5 | import socket 6 | import subprocess 7 | import sys 8 | import tempfile 9 | import time 10 | import unittest 11 | 12 | from collections import defaultdict 13 | 14 | 15 | SOCKET_TIMEOUT = 1 16 | 17 | # While debugging, set this to False 18 | QUIET = True 19 | if QUIET: 20 | DEVNULL = open('/dev/null', 'wb') 21 | POPEN_KW = {'stdout': DEVNULL, 'stderr': DEVNULL} 22 | else: 23 | POPEN_KW = {} 24 | 25 | 26 | class TestCase(unittest.TestCase): 27 | 28 | def setUp(self): 29 | super(TestCase, self).setUp() 30 | self.tcp_cork = 'false' 31 | self.proc = None 32 | 33 | def tearDown(self): 34 | super(TestCase, self).tearDown() 35 | if self.proc: 36 | try: 37 | self.proc.kill() 38 | except OSError: 39 | pass 40 | 41 | def launch_process(self, config_path): 42 | args = ['./statsrelay', '--verbose', '--log-level=DEBUG'] 43 | args.append('--config=' + config_path) 44 | self.proc = subprocess.Popen(args, **POPEN_KW) 45 | time.sleep(0.5) 46 | 47 | def reload_process(self, proc): 48 | proc.send_signal(signal.SIGHUP) 49 | time.sleep(0.1) 50 | 51 | def check_recv(self, fd, expected, size=1024): 52 | bytes_read = fd.recv(size) 53 | self.assertEqual(bytes_read, expected) 54 | 55 | def recv_status(self, fd): 56 | return fd.recv(65536) 57 | 58 | @contextlib.contextmanager 59 | def generate_config(self, mode): 60 | if mode.lower() == 'tcp': 61 | sock_type = socket.SOCK_STREAM 62 | config_path = 'tests/statsrelay.yaml' 63 | elif mode.lower() == 'udp': 64 | sock_type = socket.SOCK_DGRAM 65 | config_path = 'tests/statsrelay_udp.yaml' 66 | else: 67 | raise ValueError() 68 | 69 | try: 70 | self.bind_carbon_port = self.choose_port(sock_type) 71 | self.bind_statsd_port = self.choose_port(sock_type) 72 | 73 | self.carbon_listener = socket.socket(socket.AF_INET, sock_type) 74 | self.carbon_listener.bind(('127.0.0.1', 0)) 75 | self.carbon_listener.settimeout(SOCKET_TIMEOUT) 76 | self.carbon_port = self.carbon_listener.getsockname()[1] 77 | 78 | self.statsd_listener = socket.socket(socket.AF_INET, sock_type) 79 | self.statsd_listener.bind(('127.0.0.1', 0)) 80 | self.statsd_listener.settimeout(SOCKET_TIMEOUT) 81 | self.statsd_port = self.statsd_listener.getsockname()[1] 82 | 83 | if mode.lower() == 'tcp': 84 | self.carbon_listener.listen(1) 85 | self.statsd_listener.listen(1) 86 | 87 | new_config = tempfile.NamedTemporaryFile() 88 | with open(config_path) as config_file: 89 | data = config_file.read() 90 | for var, replacement in [ 91 | ('BIND_CARBON_PORT', self.bind_carbon_port), 92 | ('BIND_STATSD_PORT', self.bind_statsd_port), 93 | ('SEND_CARBON_PORT', self.carbon_port), 94 | ('SEND_STATSD_PORT', self.statsd_port), 95 | ('TCP_CORK', self.tcp_cork)]: 96 | data = data.replace(var, str(replacement)) 97 | new_config.write(data) 98 | new_config.flush() 99 | yield new_config.name 100 | finally: 101 | self.statsd_listener.close() 102 | self.carbon_listener.close() 103 | 104 | def connect(self, sock_type, port): 105 | if sock_type.lower() == 'tcp': 106 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 107 | else: 108 | sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 109 | sock.connect(('127.0.0.1', port)) 110 | sock.settimeout(SOCKET_TIMEOUT) 111 | return sock 112 | 113 | def choose_port(self, sock_type): 114 | s = socket.socket(socket.AF_INET, sock_type) 115 | s.bind(('127.0.0.1', 0)) 116 | port = s.getsockname()[1] 117 | s.close() 118 | return port 119 | 120 | 121 | class ConfigTestCase(TestCase): 122 | """Test config option parsing.""" 123 | 124 | def test_invalid_config_file(self): 125 | """Test that directories are correctly ignored as config files.""" 126 | self.launch_process('.') 127 | self.proc.wait() 128 | self.assertEqual(self.proc.returncode, 1) 129 | 130 | self.launch_process('/etc/passwd') 131 | self.proc.wait() 132 | self.assertEqual(self.proc.returncode, 1) 133 | 134 | def test_check_invalid_config_file(self): 135 | proc = subprocess.Popen( 136 | ['./statsrelay', '-t', '/etc/passwd'], **POPEN_KW) 137 | proc.wait() 138 | self.assertEqual(proc.returncode, 1) 139 | 140 | def test_check_valid_tcp_file(self): 141 | with self.generate_config('tcp') as config_path: 142 | proc = subprocess.Popen(['./statsrelay', '-t', config_path]) 143 | proc.wait() 144 | self.assertEqual(proc.returncode, 0) 145 | 146 | def test_check_valid_udp_file(self): 147 | with self.generate_config('udp') as config_path: 148 | proc = subprocess.Popen(['./statsrelay', '-t', config_path]) 149 | proc.wait() 150 | self.assertEqual(proc.returncode, 0) 151 | 152 | def test_check_empty_file(self): 153 | proc = subprocess.Popen(['./statsrelay', '-c', 'tests/empty.yaml']) 154 | proc.wait() 155 | self.assertEqual(proc.returncode, 1) 156 | 157 | 158 | class StatsdTestCase(TestCase): 159 | 160 | def test_tcp_listener(self): 161 | with self.generate_config('tcp') as config_path: 162 | self.launch_process(config_path) 163 | fd, addr = self.statsd_listener.accept() 164 | sender = self.connect('udp', self.bind_statsd_port) 165 | sender.sendall('test:1|c\n') 166 | self.check_recv(fd, 'test:1|c\n') 167 | sender.sendall('test:1|c\n') 168 | self.check_recv(fd, 'test:1|c\n') 169 | fd.close() 170 | time.sleep(6.0) 171 | sender.sendall('test:xxx\n') 172 | sender.sendall('test:1|c\n') 173 | fd, addr = self.statsd_listener.accept() 174 | self.check_recv(fd, 'test:1|c\n') 175 | sender.close() 176 | 177 | sender = self.connect('tcp', self.bind_statsd_port) 178 | sender.sendall('tcptest:1|c\n') 179 | self.check_recv(fd, 'tcptest:1|c\n') 180 | 181 | sender.sendall('status\n') 182 | status = sender.recv(65536) 183 | sender.close() 184 | 185 | backends = defaultdict(dict) 186 | for line in status.split('\n'): 187 | if not line: 188 | break 189 | if not line.startswith('backend:'): 190 | continue 191 | backend, key, valuetype, value = line.split(' ', 3) 192 | backend = backend.split(':', 1)[1] 193 | backends[backend][key] = int(value) 194 | 195 | key = '127.0.0.1:%d:tcp' % (self.statsd_listener.getsockname()[1],) 196 | self.assertEqual(backends[key]['relayed_lines'], 4) 197 | self.assertEqual(backends[key]['dropped_lines'], 0) 198 | self.assertEqual(backends[key]['bytes_queued'], 199 | backends[key]['bytes_sent']) 200 | 201 | def test_udp_listener(self): 202 | with self.generate_config('udp') as config_path: 203 | self.launch_process(config_path) 204 | sender = self.connect('udp', self.bind_statsd_port) 205 | sender.sendall('test:1|c\n') 206 | fd = self.statsd_listener 207 | self.check_recv(fd, 'test:1|c\n') 208 | sender.sendall('test:1|c\n') 209 | self.check_recv(fd, 'test:1|c\n') 210 | sender.sendall('test:xxx\n') 211 | sender.sendall('test:1|c\n') 212 | self.check_recv(fd, 'test:1|c\n') 213 | sender.close() 214 | 215 | sender = self.connect('tcp', self.bind_statsd_port) 216 | sender.sendall('tcptest:1|c\n') 217 | self.check_recv(fd, 'tcptest:1|c\n') 218 | 219 | sender.sendall('status\n') 220 | status = sender.recv(65536) 221 | sender.close() 222 | 223 | backends = defaultdict(dict) 224 | for line in status.split('\n'): 225 | if not line: 226 | break 227 | if not line.startswith('backend:'): 228 | continue 229 | backend, key, valuetype, value = line.split(' ', 3) 230 | backend = backend.split(':', 1)[1] 231 | backends[backend][key] = int(value) 232 | key = '127.0.0.1:%d:udp' % (self.statsd_listener.getsockname()[1],) 233 | self.assertEqual(backends[key]['relayed_lines'], 4) 234 | self.assertEqual(backends[key]['dropped_lines'], 0) 235 | self.assertEqual(backends[key]['bytes_queued'], 236 | backends[key]['bytes_sent']) 237 | 238 | def test_tcp_cork(self): 239 | if not sys.platform.startswith('linux'): 240 | return 241 | if sys.version_info[:2] < (2, 7): 242 | return 243 | cork_time = 0.200 # cork time is 200ms on linux 244 | self.tcp_cork = 'true' 245 | msg = 'test:1|c\n' 246 | with self.generate_config('tcp') as config_path: 247 | self.launch_process(config_path) 248 | fd, addr = self.statsd_listener.accept() 249 | sender = self.connect('udp', self.bind_statsd_port) 250 | t0 = time.time() 251 | sender.sendall(msg) 252 | self.check_recv(fd, msg) 253 | elapsed = time.time() - t0 254 | 255 | # ensure it took about cork_time ms 256 | self.assertGreater(elapsed, cork_time * 0.95) 257 | self.assertLess(elapsed, cork_time * 1.25) 258 | 259 | # try sending w/o corking... this assumes the mtu of the 260 | # loopback interface is 64k. need to send multiple 261 | # messages to avoid getting EMSGSIZE 262 | needed_messages = ((1 << 16) / len(msg)) / 2 263 | buf = msg * needed_messages 264 | t0 = time.time() 265 | sender.sendall(buf) 266 | sender.sendall(buf) 267 | sender.sendall(msg) 268 | self.assertEqual(len(fd.recv(1024)), 1024) 269 | 270 | # we can tell data wasn't corked because we get a response 271 | # fast enough 272 | elapsed = time.time() - t0 273 | self.assertLess(elapsed, cork_time / 2) 274 | 275 | def test_invalid_line_for_pull_request_35(self): 276 | with self.generate_config('udp') as config_path: 277 | self.launch_process(config_path) 278 | sender = self.connect('udp', self.bind_statsd_port) 279 | sender.sendall('foo.bar:undefined|quux.quuxly.200:1c\n') 280 | sender.sendall('foo.bar:1|c\n') 281 | fd = self.statsd_listener 282 | self.check_recv(fd, 'foo.bar:1|c\n') 283 | self.assert_(self.proc.returncode is None) 284 | 285 | 286 | class CarbonTestCase(TestCase): 287 | 288 | def run_checks(self, fd, proto): 289 | sender = self.connect('udp', self.bind_carbon_port) 290 | sender.sendall('1 2 3\n') 291 | self.check_recv(fd, '1 2 3\n') 292 | sender.sendall('4 5 6\n') 293 | self.check_recv(fd, '4 5 6\n') 294 | sender.sendall('\n') # invalid 295 | sender.sendall('1\n') # invalid 296 | sender.sendall('1 2\n') # invalid 297 | sender.sendall('a b c\n') 298 | self.check_recv(fd, 'a b c\n') 299 | sender.sendall('1 2 3 4\n') # invalid 300 | sender.sendall('1 2 3 4 5\n') # invalid 301 | sender.sendall('d e f\n') 302 | self.check_recv(fd, 'd e f\n') 303 | sender.sendall('1 2 3\n') 304 | self.check_recv(fd, '1 2 3\n') 305 | 306 | sender = self.connect('tcp', self.bind_carbon_port) 307 | sender.sendall('status\n') 308 | status = self.recv_status(sender) 309 | sender.close() 310 | 311 | backends = defaultdict(dict) 312 | for line in status.split('\n'): 313 | if not line: 314 | break 315 | if not line.startswith('backend:'): 316 | continue 317 | backend, key, valuetype, value = line.split(' ', 3) 318 | backend = backend.split(':', 1)[1] 319 | backends[backend][key] = int(value) 320 | 321 | key = '127.0.0.1:%d:%s' % ( 322 | self.carbon_listener.getsockname()[1], proto) 323 | self.assertEqual(backends[key]['relayed_lines'], 5) 324 | self.assertEqual(backends[key]['dropped_lines'], 0) 325 | self.assertEqual(backends[key]['bytes_queued'], 326 | backends[key]['bytes_sent']) 327 | 328 | def test_carbon_tcp(self): 329 | with self.generate_config('tcp') as config: 330 | self.launch_process(config) 331 | fd, addr = self.carbon_listener.accept() 332 | self.run_checks(fd, 'tcp') 333 | 334 | def test_carbon_udp(self): 335 | with self.generate_config('udp') as config: 336 | self.launch_process(config) 337 | self.run_checks(self.carbon_listener, 'udp') 338 | 339 | 340 | class StathasherTests(unittest.TestCase): 341 | 342 | def get_foo(self, config): 343 | proc = subprocess.Popen(['./stathasher', '-c', config], 344 | stdin=subprocess.PIPE, 345 | stdout=subprocess.PIPE) 346 | proc.stdin.write('foo\n') 347 | line = proc.stdout.readline() 348 | return line 349 | 350 | def test_stathasher(self): 351 | line = self.get_foo('tests/stathasher.yaml') 352 | self.assertEqual(line, 'key=foo carbon=127.0.0.1:2001 carbon_shard=1 statsd=127.0.0.1:3001 statsd_shard=1\n') # noqa 353 | 354 | def test_stathasher_empty(self): 355 | line = self.get_foo('tests/empty.yaml') 356 | self.assertEqual(line, 'key=foo\n') 357 | 358 | def test_stathasher_just_carbon(self): 359 | line = self.get_foo('tests/stathasher_just_carbon.yaml') 360 | self.assertEqual(line, 'key=foo carbon=127.0.0.1:2001 carbon_shard=1\n') 361 | 362 | def test_stathasher_just_statsd(self): 363 | line = self.get_foo('tests/stathasher_just_statsd.yaml') 364 | self.assertEqual(line, 'key=foo statsd=127.0.0.1:3001 statsd_shard=1\n') 365 | 366 | 367 | def main(): 368 | unittest.main() 369 | 370 | 371 | if __name__ == '__main__': 372 | sys.exit(main()) 373 | -------------------------------------------------------------------------------- /src/tests/test_hashlib.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "../hashlib.h" 5 | 6 | 7 | int main(int argc, char **argv) { 8 | assert(stats_hash("apple", strlen("apple"), UINT32_MAX) == 2699884538l); 9 | assert(stats_hash("banana", strlen("banana"), UINT32_MAX) == 558421143l); 10 | assert(stats_hash("orange", strlen("orange"), UINT32_MAX) == 2279140812l); 11 | assert(stats_hash("lemon", strlen("lemon"), UINT32_MAX) == 4183924513l); 12 | return 0; 13 | } 14 | -------------------------------------------------------------------------------- /src/tests/test_hashring.c: -------------------------------------------------------------------------------- 1 | #include "../hashring.h" 2 | #include "../log.h" 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | 12 | static void *my_strdup(const char *str, void *data) { 13 | return strdup(str); 14 | } 15 | 16 | static hashring_t create_ring(const char *filename) { 17 | hashring_t ring = hashring_init(NULL, my_strdup, free); 18 | assert(ring != NULL); 19 | 20 | FILE *fp = fopen(filename, "r"); 21 | assert(fp != NULL); 22 | char *line = NULL; 23 | size_t len = 0; 24 | ssize_t read; 25 | while ((read = getline(&line, &len, fp)) != -1) { 26 | for (ssize_t i = 0; i < read; i++) { 27 | if (isspace(line[i])) { 28 | line[i] = '\0'; 29 | break; 30 | } 31 | } 32 | assert(hashring_add(ring, line)); 33 | } 34 | fclose(fp); 35 | free(line); 36 | return ring; 37 | } 38 | 39 | // Test the hashring. Note that when the hash space is expanded in 40 | // hashring1 -> hashring2, we are checking explicitly that apple and 41 | // orange do not move to new nodes. 42 | int main(int argc, char **argv) { 43 | stats_log_verbose(1); 44 | 45 | hashring_t ring = create_ring("tests/hashring1.txt"); 46 | assert(ring != NULL); 47 | uint32_t i; 48 | assert(strcmp(hashring_choose(ring, "apple", &i), "127.0.0.1:9001") == 0); 49 | assert(i == 2); 50 | assert(strcmp(hashring_choose(ring, "banana", &i), "127.0.0.1:9001") == 0); 51 | assert(i == 3); 52 | assert(strcmp(hashring_choose(ring, "orange", &i), "127.0.0.1:9000") == 0); 53 | assert(i == 0); 54 | assert(strcmp(hashring_choose(ring, "lemon", &i), "127.0.0.1:9000") == 0); 55 | assert(i == 1); 56 | hashring_dealloc(ring); 57 | 58 | ring = create_ring("tests/hashring2.txt"); 59 | assert(strcmp(hashring_choose(ring, "apple", &i), "127.0.0.1:9001") == 0); 60 | assert(i == 2); 61 | assert(strcmp(hashring_choose(ring, "banana", &i), "127.0.0.1:9003") == 0); 62 | assert(i == 3); 63 | assert(strcmp(hashring_choose(ring, "orange", &i), "127.0.0.1:9000") == 0); 64 | assert(i == 0); 65 | assert(strcmp(hashring_choose(ring, "lemon", &i), "127.0.0.1:9002") == 0); 66 | assert(i == 1); 67 | hashring_dealloc(ring); 68 | 69 | return 0; 70 | } 71 | -------------------------------------------------------------------------------- /src/udpserver.c: -------------------------------------------------------------------------------- 1 | #include "udpserver.h" 2 | #include "log.h" 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include 16 | 17 | #define MAX_UDP_HANDLERS 32 18 | 19 | typedef struct udplistener_t udplistener_t; 20 | 21 | 22 | // udpserver_t represents an event loop bound to multiple sockets 23 | struct udpserver_t { 24 | struct ev_loop *loop; 25 | udplistener_t *listeners[MAX_UDP_HANDLERS]; 26 | int listeners_len; 27 | void *data; 28 | }; 29 | 30 | // udplistener_t represents a socket listening on a port 31 | struct udplistener_t { 32 | struct ev_loop *loop; 33 | int sd; 34 | struct ev_io *watcher; 35 | void *data; 36 | int (*cb_recv)(int, void *); 37 | }; 38 | 39 | 40 | udpserver_t *udpserver_create(struct ev_loop *loop, void *data) { 41 | udpserver_t *server; 42 | server = malloc(sizeof(udpserver_t)); 43 | server->loop = loop; 44 | server->listeners_len = 0; 45 | server->data = data; 46 | return server; 47 | } 48 | 49 | static void udplistener_recv_callback(struct ev_loop *loop, struct ev_io *watcher, int revents) { 50 | udplistener_t *listener; 51 | listener = (udplistener_t *)watcher->data; 52 | 53 | if (revents & EV_ERROR) { 54 | stats_log("udplistener: libev server socket error"); 55 | return; 56 | } 57 | 58 | if (listener->cb_recv(listener->sd, listener->data) != 0) { 59 | //stats_log("udplistener: recv callback returned non-zero"); 60 | return; 61 | } 62 | } 63 | 64 | static udplistener_t *udplistener_create(udpserver_t *server, struct addrinfo *addr, int (*cb_recv)(int, void *)) { 65 | udplistener_t *listener; 66 | char addr_string[INET6_ADDRSTRLEN]; 67 | void *ip; 68 | int port; 69 | int yes = 1; 70 | int err; 71 | 72 | listener = (udplistener_t *)malloc(sizeof(udplistener_t)); 73 | listener->loop = server->loop; 74 | listener->data = server->data; 75 | listener->cb_recv = cb_recv; 76 | listener->sd = socket(addr->ai_family, 77 | addr->ai_socktype, 78 | addr->ai_protocol); 79 | 80 | memset(addr_string, 0, INET6_ADDRSTRLEN); 81 | if (addr->ai_family == AF_INET) { 82 | struct sockaddr_in *ipv4 = (struct sockaddr_in *)addr->ai_addr; 83 | ip = &(ipv4->sin_addr); 84 | port = ntohs(ipv4->sin_port); 85 | } else { 86 | struct sockaddr_in6 *ipv6 = (struct sockaddr_in6 *)addr->ai_addr; 87 | ip = &(ipv6->sin6_addr); 88 | port = ntohs(ipv6->sin6_port); 89 | } 90 | if (inet_ntop(addr->ai_family, ip, addr_string, addr->ai_addrlen) == NULL) { 91 | stats_log("udplistener: Unable to format network address string"); 92 | free(listener); 93 | return NULL; 94 | } 95 | 96 | if (listener->sd < 0) { 97 | stats_log("udplistener: Error creating socket %s[:%i]: %s", addr_string, port, strerror(errno)); 98 | free(listener); 99 | return NULL; 100 | } 101 | 102 | err = setsockopt(listener->sd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(int)); 103 | if (err != 0) { 104 | stats_log("udplistener: Error setting SO_REUSEADDR on %s[:%i]: %s", addr_string, port, strerror(errno)); 105 | free(listener); 106 | return NULL; 107 | } 108 | 109 | err = fcntl(listener->sd, F_SETFL, (fcntl(listener->sd, F_GETFL) | O_NONBLOCK)); 110 | if (err != 0) { 111 | stats_log("udplistener: Error setting socket to non-blocking for %s[:%i]: %s", addr_string, port, strerror(errno)); 112 | free(listener); 113 | return NULL; 114 | } 115 | 116 | err = bind(listener->sd, addr->ai_addr, addr->ai_addrlen); 117 | if (err != 0) { 118 | stats_log("udplistener: Error binding socket for %s[:%i]: %s", addr_string, port, strerror(errno)); 119 | free(listener); 120 | return NULL; 121 | } 122 | 123 | listener->watcher = (struct ev_io *)malloc(sizeof(struct ev_io)); 124 | listener->watcher->data = (void *)listener; 125 | 126 | ev_io_init(listener->watcher, udplistener_recv_callback, listener->sd, EV_READ); 127 | stats_log("udpserver: Listening on frontend %s[:%i], fd = %d", addr_string, port, listener->sd); 128 | 129 | return listener; 130 | } 131 | 132 | 133 | static void udplistener_destroy(udpserver_t *server, udplistener_t *listener) { 134 | if (listener->watcher != NULL) { 135 | ev_io_stop(server->loop, listener->watcher); 136 | free(listener->watcher); 137 | } 138 | free(listener); 139 | } 140 | 141 | 142 | int udpserver_bind(udpserver_t *server, 143 | const char *address_and_port, 144 | int (*cb_recv)(int, void *)) { 145 | udplistener_t *listener; 146 | struct addrinfo hints; 147 | struct addrinfo *addrs, *p; 148 | int err; 149 | 150 | char *address = strdup(address_and_port); 151 | if (address == NULL) { 152 | stats_log("udpserver: strdup(3) failed"); 153 | return 1; 154 | } 155 | 156 | char *ptr = strrchr(address_and_port, ':'); 157 | if (ptr == NULL) { 158 | stats_error_log("udpserver: missing port"); 159 | free(address); 160 | return 1; 161 | } 162 | const char *port = ptr + 1; 163 | address[ptr - address_and_port] = '\0'; 164 | 165 | memset(&hints, 0, sizeof(struct addrinfo)); 166 | hints.ai_family = AF_UNSPEC; 167 | hints.ai_socktype = SOCK_DGRAM; 168 | hints.ai_flags = AI_PASSIVE; 169 | 170 | err = getaddrinfo(address, port, &hints, &addrs); 171 | if (err != 0) { 172 | free(address); 173 | stats_log("udpserver: getaddrinfo error: %s", gai_strerror(err)); 174 | return 1; 175 | } 176 | 177 | for (p = addrs; p != NULL; p = p->ai_next) { 178 | if (server->listeners_len >= MAX_UDP_HANDLERS) { 179 | stats_log("udpserver: Unable to create more than %i UDP listeners", MAX_UDP_HANDLERS); 180 | free(address); 181 | freeaddrinfo(addrs); 182 | return 1; 183 | } 184 | listener = udplistener_create(server, p, cb_recv); 185 | if (listener == NULL) { 186 | continue; 187 | } 188 | server->listeners[server->listeners_len] = listener; 189 | server->listeners_len++; 190 | ev_io_start(server->loop, listener->watcher); 191 | } 192 | 193 | free(address); 194 | freeaddrinfo(addrs); 195 | return 0; 196 | } 197 | 198 | 199 | void udpserver_destroy(udpserver_t *server) { 200 | int i; 201 | 202 | for (i = 0; i < server->listeners_len; i++) { 203 | udplistener_destroy(server, server->listeners[i]); 204 | } 205 | //ev_break(server->loop, EVBREAK_ALL); 206 | //ev_loop_destroy(server->loop); 207 | free(server); 208 | } 209 | -------------------------------------------------------------------------------- /src/udpserver.h: -------------------------------------------------------------------------------- 1 | #ifndef UDPSERVER_H 2 | #define UDPSERVER_H 3 | 4 | #include "config.h" 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | typedef struct udpserver_t udpserver_t; 11 | 12 | udpserver_t *udpserver_create(struct ev_loop *loop, void *data); 13 | int udpserver_bind(udpserver_t *server, 14 | const char *address_and_port, 15 | int (*cb_recv)(int, void *)); 16 | void udpserver_destroy(udpserver_t *server); 17 | 18 | #endif 19 | -------------------------------------------------------------------------------- /src/validate.c: -------------------------------------------------------------------------------- 1 | #include "validate.h" 2 | 3 | #include "log.h" 4 | 5 | #include 6 | 7 | static char *valid_stat_types[6] = { 8 | "c", 9 | "ms", 10 | "kv", 11 | "g", 12 | "h", 13 | "s" 14 | }; 15 | static size_t valid_stat_types_len = 6; 16 | 17 | 18 | int validate_statsd(const char *line, size_t len) { 19 | size_t plen; 20 | char c; 21 | int i, valid; 22 | 23 | // FIXME: this is dumb, don't do a memory copy 24 | char *line_copy = strndup(line, len); 25 | char *start, *end; 26 | char *err; 27 | 28 | start = line_copy; 29 | plen = len; 30 | end = memchr(start, ':', plen); 31 | if (end == NULL) { 32 | stats_log("validate: Invalid line \"%.*s\" missing ':'", len, line); 33 | goto statsd_err; 34 | } 35 | 36 | if ((end - start) < 1) { 37 | stats_log("validate: Invalid line \"%.*s\" zero length key", len, line); 38 | goto statsd_err; 39 | } 40 | 41 | start = end + 1; 42 | plen = len - (start - line_copy); 43 | 44 | c = end[0]; 45 | end[0] = '\0'; 46 | if ((strtod(start, &err) == 0.0) && (err == start)) { 47 | stats_log("validate: Invalid line \"%.*s\" unable to parse value as double", len, line); 48 | goto statsd_err; 49 | } 50 | end[0] = c; 51 | 52 | end = memchr(start, '|', plen); 53 | if (end == NULL) { 54 | stats_log("validate: Invalid line \"%.*s\" missing '|'", len, line); 55 | goto statsd_err; 56 | } 57 | 58 | start = end + 1; 59 | plen = len - (start - line_copy); 60 | 61 | end = memchr(start, '|', plen); 62 | if (end != NULL) { 63 | c = end[0]; 64 | end[0] = '\0'; 65 | plen = end - start; 66 | } 67 | 68 | valid = 0; 69 | for (i = 0; i < valid_stat_types_len; i++) { 70 | if (strlen(valid_stat_types[i]) != plen) { 71 | continue; 72 | } 73 | if (strncmp(start, valid_stat_types[i], plen) == 0) { 74 | valid = 1; 75 | break; 76 | } 77 | } 78 | 79 | if (valid == 0) { 80 | stats_log("validate: Invalid line \"%.*s\" unknown stat type \"%.*s\"", len, line, plen, start); 81 | goto statsd_err; 82 | } 83 | 84 | if (end != NULL) { 85 | end[0] = c; 86 | // end[0] is currently the second | char 87 | // test if we have at least 1 char following it (@) 88 | if ((len - (end - line_copy) > 1) && (end[1] == '@')) { 89 | start = end + 2; 90 | plen = len - (start - line_copy); 91 | if (plen == 0) { 92 | stats_log("validate: Invalid line \"%.*s\" @ sample with no rate", len, line); 93 | goto statsd_err; 94 | } 95 | if ((strtod(start, &err) == 0.0) && err == start) { 96 | stats_log("validate: Invalid line \"%.*s\" invalid sample rate", len, line); 97 | goto statsd_err; 98 | } 99 | } else { 100 | stats_log("validate: Invalid line \"%.*s\" no @ sample rate specifier", len, line); 101 | goto statsd_err; 102 | } 103 | } 104 | 105 | free(line_copy); 106 | return 0; 107 | 108 | statsd_err: 109 | free(line_copy); 110 | return 1; 111 | } 112 | 113 | int validate_carbon(const char *line, size_t len) { 114 | int spaces_found = 0; 115 | const char *p = line; 116 | size_t n = len; 117 | while (1) { 118 | const char *s = memchr(p, ' ', n); 119 | if (s == NULL) { 120 | break; 121 | } 122 | spaces_found++; 123 | n = len - (s - line) - 1; 124 | p = s + 1; 125 | if (spaces_found > 2) { 126 | break; 127 | } 128 | } 129 | if (spaces_found != 2) { 130 | stats_log("validate: found %d spaces in invalid carbon line", spaces_found); 131 | return 1; 132 | } 133 | return 0; 134 | } 135 | -------------------------------------------------------------------------------- /src/validate.h: -------------------------------------------------------------------------------- 1 | #ifndef STATSRELAY_VALIDATE_H 2 | #define STATSRELAY_VALIDATE_H 3 | 4 | #include 5 | 6 | typedef int (*validate_line_validator_t)(const char *, size_t); 7 | 8 | int validate_statsd(const char *, size_t); 9 | int validate_carbon(const char *, size_t); 10 | 11 | #endif // STATSRELAY_VALIDATE_H 12 | -------------------------------------------------------------------------------- /src/yaml_config.c: -------------------------------------------------------------------------------- 1 | #include "yaml_config.h" 2 | 3 | #include "log.h" 4 | 5 | #include 6 | #include 7 | #include 8 | 9 | // convert a string to a number; stores the result in num, and returns 10 | // true if it really was a number, and false otherwise 11 | static bool convert_number(const char *str, long *num) { 12 | char *endptr; 13 | *num = strtol(str, &endptr, 10); 14 | return endptr != str; 15 | } 16 | 17 | static bool set_boolean(const char *strval, bool *bool_val) { 18 | if (strcmp(strval, "true") == 0) { 19 | *bool_val = true; 20 | } else if (strcmp(strval, "false") == 0) { 21 | *bool_val = false; 22 | } else { 23 | stats_error_log("unexpected value \"%s\" for boolean field, " 24 | "must be true/false", strval); 25 | return false; 26 | } 27 | return true; 28 | } 29 | 30 | static void init_proto_config(struct proto_config *protoc) { 31 | protoc->initialized = false; 32 | protoc->bind = NULL; 33 | protoc->enable_validation = true; 34 | protoc->enable_tcp_cork = true; 35 | protoc->always_resolve_dns = false; 36 | protoc->max_send_queue = 134217728; 37 | protoc->ring = statsrelay_list_new(); 38 | } 39 | 40 | struct config* parse_config(FILE *input) { 41 | struct config *config = malloc(sizeof(struct config)); 42 | if (config == NULL) { 43 | stats_error_log("malloc() error"); 44 | return NULL; 45 | } 46 | 47 | init_proto_config(&config->carbon_config); 48 | if (config->carbon_config.ring == NULL) { 49 | stats_error_log("failed to allocate ring"); 50 | free(config); 51 | return NULL; 52 | } 53 | config->carbon_config.bind = strdup("127.0.0.1:2003"); 54 | 55 | init_proto_config(&config->statsd_config); 56 | config->statsd_config.bind = strdup("127.0.0.1:8125"); 57 | 58 | yaml_parser_t parser; 59 | yaml_event_t event; 60 | 61 | if (!yaml_parser_initialize(&parser)) { 62 | stats_log("failed to initialize yaml parser"); 63 | goto parse_err; 64 | } 65 | yaml_parser_set_input_file(&parser, input); 66 | 67 | struct proto_config *protoc = NULL; 68 | char *strval; 69 | long numval; 70 | int shard_count = -1; 71 | int map_nesting = 0; 72 | bool in_document = false; 73 | bool keep_going = true; 74 | bool is_key = false; 75 | bool update_bind = false; 76 | bool update_send_queue = false; 77 | bool update_validate = false; 78 | bool update_tcp_cork = false; 79 | bool always_resolve_dns = false; 80 | bool expect_shard_map = false; 81 | while (keep_going) { 82 | if (!yaml_parser_parse(&parser, &event)) { 83 | goto parse_err; 84 | } 85 | 86 | switch(event.type) { 87 | case YAML_NO_EVENT: 88 | case YAML_STREAM_START_EVENT: 89 | break; // nothing to do 90 | case YAML_STREAM_END_EVENT: 91 | keep_going = false; 92 | break; 93 | case YAML_DOCUMENT_START_EVENT: 94 | if (in_document) { 95 | stats_error_log("config should not have nested documents"); 96 | goto parse_err; 97 | } 98 | in_document = true; 99 | break; 100 | case YAML_DOCUMENT_END_EVENT: 101 | in_document = false; 102 | break; 103 | case YAML_SEQUENCE_START_EVENT: 104 | case YAML_SEQUENCE_END_EVENT: 105 | stats_error_log("unexpectedly got sequence"); 106 | goto parse_err; 107 | break; 108 | case YAML_MAPPING_START_EVENT: 109 | is_key = true; 110 | map_nesting++; 111 | break; 112 | case YAML_MAPPING_END_EVENT: 113 | map_nesting--; 114 | break; 115 | case YAML_ALIAS_EVENT: 116 | stats_error_log("don't know how to handle yaml aliases"); 117 | goto parse_err; 118 | break; 119 | case YAML_SCALAR_EVENT: 120 | strval = (char *) event.data.scalar.value; 121 | switch (map_nesting) { 122 | case 0: 123 | stats_error_log("unexpectedly got scalar outside of a map"); 124 | goto parse_err; 125 | break; 126 | case 1: 127 | if (strcmp(strval, "carbon") == 0) { 128 | protoc = &config->carbon_config; 129 | config->carbon_config.initialized = true; 130 | } else if (strcmp(strval, "statsd") == 0) { 131 | protoc = &config->statsd_config; 132 | config->statsd_config.initialized = true; 133 | } else { 134 | stats_error_log("unexpectedly got map value: \"%s\"", strval); 135 | goto parse_err; 136 | } 137 | break; 138 | case 2: 139 | if (is_key) { 140 | if (strcmp(strval, "bind") == 0) { 141 | update_bind = true; 142 | } else if (strcmp(strval, "max_send_queue") == 0) { 143 | update_send_queue = true; 144 | } else if (strcmp(strval, "shard_map") == 0) { 145 | shard_count = -1; 146 | expect_shard_map = true; 147 | } else if (strcmp(strval, "validate") == 0) { 148 | update_validate = true; 149 | } else if (strcmp(strval, "tcp_cork") == 0) { 150 | update_tcp_cork = true; 151 | } else if (strcmp(strval, "always_resolve_dns") == 0) { 152 | always_resolve_dns = true; 153 | } 154 | } else { 155 | if (update_bind) { 156 | free(protoc->bind); 157 | protoc->bind = strdup(strval); 158 | update_bind = false; 159 | } else if (update_send_queue) { 160 | if (!convert_number(strval, &numval)) { 161 | stats_error_log("max_send_queue was not a number: %s", strval); 162 | } 163 | protoc->max_send_queue = numval; 164 | update_send_queue = false; 165 | } else if (update_validate) { 166 | if (!set_boolean(strval, &protoc->enable_validation)) { 167 | goto parse_err; 168 | } 169 | update_validate = false; 170 | } else if (update_tcp_cork) { 171 | if (!set_boolean(strval, &protoc->enable_tcp_cork)) { 172 | goto parse_err; 173 | } 174 | update_tcp_cork = false; 175 | } else if (always_resolve_dns) { 176 | if (!set_boolean(strval, &protoc->always_resolve_dns)) { 177 | goto parse_err; 178 | } 179 | } 180 | } 181 | break; 182 | case 3: 183 | if (!expect_shard_map) { 184 | stats_error_log("was not expecting shard map"); 185 | goto parse_err; 186 | } else if (is_key) { 187 | if (!convert_number(strval, &numval)) { 188 | stats_error_log("shard key was not a number: \"%s\"", strval); 189 | goto parse_err; 190 | 191 | } 192 | shard_count++; 193 | if (numval != shard_count) { 194 | stats_error_log("expected to see shard key %d, instead saw %d", 195 | shard_count, numval); 196 | goto parse_err; 197 | } 198 | } else { 199 | if (statsrelay_list_expand(protoc->ring) == NULL) { 200 | stats_error_log("unable to expand list"); 201 | goto parse_err; 202 | } 203 | if ((protoc->ring->data[protoc->ring->size - 1] = strdup(strval)) == NULL) { 204 | stats_error_log("failed to copy string"); 205 | goto parse_err; 206 | } 207 | } 208 | } 209 | is_key = !is_key; 210 | break; 211 | default: 212 | stats_error_log("unhandled yaml event"); 213 | goto parse_err; 214 | } 215 | yaml_event_delete(&event); 216 | } 217 | 218 | yaml_parser_delete(&parser); 219 | return config; 220 | 221 | parse_err: 222 | destroy_config(config); 223 | yaml_event_delete(&event); 224 | yaml_parser_delete(&parser); 225 | return NULL; 226 | } 227 | 228 | void destroy_config(struct config *config) { 229 | if (config != NULL) { 230 | statsrelay_list_destroy_full(config->carbon_config.ring); 231 | free(config->carbon_config.bind); 232 | statsrelay_list_destroy_full(config->statsd_config.ring); 233 | free(config->statsd_config.bind); 234 | free(config); 235 | } 236 | } 237 | -------------------------------------------------------------------------------- /src/yaml_config.h: -------------------------------------------------------------------------------- 1 | #ifndef STATSRELAY_YAML_CONFIG_H 2 | #define STATSRELAY_YAML_CONFIG_H 3 | 4 | #include "./list.h" 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | struct proto_config { 11 | bool initialized; 12 | char *bind; 13 | bool enable_validation; 14 | bool enable_tcp_cork; 15 | bool always_resolve_dns; 16 | uint64_t max_send_queue; 17 | list_t ring; 18 | }; 19 | 20 | struct config { 21 | struct proto_config statsd_config; 22 | struct proto_config carbon_config; 23 | }; 24 | 25 | 26 | static const char default_config[] = "/etc/statsrelay.yaml"; 27 | 28 | struct config* parse_config(FILE *input); 29 | 30 | // release the memory associated with a config 31 | void destroy_config(struct config *); 32 | 33 | #endif // STATSRELAY_YAML_CONFIG_H 34 | -------------------------------------------------------------------------------- /tag-release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | VERSION=$1 4 | 5 | if [ -z "$VERSION" ]; then 6 | echo "Usage: ./tag-release.sh " 7 | exit 1 8 | fi 9 | 10 | echo "Tagging master as $VERSION" 11 | git checkout master 12 | echo "$VERSION" > VERSION 13 | git commit -m "Release $VERSION" VERSION 14 | git tag upstream/$VERSION 15 | git checkout debian 16 | git merge upstream/$VERSION 17 | debchange -v $VERSION-1 -D unstable "New upstream release" 18 | git commit -m "Updated changelog for $VERSION-1" debian/changelog 19 | git tag debian/$VERSION-1 20 | 21 | echo -n "Tagging complete. Are you sure you want to push to origin (yes/no)? " 22 | read CONFIRM 23 | if [ "$CONFIRM" != "yes" ]; then 24 | echo "Abort! You need to reset to origin and delete tags or push to origin manually now" 25 | exit 2 26 | fi 27 | 28 | git push origin debian 29 | git checkout master 30 | git push origin master 31 | git push origin --tags 32 | echo "$VERSION released!" 33 | -------------------------------------------------------------------------------- /test/devnull.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | void devnull(int fd) { 9 | char buf[65536]; 10 | size_t len; 11 | 12 | while (1) { 13 | len = read(fd, buf, 65536); 14 | if (len < 1) { 15 | fprintf(stderr, "Child exiting\n"); 16 | return; 17 | } 18 | } 19 | } 20 | 21 | int main(int argc, char *argv[]) { 22 | int sd; 23 | 24 | if (argc < 2) { 25 | fprintf(stderr, "Usage: %s \n", argv[0]); 26 | return 1; 27 | } 28 | 29 | struct sockaddr_storage their_addr; 30 | socklen_t addr_size; 31 | struct addrinfo hints, *res; 32 | int sockfd, acceptfd; 33 | pid_t pid; 34 | 35 | // first, load up address structs with getaddrinfo(): 36 | 37 | memset(&hints, 0, sizeof hints); 38 | hints.ai_family = AF_UNSPEC; // use IPv4 or IPv6, whichever 39 | hints.ai_socktype = SOCK_STREAM; 40 | hints.ai_flags = AI_PASSIVE; // fill in my IP for me 41 | 42 | getaddrinfo(NULL, argv[1], &hints, &res); 43 | 44 | // make a socket: 45 | 46 | sockfd = socket(res->ai_family, res->ai_socktype, res->ai_protocol); 47 | 48 | // bind it to the port we passed in to getaddrinfo(): 49 | 50 | bind(sockfd, res->ai_addr, res->ai_addrlen); 51 | 52 | listen(sockfd, 1); 53 | 54 | while (1) { 55 | acceptfd = accept(sockfd, (struct sockaddr *)&their_addr, &addr_size); 56 | pid = fork(); 57 | if (pid == 0) { 58 | // child 59 | devnull(acceptfd); 60 | return 0; 61 | } else { 62 | fprintf(stderr, "Forked pid %i\n", pid); 63 | } 64 | } 65 | 66 | return 0; 67 | } 68 | -------------------------------------------------------------------------------- /test/ketama.servers: -------------------------------------------------------------------------------- 1 | #------ Server ------- -Mem-# 2 | #255.255.255.255:65535 66666# 3 | 10.0.1.1:8125 600 4 | 10.0.1.2:8125 300 5 | 10.0.1.3:8125 200 6 | 10.0.1.4:8125 350 7 | 10.0.1.5:8125 1000 8 | 10.0.1.6:8125 800 9 | 10.0.1.7:8125 950 10 | 10.0.1.8:8125 100 11 | -------------------------------------------------------------------------------- /test/listener.py: -------------------------------------------------------------------------------- 1 | import socket 2 | 3 | sock = socket.socket() 4 | sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 5 | sock.bind(('', 8127)) 6 | sock.listen(1) 7 | 8 | while True: 9 | client, addr = sock.accept() 10 | print 'Connection from', addr 11 | while True: 12 | data = client.recv(1024) 13 | if not data: 14 | client.close() 15 | break 16 | print repr(data) 17 | -------------------------------------------------------------------------------- /test/poll_stats.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | import socket 3 | import time 4 | import sys 5 | 6 | def get_stats(): 7 | s = socket.socket() 8 | s.connect(('127.0.0.1', 8125)) 9 | s.sendall('status\n') 10 | lines = s.recv(65536) 11 | result = defaultdict(int) 12 | for line in lines.split('\n'): 13 | line = line.strip('\r\n\t ') 14 | if not line: 15 | continue 16 | line = line.split(' ') 17 | result[line[1]] += int(line[3]) 18 | s.close() 19 | return result 20 | 21 | last = get_stats() 22 | interval = float(sys.argv[1]) 23 | while True: 24 | time.sleep(interval) 25 | stats = get_stats() 26 | 27 | print 'Relayed: %i/s' % ((stats['relayed_lines'] - last['relayed_lines']) / interval) 28 | print 'Dropped: %i' % (stats['dropped_lines'] - last['dropped_lines']) 29 | print 'In queue: %i bytes' % (stats['bytes_queued'] - stats['bytes_sent']) 30 | 31 | last = stats 32 | -------------------------------------------------------------------------------- /test/spam.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import socket 3 | 4 | 5 | if __name__ == '__main__': 6 | parser = argparse.ArgumentParser() 7 | parser.add_argument('-p', '--port', type=int, default=8125) 8 | parser.add_argument('-n', '--num-stats', type=int, default=0) 9 | parser.add_argument('-r', '--reconnect-interval', type=int, default=0) 10 | parser.add_argument('--word-file') 11 | args = parser.parse_args() 12 | 13 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 14 | sock.connect(('localhost', args.port)) 15 | 16 | reconn = args.reconnect_interval 17 | count = args.num_stats 18 | words = [] 19 | if args.word_file: 20 | with open(args.word_file) as wf: 21 | x = 0 22 | for line in wf: 23 | words.append(line.strip()) 24 | x += 1 25 | if count == 0: 26 | count = x 27 | else: 28 | words = ['test'] 29 | 30 | x = 0 31 | while True: 32 | break_out = False 33 | for word in words: 34 | x += 1 35 | print x, count 36 | stat = word + ':1|c\n' 37 | sock.sendall(stat) 38 | if count and x >= count: 39 | break_out = True 40 | break 41 | elif reconn and x % reconn == 0: 42 | sock.close() 43 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 44 | sock.connect(('localhost', args.port)) 45 | if break_out: 46 | break 47 | -------------------------------------------------------------------------------- /test/test.servers: -------------------------------------------------------------------------------- 1 | #------ Server ------- -Mem-# 2 | #255.255.255.255:65535 66666# 3 | 127.0.0.1:8127 600 4 | 127.0.0.1:8128 600 5 | #127.0.0.1:8129 600 6 | #127.0.0.1:8130 600 7 | --------------------------------------------------------------------------------