├── tests ├── .gitignore ├── thread_helper.h ├── ftoa.c ├── atomic.c ├── mstore.c ├── main.c ├── statsd_msg.c ├── histogram.c └── sput.h ├── .gitignore ├── .gitmodules ├── script ├── bootstrap └── cibuild ├── src ├── bloom.h ├── backends │ ├── carbon.h │ └── carbon.c ├── slab.h ├── histogram.h ├── ht.h ├── sampler.c ├── brubeck.h ├── backend.h ├── log.h ├── slab.c ├── sampler.h ├── samplers │ ├── statsd.h │ ├── statsd-secure.c │ └── statsd.c ├── backend.c ├── bloom.c ├── log.c ├── server.h ├── metric.h ├── setproctitle.c ├── histogram.c ├── internal_sampler.c ├── utils.h ├── ht.c ├── city.c ├── utils.c ├── http.c ├── metric.c └── server.c ├── config.default.json.example ├── brubeck.c ├── LICENSE ├── CONTRIBUTING.md ├── Makefile ├── test-bin ├── udp-stress.c ├── statsd-secure-send.c └── balancer.c └── README.md /tests/.gitignore: -------------------------------------------------------------------------------- 1 | .clarcache 2 | clar.suite 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | brubeck 2 | *.o 3 | *.swp 4 | *~ 5 | .DS_Store 6 | build/* 7 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "vendor/ck"] 2 | path = vendor/ck 3 | url = https://github.com/concurrencykit/ck 4 | -------------------------------------------------------------------------------- /script/bootstrap: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ROOT=$(dirname "$0")/.. 4 | cd "$ROOT" && git submodule update --init && make brubeck 5 | -------------------------------------------------------------------------------- /script/cibuild: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | ROOT=$(dirname "$0")/.. 5 | 6 | echo "=> Cleaning @ $(date)" 7 | git clean -fdx 8 | git submodule update --init 9 | 10 | echo "=> Building @ $(date)" 11 | cd $ROOT 12 | 13 | make test && ./brubeck_test 14 | -------------------------------------------------------------------------------- /src/bloom.h: -------------------------------------------------------------------------------- 1 | #ifndef __BLOOM_FILTER_H 2 | #define __BLOOM_FILTER_H 3 | 4 | #include 5 | 6 | struct multibloom { 7 | int bits; 8 | int bytes; 9 | int hashes; 10 | unsigned char *filters[]; 11 | }; 12 | 13 | int multibloom_check(struct multibloom *bloom, int f, uint32_t a, uint32_t b); 14 | void multibloom_reset(struct multibloom *bloom, int f); 15 | struct multibloom *multibloom_new(int filters, int entries, double error); 16 | 17 | #endif 18 | -------------------------------------------------------------------------------- /tests/thread_helper.h: -------------------------------------------------------------------------------- 1 | #ifndef __CL_THREAD_HELPER_H__ 2 | #define __CL_THREAD_HELPER_H__ 3 | 4 | #include 5 | 6 | #define MAX_THREADS 16 7 | 8 | static inline void 9 | spawn_threads(void *(*thread_ptr)(void*), void *ptr) 10 | { 11 | size_t i; 12 | pthread_t threads[MAX_THREADS]; 13 | 14 | for (i = 0; i < MAX_THREADS; ++i) 15 | pthread_create(&threads[i], NULL, thread_ptr, ptr); 16 | 17 | for (i =0; i < MAX_THREADS; ++i) 18 | pthread_join(threads[i], NULL); 19 | } 20 | 21 | #endif 22 | -------------------------------------------------------------------------------- /tests/ftoa.c: -------------------------------------------------------------------------------- 1 | #include "sput.h" 2 | #include "brubeck.h" 3 | 4 | static void check_eq(float f, const char *str) 5 | { 6 | char buf[16]; 7 | brubeck_ftoa(buf, f); 8 | sput_fail_unless(strcmp(str, buf) == 0, str); 9 | } 10 | 11 | void test_ftoa(void) 12 | { 13 | check_eq(0.0, "0"); 14 | check_eq(15.0, "15"); 15 | check_eq(15.5, "15.5"); 16 | check_eq(15.505, "15.505"); 17 | check_eq(0.125, "0.125"); 18 | check_eq(1234.567, "1234.567"); 19 | check_eq(99999.999, "100000"); 20 | check_eq(0.999, "0.999"); 21 | } 22 | -------------------------------------------------------------------------------- /src/backends/carbon.h: -------------------------------------------------------------------------------- 1 | #ifndef __BRUBECK_CARBON_H__ 2 | #define __BRUBECK_CARBON_H__ 3 | 4 | #define MAX_PICKLE_SIZE 256 5 | #define PICKLE_BUFFER_SIZE 4096 6 | #define PICKLE1_SIZE(key_len) (32 + key_len) 7 | 8 | struct brubeck_carbon { 9 | struct brubeck_backend backend; 10 | 11 | int out_sock; 12 | struct sockaddr_in out_sockaddr; 13 | struct pickler { 14 | char *ptr; 15 | uint16_t pos; 16 | uint16_t pt; 17 | } pickler; 18 | size_t sent; 19 | }; 20 | 21 | struct brubeck_backend *brubeck_carbon_new( 22 | struct brubeck_server *server, json_t *settings, int shard_n); 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /src/slab.h: -------------------------------------------------------------------------------- 1 | #ifndef __BRUBECK_SLAB_H__ 2 | #define __BRUBECK_SLAB_H__ 3 | 4 | /* Each slab has 32 bytes; 128 slabs per node = 4096 bytes (one page) */ 5 | #define SLAB_SIZE 32 6 | #define SLABS_PER_NODE 128 7 | #define NODE_SIZE (SLAB_SIZE * (SLABS_PER_NODE - 1)) 8 | 9 | struct brubeck_slab_node { 10 | struct brubeck_slab_node *next; 11 | size_t alloc; 12 | char heap[]; 13 | }; 14 | 15 | struct brubeck_slab { 16 | struct brubeck_slab_node *current; 17 | size_t total_alloc; 18 | pthread_mutex_t lock; 19 | }; 20 | 21 | void brubeck_slab_init(struct brubeck_slab *slab); 22 | void *brubeck_slab_alloc(struct brubeck_slab *slab, size_t need); 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /src/histogram.h: -------------------------------------------------------------------------------- 1 | #ifndef __BRUBECK_HISTO_H__ 2 | #define __BRUBECK_HISTO_H__ 3 | 4 | struct brubeck_histo { 5 | value_t *values; 6 | uint32_t count; 7 | uint16_t alloc, size; 8 | }; 9 | 10 | struct brubeck_histo_sample { 11 | value_t sum; 12 | value_t min; 13 | value_t max; 14 | value_t mean; 15 | value_t median; 16 | value_t count; 17 | 18 | value_t percentile[5]; 19 | }; 20 | 21 | enum { PC_75, PC_95, PC_98, PC_99, PC_999 }; 22 | 23 | void brubeck_histo_push(struct brubeck_histo *histo, value_t value, value_t sample_rate); 24 | void brubeck_histo_sample( 25 | struct brubeck_histo_sample *sample, 26 | struct brubeck_histo *histo); 27 | 28 | #endif 29 | -------------------------------------------------------------------------------- /src/ht.h: -------------------------------------------------------------------------------- 1 | #ifndef _HT_H_ 2 | #define _HT_H_ 3 | 4 | #include 5 | 6 | struct brubeck_metric; 7 | typedef struct brubeck_hashtable_t brubeck_hashtable_t; 8 | 9 | brubeck_hashtable_t *brubeck_hashtable_new(const uint64_t size); 10 | void brubeck_hashtable_free(brubeck_hashtable_t *ht); 11 | struct brubeck_metric *brubeck_hashtable_find(brubeck_hashtable_t *ht, const char *key, uint16_t key_len); 12 | bool brubeck_hashtable_insert(brubeck_hashtable_t *ht, const char *key, uint16_t key_len, struct brubeck_metric *val); 13 | size_t brubeck_hashtable_size(brubeck_hashtable_t *ht); 14 | void brubeck_hashtable_foreach(brubeck_hashtable_t *ht, void (*callback)(struct brubeck_metric *, void *), void *payload); 15 | struct brubeck_metric **brubeck_hashtable_to_a(brubeck_hashtable_t *ht, size_t *length); 16 | 17 | #endif 18 | -------------------------------------------------------------------------------- /src/sampler.c: -------------------------------------------------------------------------------- 1 | #include "brubeck.h" 2 | 3 | void 4 | brubeck_sampler_init_inet(struct brubeck_sampler *sampler, struct brubeck_server *server, const char *url, int port) 5 | { 6 | sampler->server = server; 7 | url_to_inaddr2(&sampler->addr, url, port); 8 | 9 | log_splunk("sampler=%s event=load_udp addr=0.0.0.0:%d", 10 | brubeck_sampler_name(sampler), port); 11 | } 12 | 13 | int brubeck_sampler_socket(struct brubeck_sampler *sampler, int multisock) 14 | { 15 | int sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); 16 | 17 | assert(sock >= 0); 18 | 19 | sock_enlarge_in(sock); 20 | sock_setreuse(sock, 1); 21 | 22 | if (multisock) 23 | sock_setreuse_port(sock, 1); 24 | 25 | if (bind(sock, (struct sockaddr *)&sampler->addr, sizeof(sampler->addr)) < 0) 26 | die("failed to bind socket"); 27 | 28 | return sock; 29 | } 30 | -------------------------------------------------------------------------------- /config.default.json.example: -------------------------------------------------------------------------------- 1 | { 2 | "sharding" : false, 3 | "server_name" : "brubeck_debug", 4 | "dumpfile" : "./brubeck.dump", 5 | "capacity" : 15, 6 | "expire" : 20, 7 | "http" : ":8080", 8 | 9 | "backends" : [ 10 | { 11 | "type" : "carbon", 12 | "address" : "localhost", 13 | "port" : 2003, 14 | "frequency" : 10 15 | } 16 | ], 17 | 18 | "samplers" : [ 19 | { 20 | "type" : "statsd", 21 | "address" : "0.0.0.0", 22 | "port" : 8126, 23 | "workers" : 4, 24 | "multisock" : true, 25 | "multimsg" : 8 26 | }, 27 | { 28 | "type" : "statsd-secure", 29 | "address" : "0.0.0.0", 30 | "port" : 9126, 31 | "max_drift" : 3, 32 | "hmac_key" : "750c783e6ab0b503eaa86e310a5db738", 33 | "replay_len" : 8000 34 | } 35 | ] 36 | } 37 | -------------------------------------------------------------------------------- /tests/atomic.c: -------------------------------------------------------------------------------- 1 | #include "brubeck.h" 2 | #include "thread_helper.h" 3 | #include "sput.h" 4 | 5 | #define INCREMENTS (1024*4) 6 | #define DELTA 0.5 7 | 8 | struct spinlock_test { 9 | double value; 10 | pthread_spinlock_t lock; 11 | }; 12 | 13 | static void *thread_spinlock(void *ptr) 14 | { 15 | struct spinlock_test *spt = ptr; 16 | size_t i; 17 | 18 | for (i = 0; i < INCREMENTS; ++i) { 19 | pthread_spin_lock(&spt->lock); 20 | { 21 | volatile double v = spt->value; 22 | v = v + DELTA; 23 | spt->value = v; 24 | } 25 | pthread_spin_unlock(&spt->lock); 26 | } 27 | 28 | return NULL; 29 | } 30 | 31 | void test_atomic_spinlocks(void) 32 | { 33 | struct spinlock_test spt; 34 | 35 | pthread_spin_init(&spt.lock, 0); 36 | spt.value = 0.0; 37 | 38 | spawn_threads(&thread_spinlock, &spt); 39 | sput_fail_unless(spt.value == (double)(INCREMENTS * MAX_THREADS * DELTA), 40 | "spinlock doesn't race"); 41 | } 42 | -------------------------------------------------------------------------------- /src/brubeck.h: -------------------------------------------------------------------------------- 1 | #ifndef __BRUBECK__H_ 2 | #define __BRUBECK__H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #define _GNU_SOURCE 21 | #include 22 | #undef _GNU_SOURCE 23 | 24 | #define BRUBECK_STATS 1 25 | #define MAX_ADDR 256 26 | 27 | typedef double value_t; 28 | typedef uint64_t hash_t; 29 | 30 | struct brubeck_server; 31 | struct brubeck_metric; 32 | 33 | #include "jansson.h" 34 | #include "log.h" 35 | #include "utils.h" 36 | #include "slab.h" 37 | #include "histogram.h" 38 | #include "metric.h" 39 | #include "sampler.h" 40 | #include "backend.h" 41 | #include "ht.h" 42 | #include "server.h" 43 | 44 | #endif 45 | -------------------------------------------------------------------------------- /src/backend.h: -------------------------------------------------------------------------------- 1 | #ifndef __BRUBECK_BACKEND_H__ 2 | #define __BRUBECK_BACKEND_H__ 3 | 4 | enum brubeck_backend_t { 5 | BRUBECK_BACKEND_CARBON 6 | }; 7 | 8 | struct brubeck_backend { 9 | enum brubeck_backend_t type; 10 | struct brubeck_server *server; 11 | int sample_freq; 12 | int shard_n; 13 | 14 | int (*connect)(void *); 15 | bool (*is_connected)(void *); 16 | void (*sample)(const char *, value_t, void *); 17 | void (*flush)(void *); 18 | 19 | uint32_t tick_time; 20 | pthread_t thread; 21 | 22 | struct brubeck_metric *queue; 23 | }; 24 | 25 | void brubeck_backend_run_threaded(struct brubeck_backend *); 26 | void brubeck_backend_register_metric(struct brubeck_backend *self, struct brubeck_metric *metric); 27 | 28 | static inline const char *brubeck_backend_name(struct brubeck_backend *backend) 29 | { 30 | switch (backend->type) { 31 | case BRUBECK_BACKEND_CARBON: return "carbon"; 32 | default: return NULL; 33 | } 34 | } 35 | 36 | #include "backends/carbon.h" 37 | 38 | #endif 39 | -------------------------------------------------------------------------------- /src/log.h: -------------------------------------------------------------------------------- 1 | #ifndef __GH_LOG_H__ 2 | #define __GH_LOG_H__ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | const char *gh_log_instance(void); 10 | void gh_log_set_instance(const char *instance); 11 | void gh_log_open(const char *path); 12 | void gh_log_reopen(void); 13 | void gh_log_die(void) __attribute__ ((noreturn)); 14 | void gh_log_write(const char *message, ...) 15 | __attribute__((format (printf, 1, 2))); 16 | 17 | #define log_splunk(M, ...) gh_log_write("instance=%s " M "\n", gh_log_instance(), ##__VA_ARGS__); 18 | 19 | #define log_splunk_errno(M, ...) log_splunk( \ 20 | M " errno=%d msg=\"%s\"", ##__VA_ARGS__, errno, strerror(errno)); 21 | 22 | #ifdef NDEBUG 23 | #define debug(M, ...) 24 | #else 25 | #define debug(M, ...) gh_log_write("[DBG]: " M "\n", ##__VA_ARGS__) 26 | #endif 27 | 28 | #define die(M, ...) do {\ 29 | fprintf(stderr, "[FATAL]: " M "\n", ##__VA_ARGS__); \ 30 | gh_log_die(); \ 31 | } while(0) 32 | 33 | #endif 34 | -------------------------------------------------------------------------------- /src/slab.c: -------------------------------------------------------------------------------- 1 | #include "brubeck.h" 2 | 3 | static struct brubeck_slab_node *push_node(struct brubeck_slab *slab) 4 | { 5 | struct brubeck_slab_node *node = xmalloc(SLAB_SIZE * SLABS_PER_NODE); 6 | 7 | node->alloc = 0; 8 | node->next = slab->current; 9 | slab->current = node; 10 | 11 | return node; 12 | } 13 | 14 | void *brubeck_slab_alloc(struct brubeck_slab *slab, size_t need) 15 | { 16 | struct brubeck_slab_node *node; 17 | void *ptr; 18 | 19 | need = ((need + SLAB_SIZE - 1) & ~(SLAB_SIZE - 1)); 20 | 21 | pthread_mutex_lock(&slab->lock); 22 | 23 | node = slab->current; 24 | 25 | if (node->alloc + need > NODE_SIZE) { 26 | node = push_node(slab); 27 | } 28 | 29 | slab->total_alloc += need; 30 | 31 | ptr = node->heap + node->alloc; 32 | node->alloc += need; 33 | 34 | pthread_mutex_unlock(&slab->lock); 35 | 36 | return ptr; 37 | } 38 | 39 | void brubeck_slab_init(struct brubeck_slab *slab) 40 | { 41 | push_node(slab); 42 | pthread_mutex_init(&slab->lock, NULL); 43 | } 44 | -------------------------------------------------------------------------------- /src/sampler.h: -------------------------------------------------------------------------------- 1 | #ifndef __BRUBECK_SAMPLER_H__ 2 | #define __BRUBECK_SAMPLER_H__ 3 | 4 | enum brubeck_sampler_t { 5 | BRUBECK_SAMPLER_STATSD, 6 | BRUBECK_SAMPLER_STATSD_SECURE, 7 | }; 8 | 9 | struct brubeck_sampler { 10 | enum brubeck_sampler_t type; 11 | struct brubeck_server *server; 12 | 13 | int in_sock; 14 | struct sockaddr_in addr; 15 | 16 | size_t inflow; 17 | size_t current_flow; 18 | 19 | void (*shutdown)(struct brubeck_sampler *); 20 | }; 21 | 22 | int brubeck_sampler_socket(struct brubeck_sampler *sampler, int multisock); 23 | void brubeck_sampler_init_inet( 24 | struct brubeck_sampler *sampler, 25 | struct brubeck_server *server, 26 | const char *url, int port); 27 | 28 | static inline const char *brubeck_sampler_name(struct brubeck_sampler *sampler) 29 | { 30 | switch (sampler->type) { 31 | case BRUBECK_SAMPLER_STATSD: return "statsd"; 32 | case BRUBECK_SAMPLER_STATSD_SECURE: return "statsd-secure"; 33 | default: return NULL; 34 | } 35 | } 36 | 37 | #include "samplers/statsd.h" 38 | 39 | #endif 40 | 41 | -------------------------------------------------------------------------------- /brubeck.c: -------------------------------------------------------------------------------- 1 | #include "brubeck.h" 2 | #include "getopt.h" 3 | 4 | int main(int argc, char *argv[]) 5 | { 6 | static struct option longopts[] = { 7 | { "log", required_argument, NULL, 'l' }, 8 | { "config", required_argument, NULL, 'c' }, 9 | { "version", no_argument, NULL, 'v' }, 10 | { NULL, 0, NULL, 0 } 11 | }; 12 | 13 | struct brubeck_server _server; 14 | const char *config_file = "config.default.json"; 15 | const char *log_file = NULL; 16 | int opt; 17 | 18 | while ((opt = getopt_long(argc, argv, ":l:c:v", longopts, NULL)) != -1) { 19 | switch (opt) { 20 | case 'l': log_file = optarg; break; 21 | case 'c': config_file = optarg; break; 22 | case 'v': 23 | puts("brubeck " GIT_SHA); 24 | return 0; 25 | 26 | default: 27 | printf("Usage: %s [--log LOG_FILE] [--config CONFIG_FILE] [--version]\n", argv[0]); 28 | return 1; 29 | } 30 | } 31 | 32 | initproctitle(argc, argv); 33 | gh_log_open(log_file); 34 | brubeck_server_init(&_server, config_file); 35 | return brubeck_server_run(&_server); 36 | } 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012-2015 GitHub, Inc. 2 | 3 | Permission is hereby granted, free of charge, to any person 4 | obtaining a copy of this software and associated documentation 5 | files (the "Software"), to deal in the Software without 6 | restriction, including without limitation the rights to use, 7 | copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the 9 | Software is furnished to do so, subject to the following 10 | conditions: 11 | 12 | The above copyright notice and this permission notice shall be 13 | included in all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 17 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 19 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 20 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /tests/mstore.c: -------------------------------------------------------------------------------- 1 | #include "sput.h" 2 | #include "brubeck.h" 3 | 4 | static struct brubeck_metric *new_metric(const char *name) 5 | { 6 | size_t name_len = strlen(name); 7 | struct brubeck_metric *metric = malloc(sizeof(struct brubeck_metric) + name_len + 1); 8 | 9 | memcpy(metric->key, name, name_len); 10 | metric->key[name_len] = 0; 11 | metric->key_len = name_len; 12 | 13 | return metric; 14 | } 15 | 16 | void test_mstore__save(void) 17 | { 18 | static const int nmetrics = 15000; 19 | brubeck_hashtable_t *store; 20 | int i; 21 | 22 | store = brubeck_hashtable_new(4096); 23 | 24 | for (i = 0; i < nmetrics; ++i) { 25 | char buffer[64]; 26 | struct brubeck_metric *metric; 27 | 28 | sprintf(buffer, "github.test.metric.%d", i); 29 | metric = new_metric(buffer); 30 | 31 | if (!brubeck_hashtable_insert(store, metric->key, metric->key_len, metric)) 32 | break; 33 | } 34 | 35 | sput_fail_unless(i == nmetrics, "stored 15000 metrics in table"); 36 | 37 | for (i = 0; i < nmetrics; ++i) { 38 | char buffer[64]; 39 | uint16_t len; 40 | 41 | len = sprintf(buffer, "github.test.metric.%d", i); 42 | 43 | if (!brubeck_hashtable_find(store, buffer, len)) 44 | break; 45 | } 46 | 47 | sput_fail_unless(i == nmetrics, "lookup all metrics from table"); 48 | } 49 | -------------------------------------------------------------------------------- /src/samplers/statsd.h: -------------------------------------------------------------------------------- 1 | #ifndef __BRUBECK_STATSD_H__ 2 | #define __BRUBECK_STATSD_H__ 3 | 4 | #include "bloom.h" 5 | 6 | struct brubeck_statsd_msg { 7 | char *key; /* The key of the message, NULL terminated */ 8 | uint16_t key_len; /* length of the key */ 9 | uint16_t type; /* type of the messaged, as a brubeck_mt_t */ 10 | value_t value; /* floating point value of the message */ 11 | value_t sample_freq; /* floating poit sample freq (1.0 / sample_rate) */ 12 | uint8_t modifiers; /* modifiers, as a brubeck_metric_mod_t */ 13 | }; 14 | 15 | struct brubeck_statsd { 16 | struct brubeck_sampler sampler; 17 | pthread_t *workers; 18 | unsigned int worker_count; 19 | unsigned int mmsg_count; 20 | }; 21 | 22 | struct brubeck_statsd_secure { 23 | struct brubeck_sampler sampler; 24 | const char *hmac_key; 25 | 26 | struct multibloom *replays; 27 | time_t now; 28 | time_t drift; 29 | 30 | pthread_t thread; 31 | }; 32 | 33 | void brubeck_statsd_packet_parse(struct brubeck_server *server, char *buffer, char *end); 34 | int brubeck_statsd_msg_parse(struct brubeck_statsd_msg *msg, char *buffer, char *end); 35 | 36 | struct brubeck_sampler * brubeck_statsd_secure_new(struct brubeck_server *server, json_t *settings); 37 | struct brubeck_sampler *brubeck_statsd_new(struct brubeck_server *server, json_t *settings); 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /src/backend.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "brubeck.h" 4 | 5 | void brubeck_backend_register_metric(struct brubeck_backend *self, struct brubeck_metric *metric) 6 | { 7 | for (;;) { 8 | struct brubeck_metric *next = self->queue; 9 | metric->next = next; 10 | 11 | if (__sync_bool_compare_and_swap(&self->queue, next, metric)) 12 | break; 13 | } 14 | } 15 | 16 | static void *backend__thread(void *_ptr) 17 | { 18 | struct brubeck_backend *self = (struct brubeck_backend *)_ptr; 19 | 20 | for (;;) { 21 | struct timespec now, then; 22 | 23 | clock_gettime(CLOCK_MONOTONIC, &then); 24 | then.tv_sec += self->sample_freq; 25 | 26 | if (!self->connect(self)) { 27 | struct brubeck_metric *mt; 28 | 29 | clock_gettime(CLOCK_REALTIME, &now); 30 | self->tick_time = now.tv_sec; 31 | 32 | for (mt = self->queue; mt; mt = mt->next) { 33 | if (mt->expire > BRUBECK_EXPIRE_DISABLED) 34 | brubeck_metric_sample(mt, self->sample, self); 35 | } 36 | 37 | if (self->flush) 38 | self->flush(self); 39 | } 40 | 41 | clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, &then, NULL); 42 | } 43 | return NULL; 44 | } 45 | 46 | void brubeck_backend_run_threaded(struct brubeck_backend *self) 47 | { 48 | if (pthread_create(&self->thread, NULL, &backend__thread, self) != 0) 49 | die("failed to start backend thread"); 50 | } 51 | 52 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Contributing 2 | 3 | [fork]: https://github.com/github/brubeck/fork 4 | [pr]: https://github.com/github/brubeck/compare 5 | 6 | Hi there! We're thrilled that you'd liske to contribute to this project. Your help is essential for keeping it great. 7 | 8 | Contributions to this project are [released](https://help.github.com/articles/github-terms-of-service/#6-contributions-under-repository-license) to the public under the [project's open source license](LICENSE). 9 | 10 | ## Submitting a pull request 11 | 12 | 0. [Fork][fork] and clone the repository 13 | 0. Create a new branch: `git checkout -b my-branch-name` 14 | 0. Make your change, push to your fork and [submit a pull request][pr] 15 | 0. Pat your self on the back and wait for your pull request to be reviewed and merged. 16 | 17 | Here are a few things you can do that will increase the likelihood of your pull request being accepted: 18 | 19 | - Keep your change as focused as possible. If there are multiple changes you would like to make that are not dependent upon each other, consider submitting them as separate pull requests. 20 | - Write a [good commit message](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html). 21 | 22 | ## Resources 23 | 24 | - [How to Contribute to Open Source](https://opensource.guide/how-to-contribute/) 25 | - [Using Pull Requests](https://help.github.com/articles/about-pull-requests/) 26 | - [GitHub Help](https://help.github.com) 27 | -------------------------------------------------------------------------------- /tests/main.c: -------------------------------------------------------------------------------- 1 | #include "brubeck.h" 2 | #include "sput.h" 3 | 4 | struct sput __sput; 5 | 6 | void test_histogram__sampling(void); 7 | void test_histogram__single_element(void); 8 | void test_histogram__large_range(void); 9 | void test_histogram__multisamples(void); 10 | void test_histogram__with_sample_rate(void); 11 | void test_histogram__capacity(void); 12 | 13 | void test_mstore__save(void); 14 | void test_atomic_spinlocks(void); 15 | void test_ftoa(void); 16 | void test_statsd_msg__parse_strings(void); 17 | 18 | int main(int argc, char *argv[]) 19 | { 20 | sput_start_testing(); 21 | 22 | sput_enter_suite("histogram: time/data series aggregation"); 23 | sput_run_test(test_histogram__sampling); 24 | sput_run_test(test_histogram__single_element); 25 | sput_run_test(test_histogram__large_range); 26 | sput_run_test(test_histogram__multisamples); 27 | sput_run_test(test_histogram__with_sample_rate); 28 | sput_run_test(test_histogram__capacity); 29 | 30 | sput_enter_suite("mstore: concurrency test for metrics hash table"); 31 | sput_run_test(test_mstore__save); 32 | 33 | sput_enter_suite("atomic: atomic primitives"); 34 | sput_run_test(test_atomic_spinlocks); 35 | 36 | sput_enter_suite("ftoa: double-to-string conversion"); 37 | sput_run_test(test_ftoa); 38 | 39 | sput_enter_suite("statsd: packet parsing"); 40 | sput_run_test(test_statsd_msg__parse_strings); 41 | 42 | sput_finish_testing(); 43 | return sput_get_return_value(); 44 | } 45 | -------------------------------------------------------------------------------- /src/bloom.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "brubeck.h" 10 | #include "bloom.h" 11 | 12 | int multibloom_check(struct multibloom *bloom, int f, uint32_t a, uint32_t b) 13 | { 14 | unsigned char *filter = bloom->filters[f]; 15 | 16 | int hits = 0; 17 | uint32_t x, i, byte, mask; 18 | unsigned char c; 19 | 20 | for (i = 0; i < bloom->hashes; i++) { 21 | x = (a + i*b) % bloom->bits; 22 | byte = x >> 3; 23 | c = filter[byte]; 24 | mask = 1 << (x % 8); 25 | 26 | if (c & mask) { 27 | hits++; 28 | } else { 29 | filter[byte] = c | mask; 30 | } 31 | } 32 | 33 | return (hits == bloom->hashes); 34 | } 35 | 36 | void multibloom_reset(struct multibloom *bloom, int f) 37 | { 38 | memset(bloom->filters[f], 0x0, bloom->bytes); 39 | } 40 | 41 | struct multibloom *multibloom_new(int filters, int entries, double error) 42 | { 43 | const double bpe = -(log(error) / 0.480453013918201); 44 | int i; 45 | struct multibloom *bloom = xmalloc(sizeof(struct multibloom) + filters * sizeof(void *)); 46 | 47 | assert(entries > 1 && error > 0.0); 48 | 49 | bloom->bits = (int)((double)entries * bpe); 50 | bloom->bytes = (bloom->bits / 8); 51 | 52 | if (bloom->bits % 8) 53 | bloom->bytes++; 54 | 55 | bloom->hashes = (int)ceil(0.693147180559945 * bpe); // ln(2) 56 | 57 | for (i = 0; i < filters; ++i) 58 | bloom->filters[i] = xcalloc(1, bloom->bytes); 59 | 60 | log_splunk( 61 | "event=bloom_init entries=%d error=%f bits=%d bpe=%f " 62 | "bytes=%d hash_funcs=%d", 63 | entries, error, bloom->bits, bpe, bloom->bytes, bloom->hashes 64 | ); 65 | 66 | return bloom; 67 | } 68 | 69 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | GIT_SHA = $(shell git rev-parse --short HEAD) 2 | TARGET = brubeck 3 | LIBS = -lm -pthread -lrt -lcrypto -ljansson 4 | CC = gcc 5 | CXX = g++ 6 | CFLAGS = -g -Wall -O3 -Wno-strict-aliasing -Isrc -Ivendor/ck/include -DNDEBUG=1 -DGIT_SHA=\"$(GIT_SHA)\" 7 | 8 | .PHONY: default all clean 9 | 10 | default: $(TARGET) 11 | all: default 12 | 13 | SOURCES = \ 14 | src/backend.c \ 15 | src/backends/carbon.c \ 16 | src/bloom.c \ 17 | src/city.c \ 18 | src/histogram.c \ 19 | src/ht.c \ 20 | src/http.c \ 21 | src/internal_sampler.c \ 22 | src/log.c \ 23 | src/metric.c \ 24 | src/sampler.c \ 25 | src/samplers/statsd-secure.c \ 26 | src/samplers/statsd.c \ 27 | src/server.c \ 28 | src/setproctitle.c \ 29 | src/slab.c \ 30 | src/utils.c 31 | 32 | ifndef BRUBECK_NO_HTTP 33 | LIBS += -lmicrohttpd 34 | CFLAGS += -DBRUBECK_HAVE_MICROHTTPD 35 | endif 36 | 37 | OBJECTS = $(patsubst %.c, %.o, $(SOURCES)) 38 | HEADERS = $(wildcard src/*.h) $(wildcard src/libcuckoo/*.h) 39 | 40 | TEST_SRC = $(wildcard tests/*.c) 41 | TEST_OBJ = $(patsubst %.c, %.o, $(TEST_SRC)) 42 | 43 | %.o: %.c $(HEADERS) vendor/ck/src/libck.a 44 | $(CC) $(CFLAGS) -c $< -o $@ 45 | 46 | .PRECIOUS: $(TARGET) $(OBJECTS) 47 | 48 | $(TARGET): $(OBJECTS) brubeck.o 49 | $(CC) -flto brubeck.o $(OBJECTS) $(LIBS) vendor/ck/src/libck.a -o $@.new 50 | mv $@.new $@ 51 | 52 | $(TARGET)_test: $(OBJECTS) $(TEST_OBJ) 53 | $(CC) $(OBJECTS) $(TEST_OBJ) $(LIBS) vendor/ck/src/libck.a -o $@ 54 | 55 | test: $(TARGET)_test 56 | ./$(TARGET)_test 57 | 58 | vendor/ck/Makefile: 59 | cd vendor/ck && ./configure 60 | 61 | vendor/ck/src/libck.a: vendor/ck/Makefile 62 | $(MAKE) -C vendor/ck 63 | 64 | clean: 65 | -rm -f $(OBJECTS) brubeck.o 66 | -rm -f $(TEST_OBJ) 67 | -rm -f $(TARGET) $(TARGET)_test 68 | -------------------------------------------------------------------------------- /src/log.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include "log.h" 8 | 9 | static const char *gh_log_path = NULL; 10 | 11 | static FILE *gh_log_file; 12 | static int gh_syslog_enabled; 13 | 14 | void gh_log_open(const char *path) 15 | { 16 | FILE *new_log; 17 | 18 | if (path == NULL) { 19 | gh_syslog_enabled = 0; 20 | gh_log_file = NULL; 21 | return; 22 | } 23 | 24 | if (!strcmp(path, "syslog")) { 25 | openlog(NULL, LOG_PID, LOG_LOCAL7); 26 | gh_syslog_enabled = 1; 27 | return; 28 | } 29 | 30 | new_log = fopen(path, "a"); 31 | 32 | if (new_log == NULL) { 33 | fprintf(stderr, "Failed to open log file at '%s'\n", path); 34 | return; 35 | } 36 | 37 | if (gh_log_file) 38 | fclose(gh_log_file); 39 | 40 | if (gh_syslog_enabled) { 41 | closelog(); 42 | gh_syslog_enabled = 0; 43 | } 44 | 45 | gh_log_file = new_log; 46 | gh_log_path = path; 47 | } 48 | 49 | void gh_log_reopen(void) 50 | { 51 | if (gh_log_path != NULL) 52 | gh_log_open(gh_log_path); 53 | } 54 | 55 | void gh_log_write(const char *message, ...) 56 | { 57 | va_list vl; 58 | va_start(vl, message); 59 | 60 | if (gh_syslog_enabled) { 61 | vsyslog(LOG_INFO, message, vl); 62 | } else if (gh_log_file) { 63 | /* TODO: add syslog-like extra args here */ 64 | vfprintf(gh_log_file, message, vl); 65 | fflush(gh_log_file); 66 | } else { 67 | vfprintf(stderr, message, vl); 68 | fflush(stderr); 69 | } 70 | 71 | va_end(vl); 72 | } 73 | 74 | void gh_log_die(void) 75 | { 76 | exit(1); 77 | } 78 | 79 | static const char *_app_instance = NULL; 80 | 81 | const char *gh_log_instance(void) 82 | { 83 | return _app_instance; 84 | } 85 | 86 | void gh_log_set_instance(const char *instance) 87 | { 88 | _app_instance = instance; 89 | } 90 | -------------------------------------------------------------------------------- /src/server.h: -------------------------------------------------------------------------------- 1 | #ifndef __BRUBECK_SERVER_H__ 2 | #define __BRUBECK_SERVER_H__ 3 | 4 | struct brubeck_internal_stats { 5 | int sample_freq; 6 | struct { 7 | uint32_t metrics; 8 | uint32_t errors; 9 | uint32_t unique_keys; 10 | 11 | struct { 12 | uint32_t failed; 13 | uint32_t from_future; 14 | uint32_t delayed; 15 | uint32_t replayed; 16 | } secure; 17 | } live, sample; 18 | }; 19 | 20 | // Server 21 | struct brubeck_server { 22 | const char *name; 23 | const char *dump_path; 24 | const char *config_name; 25 | int running; 26 | int active_backends; 27 | int active_samplers; 28 | 29 | int fd_signal; 30 | int fd_expire; 31 | int fd_update; 32 | 33 | struct brubeck_slab slab; 34 | 35 | brubeck_hashtable_t *metrics; 36 | int at_capacity; 37 | 38 | struct brubeck_sampler *samplers[8]; 39 | struct brubeck_backend *backends[8]; 40 | 41 | json_t *config; 42 | struct brubeck_internal_stats internal_stats; 43 | }; 44 | 45 | #define brubeck_stats_inc(server, STAT) brubeck_atomic_inc(&server->internal_stats.live.STAT) 46 | #define brubeck_stats_sample(server, STAT) (server->internal_stats.sample.STAT) 47 | 48 | void brubeck_http_endpoint_init(struct brubeck_server *server, const char *listen_on); 49 | 50 | void brubeck_internal__init(struct brubeck_server *server); 51 | void brubeck_internal__sample(struct brubeck_metric *metric, brubeck_sample_cb sample, void *opaque); 52 | 53 | void brubeck_server_new_metric(struct brubeck_server *server, struct brubeck_metric *metric); 54 | 55 | int brubeck_server_run(struct brubeck_server *server); 56 | void brubeck_server_init(struct brubeck_server *server, const char *config); 57 | void brubeck_server_conf(struct brubeck_server *server, int argc, char *argv[]); 58 | 59 | void brubeck_cache_load(struct brubeck_server *server); 60 | void brubeck_cache_save(struct brubeck_server *server); 61 | 62 | #endif 63 | -------------------------------------------------------------------------------- /src/metric.h: -------------------------------------------------------------------------------- 1 | #ifndef __BRUBECK_METRIC_H__ 2 | #define __BRUBECK_METRIC_H__ 3 | 4 | enum brubeck_metric_t { 5 | BRUBECK_MT_GAUGE, /** g */ 6 | BRUBECK_MT_METER, /** c */ 7 | BRUBECK_MT_COUNTER, /** C */ 8 | BRUBECK_MT_HISTO, /** h */ 9 | BRUBECK_MT_TIMER, /** ms */ 10 | BRUBECK_MT_INTERNAL_STATS 11 | }; 12 | 13 | enum brubeck_metric_mod_t { 14 | BRUBECK_MOD_RELATIVE_VALUE = 1 15 | }; 16 | 17 | enum brubeck_aggregate_t { 18 | BRUBECK_AG_LAST, 19 | BRUBECK_AG_SUM, 20 | BRUBECK_AG_MIN, 21 | BRUBECK_AG_MAX, 22 | BRUBECK_AG_AVERAGE 23 | }; 24 | 25 | enum { 26 | BRUBECK_EXPIRE_DISABLED = 0, 27 | BRUBECK_EXPIRE_INACTIVE = 1, 28 | BRUBECK_EXPIRE_ACTIVE = 2 29 | }; 30 | 31 | struct brubeck_metric { 32 | struct brubeck_metric *next; 33 | 34 | #ifdef BRUBECK_METRICS_FLOW 35 | uint64_t flow; 36 | #endif 37 | 38 | pthread_spinlock_t lock; 39 | uint16_t key_len; 40 | uint8_t type; 41 | uint8_t expire; 42 | 43 | union { 44 | struct { 45 | value_t value; 46 | } gauge, meter; 47 | struct { 48 | value_t value, previous; 49 | } counter; 50 | struct brubeck_histo histogram; 51 | void *other; 52 | } as; 53 | 54 | char key[]; 55 | }; 56 | 57 | typedef void (*brubeck_sample_cb)( 58 | const char *key, 59 | value_t value, 60 | void *backend); 61 | 62 | void brubeck_metric_sample(struct brubeck_metric *metric, brubeck_sample_cb cb, void *backend); 63 | void brubeck_metric_record(struct brubeck_metric *metric, value_t value, value_t sample_rate, uint8_t modifiers); 64 | 65 | struct brubeck_metric *brubeck_metric_new(struct brubeck_server *server, const char *, size_t, uint8_t); 66 | struct brubeck_metric *brubeck_metric_find(struct brubeck_server *server, const char *, size_t, uint8_t); 67 | struct brubeck_backend *brubeck_metric_shard(struct brubeck_server *server, struct brubeck_metric *); 68 | 69 | #define WITH_SUFFIX(suffix) memcpy(key + metric->key_len, suffix, strlen(suffix) + 1); 70 | 71 | #endif 72 | 73 | 74 | -------------------------------------------------------------------------------- /src/setproctitle.c: -------------------------------------------------------------------------------- 1 | /* 2 | * set process title for ps (from sendmail) 3 | * 4 | * Clobbers argv of our main procedure so ps(1) will display the title. 5 | */ 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #ifndef SPT_BUFSIZE 12 | # define SPT_BUFSIZE 2048 13 | #endif 14 | 15 | extern char **environ; 16 | 17 | static char **argv0; 18 | static int argv_lth; 19 | 20 | void initproctitle (int argc, char **argv) 21 | { 22 | int i; 23 | char **envp = environ; 24 | 25 | /* 26 | * Move the environment so we can reuse the memory. 27 | * (Code borrowed from sendmail.) 28 | * WARNING: ugly assumptions on memory layout here; 29 | * if this ever causes problems, #undef DO_PS_FIDDLING 30 | */ 31 | for (i = 0; envp[i] != NULL; i++) 32 | continue; 33 | 34 | environ = (char **) malloc(sizeof(char *) * (i + 1)); 35 | if (environ == NULL) 36 | return; 37 | 38 | for (i = 0; envp[i] != NULL; i++) 39 | if ((environ[i] = strdup(envp[i])) == NULL) 40 | return; 41 | environ[i] = NULL; 42 | 43 | argv0 = argv; 44 | if (i > 0) 45 | argv_lth = envp[i-1] + strlen(envp[i-1]) - argv0[0]; 46 | else 47 | argv_lth = argv0[argc-1] + strlen(argv0[argc-1]) - argv0[0]; 48 | } 49 | 50 | int getproctitle(char **procbuffer) 51 | { 52 | if (!argv0) 53 | return -1; 54 | 55 | memset(argv0[0], '\0', argv_lth); 56 | argv0[1] = NULL; 57 | 58 | *procbuffer = argv0[0]; 59 | return argv_lth; 60 | } 61 | 62 | void setproctitle (const char *prog, const char *txt) 63 | { 64 | int i; 65 | char buf[SPT_BUFSIZE]; 66 | 67 | if (!argv0) 68 | return; 69 | 70 | if (strlen(prog) + strlen(txt) + 5 > SPT_BUFSIZE) 71 | return; 72 | 73 | sprintf(buf, "%s -- %s", prog, txt); 74 | 75 | i = strlen(buf); 76 | if (i > argv_lth - 2) { 77 | i = argv_lth - 2; 78 | buf[i] = '\0'; 79 | } 80 | memset(argv0[0], '\0', argv_lth); /* clear the memory area */ 81 | strcpy(argv0[0], buf); 82 | 83 | argv0[1] = NULL; 84 | } 85 | -------------------------------------------------------------------------------- /test-bin/udp-stress.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | void diep(char *s) 11 | { 12 | perror(s); 13 | exit(1); 14 | } 15 | 16 | #define MAX_THREADS 4 17 | #define SERVER_IP "127.0.0.1" 18 | #define PORT 8126 19 | 20 | static uint32_t counter; 21 | 22 | static int build_packet(char *buffer) 23 | { 24 | static const char types[] = {'g', 'c', 'C', 'h'}; 25 | int stat = rand() % 128; 26 | return sprintf(buffer, "github.test.packet.%d:%d|%c", stat, rand() % 1024, types[(stat * 0x37) % 4]); 27 | } 28 | 29 | static void *report_thread(void *_) 30 | { 31 | for (;;) { 32 | uint32_t reported = __sync_lock_test_and_set(&counter, 0); 33 | printf("%d metrics/s\n", reported); 34 | sleep(1); 35 | printf("\033[F\033[J"); 36 | } 37 | } 38 | 39 | static void *spam_thread(void *_sock) 40 | { 41 | struct sockaddr_in *si_other = _sock; 42 | int s, slen = sizeof(*si_other); 43 | char packet[64]; 44 | 45 | if ((s = socket(AF_INET, SOCK_DGRAM, 0))==-1) 46 | diep("socket"); 47 | 48 | for (;;) { 49 | int len = build_packet(packet); 50 | if (sendto(s, packet, len, 0, (void *)si_other, slen) < 0) 51 | printf("C ==> DROPPED\n"); 52 | 53 | __sync_add_and_fetch(&counter, 1); 54 | } 55 | 56 | close(s); 57 | return NULL; 58 | } 59 | 60 | int main(int argc, char *argv[]) 61 | { 62 | struct sockaddr_in si_other; 63 | pthread_t threads[MAX_THREADS], report; 64 | int i; 65 | 66 | if (argc != 3) { 67 | fprintf(stderr, "Usage: 'udp-stress IP PORT'\n"); 68 | exit(-1); 69 | } 70 | 71 | srand(time(NULL)); 72 | 73 | memset(&si_other, 0, sizeof(si_other)); 74 | si_other.sin_family = AF_INET; 75 | si_other.sin_port = htons(atoi(argv[2])); 76 | if (inet_aton(argv[1], &si_other.sin_addr)==0) { 77 | fprintf(stderr, "inet_aton() failed\n"); 78 | exit(1); 79 | } 80 | 81 | pthread_create(&report, NULL, report_thread, NULL); 82 | 83 | for (i = 0; i < MAX_THREADS; ++i) 84 | pthread_create(&threads[i], NULL, spam_thread, &si_other); 85 | 86 | for (i =0; i < MAX_THREADS; ++i) 87 | pthread_join(threads[i], NULL); 88 | 89 | return 0; 90 | } 91 | -------------------------------------------------------------------------------- /src/histogram.c: -------------------------------------------------------------------------------- 1 | #include "brubeck.h" 2 | #include 3 | 4 | #define HISTO_INIT_SIZE 16 5 | 6 | void brubeck_histo_push(struct brubeck_histo *histo, value_t value, value_t sample_freq) 7 | { 8 | histo->count += sample_freq; 9 | 10 | if (histo->size == histo->alloc) { 11 | size_t new_size; 12 | 13 | if (histo->size == USHRT_MAX) 14 | return; 15 | 16 | new_size = histo->alloc * 2; 17 | 18 | if (new_size > USHRT_MAX) 19 | new_size = USHRT_MAX; 20 | if (new_size < HISTO_INIT_SIZE) 21 | new_size = HISTO_INIT_SIZE; 22 | if (new_size != histo->alloc) { 23 | histo->alloc = (uint16_t)new_size; 24 | histo->values = xrealloc(histo->values, histo->alloc * sizeof(value_t)); 25 | } 26 | } 27 | 28 | histo->values[histo->size++] = value; 29 | } 30 | 31 | static inline value_t histo_percentile(struct brubeck_histo *histo, float rank) 32 | { 33 | size_t irank = floor((rank * histo->size) + 0.5f); 34 | return histo->values[irank - 1]; 35 | } 36 | 37 | static inline value_t histo_sum(struct brubeck_histo *histo) 38 | { 39 | size_t i; 40 | value_t sum = 0.0; 41 | 42 | for (i = 0; i < histo->size; ++i) 43 | sum += histo->values[i]; 44 | 45 | return sum; 46 | } 47 | 48 | static int value_cmp(const void *a, const void *b) 49 | { 50 | const value_t va = *(const value_t *)a, vb = *(const value_t *)b; 51 | if (va < vb) 52 | return -1; 53 | if (va > vb) 54 | return 1; 55 | return 0; 56 | } 57 | 58 | static inline void histo_sort(struct brubeck_histo *histo) 59 | { 60 | qsort(histo->values, histo->size, sizeof(value_t), &value_cmp); 61 | } 62 | 63 | void brubeck_histo_sample( 64 | struct brubeck_histo_sample *sample, 65 | struct brubeck_histo *histo) 66 | { 67 | if (histo->size == 0) { 68 | memset(sample, 0x0, sizeof(struct brubeck_histo_sample)); 69 | return; 70 | } 71 | 72 | histo_sort(histo); 73 | 74 | sample->sum = histo_sum(histo); 75 | sample->min = histo->values[0]; 76 | sample->max = histo->values[histo->size - 1]; 77 | sample->mean = sample->sum / histo->size; 78 | sample->median = histo_percentile(histo, 0.5f); 79 | sample->count = histo->count; 80 | 81 | sample->percentile[PC_75] = histo_percentile(histo, 0.75f); 82 | sample->percentile[PC_95] = histo_percentile(histo, 0.95f); 83 | sample->percentile[PC_98] = histo_percentile(histo, 0.98f); 84 | sample->percentile[PC_99] = histo_percentile(histo, 0.99f); 85 | sample->percentile[PC_999] = histo_percentile(histo, 0.999f); 86 | 87 | /* empty the histogram */ 88 | histo->size = 0; 89 | histo->count = 0; 90 | } 91 | -------------------------------------------------------------------------------- /src/internal_sampler.c: -------------------------------------------------------------------------------- 1 | #include "brubeck.h" 2 | 3 | #define INTERNAL_LONGEST_KEY ".secure.from_future" 4 | 5 | void 6 | brubeck_internal__sample(struct brubeck_metric *metric, brubeck_sample_cb sample, void *opaque) 7 | { 8 | struct brubeck_internal_stats *stats = metric->as.other; 9 | uint32_t value; 10 | 11 | char *key = alloca(metric->key_len + strlen(INTERNAL_LONGEST_KEY) + 1); 12 | memcpy(key, metric->key, metric->key_len); 13 | 14 | WITH_SUFFIX(".metrics") { 15 | value = brubeck_atomic_swap(&stats->live.metrics, 0); 16 | stats->sample.metrics = value; 17 | sample(key, (value_t)value, opaque); 18 | } 19 | 20 | WITH_SUFFIX(".errors") { 21 | value = brubeck_atomic_swap(&stats->live.errors, 0); 22 | stats->sample.errors = value; 23 | sample(key, (value_t)value, opaque); 24 | } 25 | 26 | WITH_SUFFIX(".unique_keys") { 27 | value = brubeck_atomic_fetch(&stats->live.unique_keys); 28 | stats->sample.unique_keys = value; 29 | sample(key, (value_t)value, opaque); 30 | } 31 | 32 | /* Secure statsd endpoint */ 33 | WITH_SUFFIX(".secure.failed") { 34 | value = brubeck_atomic_swap(&stats->live.secure.failed, 0); 35 | stats->sample.secure.failed = value; 36 | sample(key, (value_t)value, opaque); 37 | } 38 | 39 | WITH_SUFFIX(".secure.from_future") { 40 | value = brubeck_atomic_swap(&stats->live.secure.from_future, 0); 41 | stats->sample.secure.from_future = value; 42 | sample(key, (value_t)value, opaque); 43 | } 44 | 45 | WITH_SUFFIX(".secure.delayed") { 46 | value = brubeck_atomic_swap(&stats->live.secure.delayed, 0); 47 | stats->sample.secure.delayed = value; 48 | sample(key, (value_t)value, opaque); 49 | } 50 | 51 | WITH_SUFFIX(".secure.replayed") { 52 | value = brubeck_atomic_swap(&stats->live.secure.replayed, 0); 53 | stats->sample.secure.replayed = value; 54 | sample(key, (value_t)value, opaque); 55 | } 56 | 57 | /* 58 | * Mark the metric as active so it doesn't get disabled 59 | * by the inactive metrics pruner 60 | */ 61 | metric->expire = BRUBECK_EXPIRE_ACTIVE; 62 | } 63 | 64 | void brubeck_internal__init(struct brubeck_server *server) 65 | { 66 | struct brubeck_metric *internal; 67 | struct brubeck_backend *backend; 68 | 69 | internal = brubeck_metric_new(server, 70 | server->name, strlen(server->name), 71 | BRUBECK_MT_INTERNAL_STATS); 72 | 73 | if (internal == NULL) 74 | die("Failed to initialize internal stats sampler"); 75 | 76 | internal->as.other = &server->internal_stats; 77 | 78 | backend = brubeck_metric_shard(server, internal); 79 | server->internal_stats.sample_freq = backend->sample_freq; 80 | } 81 | -------------------------------------------------------------------------------- /test-bin/statsd-secure-send.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #define DEBUG 14 | #define HMAC_KEY "750c783e6ab0b503eaa86e310a5db738" 15 | #define SHA_SIZE 32 16 | 17 | static void diep(char *s) 18 | { 19 | perror(s); 20 | exit(1); 21 | } 22 | 23 | static int build_packet(char *buffer, const char *metric) 24 | { 25 | HMAC_CTX ctx; 26 | struct timespec now; 27 | uint64_t timestamp; 28 | uint32_t nonce; 29 | unsigned int hmac_len; 30 | int metric_len = strlen(metric); 31 | 32 | RAND_pseudo_bytes((void *)&nonce, sizeof(nonce)); 33 | clock_gettime(CLOCK_REALTIME, &now); 34 | timestamp = (uint64_t)now.tv_sec - 2; /* fake delay */ 35 | 36 | HMAC_CTX_init(&ctx); 37 | HMAC_Init_ex(&ctx, HMAC_KEY, strlen(HMAC_KEY), EVP_sha256(), NULL); 38 | 39 | HMAC_Update(&ctx, (void *)×tamp, sizeof(timestamp)); 40 | HMAC_Update(&ctx, (void *)&nonce, sizeof(nonce)); 41 | HMAC_Update(&ctx, (void *)metric, metric_len); 42 | 43 | HMAC_Final(&ctx, buffer, &hmac_len); 44 | 45 | HMAC_CTX_cleanup(&ctx); 46 | 47 | memcpy(buffer + SHA_SIZE, ×tamp, sizeof(timestamp)); 48 | memcpy(buffer + SHA_SIZE + 8, &nonce, sizeof(nonce)); 49 | memcpy(buffer + SHA_SIZE + 12, metric, metric_len); 50 | 51 | #ifdef DEBUG 52 | { 53 | int i; 54 | 55 | fprintf(stderr, "HMAC: "); 56 | for (i = 0; i < SHA_SIZE; ++i) 57 | fprintf(stderr, "%02x", (unsigned char)buffer[i]); 58 | 59 | fprintf(stderr, "\nTIMESTAMP: %llu\nNONCE: %08x\nMETRIC: %s\n", 60 | (long long unsigned int)timestamp, nonce, metric); 61 | } 62 | #endif 63 | 64 | return SHA_SIZE + 12 + metric_len; 65 | } 66 | 67 | int main(int argc, char *argv[]) 68 | { 69 | struct sockaddr_in si_other; 70 | int s, len; 71 | char buffer[1024]; 72 | 73 | if (argc != 4) { 74 | fprintf(stderr, "Usage: 'udp-stress IP PORT METRIC'\n"); 75 | exit(-1); 76 | } 77 | 78 | srand(time(NULL)); 79 | 80 | memset(&si_other, 0, sizeof(si_other)); 81 | si_other.sin_family = AF_INET; 82 | si_other.sin_port = htons(atoi(argv[2])); 83 | if (inet_aton(argv[1], &si_other.sin_addr) == 0) { 84 | fprintf(stderr, "inet_aton() failed\n"); 85 | exit(1); 86 | } 87 | 88 | if ((s = socket(AF_INET, SOCK_DGRAM, 0)) < 0) 89 | diep("socket"); 90 | 91 | len = build_packet(buffer, argv[3]); 92 | sendto(s, buffer, len, 0, (void *)&si_other, sizeof(si_other)); 93 | 94 | close(s); 95 | 96 | return 0; 97 | } 98 | -------------------------------------------------------------------------------- /src/utils.h: -------------------------------------------------------------------------------- 1 | #ifndef __BRUBECK_H__ 2 | #define __BRUBECK_H__ 3 | 4 | #define likely(x) __builtin_expect((x),1) 5 | #define unlikely(x) __builtin_expect((x),0) 6 | #define ct_assert(e) ((void)sizeof(char[1 - 2*!(e)])) 7 | 8 | void url_to_inaddr2(struct sockaddr_in *addr, const char *url, int port); 9 | 10 | void sock_setnonblock(int fd); 11 | void sock_setreuse(int fd, int reuse); 12 | void sock_setreuse_port(int fd, int reuse); 13 | void sock_enlarge_out(int fd); 14 | void sock_enlarge_in(int fd); 15 | 16 | char *find_substr(const char *s, const char *find, size_t slen); 17 | 18 | int brubeck_itoa(char *ptr, uint32_t number); 19 | int brubeck_ftoa(char *outbuf, float f); 20 | 21 | static inline int starts_with(const char *str, const char *prefix) 22 | { 23 | for (; ; str++, prefix++) 24 | if (!*prefix) 25 | return 1; 26 | else if (*str != *prefix) 27 | return 0; 28 | } 29 | 30 | static inline void *xmalloc(size_t size) 31 | { 32 | void *ptr = malloc(size); 33 | 34 | if (unlikely(ptr == NULL)) 35 | die("oom"); 36 | 37 | return ptr; 38 | } 39 | 40 | static inline void *xcalloc(size_t n, size_t size) 41 | { 42 | void *ptr = calloc(n, size); 43 | 44 | if (unlikely(ptr == NULL)) 45 | die("oom"); 46 | 47 | return ptr; 48 | } 49 | 50 | static inline void *xrealloc(void *ptr, size_t size) 51 | { 52 | void *new_ptr = realloc(ptr, size); 53 | 54 | if (unlikely(new_ptr == NULL)) 55 | die("oom"); 56 | 57 | return new_ptr; 58 | } 59 | 60 | #define brubeck_atomic_inc(P) __sync_add_and_fetch((P), 1) 61 | #define brubeck_atomic_dec(P) __sync_add_and_fetch((P), -1) 62 | #define brubeck_atomic_add(P, V) __sync_add_and_fetch((P), (V)) 63 | #define brubeck_atomic_swap(P, V) __sync_lock_test_and_set((P), (V)) 64 | #define brubeck_atomic_fetch(P) __sync_add_and_fetch((P), 0) 65 | 66 | /* Compile read-write barrier */ 67 | #define brubeck_barrier() __sync_synchronize() 68 | 69 | void initproctitle (int argc, char **argv); 70 | int getproctitle(char **procbuffer); 71 | void setproctitle (const char *prog, const char *txt); 72 | 73 | static inline ssize_t xwrite(int fd, const void *buf, size_t len) 74 | { 75 | ssize_t nr; 76 | while (1) { 77 | nr = write(fd, buf, len); 78 | if ((nr < 0) && (errno == EAGAIN || errno == EINTR)) 79 | continue; 80 | return nr; 81 | } 82 | } 83 | 84 | static inline ssize_t write_in_full(int fd, const void *buf, size_t count) 85 | { 86 | const char *p = buf; 87 | ssize_t total = 0; 88 | 89 | while (count > 0) { 90 | ssize_t written = xwrite(fd, p, count); 91 | if (written < 0) 92 | return -1; 93 | if (!written) { 94 | errno = ENOSPC; 95 | return -1; 96 | } 97 | count -= written; 98 | p += written; 99 | total += written; 100 | } 101 | 102 | return total; 103 | } 104 | 105 | #define json_unpack_or_die(json, fmt, ...) { \ 106 | json_error_t _error_j; \ 107 | if (json_unpack_ex(json, &_error_j, 0, fmt, __VA_ARGS__) < 0) \ 108 | die("config error: %s", _error_j.text); } 109 | 110 | extern uint32_t CityHash32(const char *s, size_t len); 111 | 112 | #endif 113 | -------------------------------------------------------------------------------- /src/ht.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "brubeck.h" 4 | #include "ck_ht.h" 5 | #include "ck_malloc.h" 6 | 7 | struct brubeck_hashtable_t { 8 | ck_ht_t table; 9 | pthread_mutex_t write_mutex; 10 | }; 11 | 12 | static void * 13 | ht_malloc(size_t r) 14 | { 15 | return xmalloc(r); 16 | } 17 | 18 | static void 19 | ht_free(void *p, size_t b, bool r) 20 | { 21 | free(p); 22 | } 23 | 24 | static struct ck_malloc ALLOCATOR = { 25 | .malloc = ht_malloc, 26 | .free = ht_free 27 | }; 28 | 29 | brubeck_hashtable_t * 30 | brubeck_hashtable_new(const uint64_t size) 31 | { 32 | brubeck_hashtable_t *ht = xmalloc(sizeof(brubeck_hashtable_t)); 33 | pthread_mutex_init(&ht->write_mutex, NULL); 34 | 35 | if (!ck_ht_init(&ht->table, CK_HT_MODE_BYTESTRING, 36 | NULL, &ALLOCATOR, (uint64_t)size, 0xDEADBEEF)) { 37 | free(ht); 38 | return NULL; 39 | } 40 | 41 | return ht; 42 | } 43 | 44 | void 45 | brubeck_hashtable_free(brubeck_hashtable_t *ht) 46 | { 47 | /* no-op */ 48 | } 49 | 50 | struct brubeck_metric * 51 | brubeck_hashtable_find(brubeck_hashtable_t *ht, const char *key, uint16_t key_len) 52 | { 53 | ck_ht_hash_t h; 54 | ck_ht_entry_t entry; 55 | 56 | ck_ht_hash(&h, &ht->table, key, key_len); 57 | ck_ht_entry_key_set(&entry, key, key_len); 58 | 59 | if (ck_ht_get_spmc(&ht->table, h, &entry)) 60 | return ck_ht_entry_value(&entry); 61 | 62 | return NULL; 63 | } 64 | 65 | bool 66 | brubeck_hashtable_insert(brubeck_hashtable_t *ht, const char *key, uint16_t key_len, struct brubeck_metric *val) 67 | { 68 | ck_ht_hash_t h; 69 | ck_ht_entry_t entry; 70 | bool result; 71 | 72 | ck_ht_hash(&h, &ht->table, key, key_len); 73 | ck_ht_entry_set(&entry, h, key, key_len, val); 74 | 75 | pthread_mutex_lock(&ht->write_mutex); 76 | result = ck_ht_put_spmc(&ht->table, h, &entry); 77 | pthread_mutex_unlock(&ht->write_mutex); 78 | 79 | return result; 80 | } 81 | 82 | size_t 83 | brubeck_hashtable_size(brubeck_hashtable_t *ht) 84 | { 85 | size_t len; 86 | 87 | pthread_mutex_lock(&ht->write_mutex); 88 | len = ck_ht_count(&ht->table); 89 | pthread_mutex_unlock(&ht->write_mutex); 90 | 91 | return len; 92 | } 93 | 94 | void 95 | brubeck_hashtable_foreach(brubeck_hashtable_t *ht, void (*callback)(struct brubeck_metric *, void *), void *payload) 96 | { 97 | ck_ht_iterator_t iterator = CK_HT_ITERATOR_INITIALIZER; 98 | ck_ht_entry_t *entry; 99 | 100 | pthread_mutex_lock(&ht->write_mutex); 101 | 102 | while (ck_ht_next(&ht->table, &iterator, &entry)) 103 | callback(ck_ht_entry_value(entry), payload); 104 | 105 | pthread_mutex_unlock(&ht->write_mutex); 106 | } 107 | 108 | struct brubeck_metric ** 109 | brubeck_hashtable_to_a(brubeck_hashtable_t *ht, size_t *length) 110 | { 111 | ck_ht_iterator_t iterator = CK_HT_ITERATOR_INITIALIZER; 112 | ck_ht_entry_t *entry; 113 | struct brubeck_metric **array; 114 | size_t i = 0; 115 | 116 | pthread_mutex_lock(&ht->write_mutex); 117 | *length = ck_ht_count(&ht->table); 118 | array = xmalloc(*length * sizeof(void *)); 119 | 120 | while (ck_ht_next(&ht->table, &iterator, &entry)) 121 | array[i++] = ck_ht_entry_value(entry); 122 | 123 | pthread_mutex_unlock(&ht->write_mutex); 124 | 125 | assert(*length == i); 126 | return array; 127 | } 128 | -------------------------------------------------------------------------------- /tests/statsd_msg.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "sput.h" 4 | #include "brubeck.h" 5 | 6 | static void must_parse(const char *msg_text, double value, double sample, uint8_t modifiers) 7 | { 8 | struct brubeck_statsd_msg msg; 9 | char buffer[128]; 10 | size_t len = strlen(msg_text); 11 | memcpy(buffer, msg_text, len); 12 | 13 | sput_fail_unless(brubeck_statsd_msg_parse(&msg, buffer, buffer + len) == 0, msg_text); 14 | sput_fail_unless(value == msg.value, "msg.value == expected"); 15 | sput_fail_unless(sample == msg.sample_freq, "msg.sample_rate == expected"); 16 | sput_fail_unless(modifiers == msg.modifiers, "msg.modifiers == expected"); 17 | } 18 | 19 | static void must_not_parse(const char *msg_text) 20 | { 21 | struct brubeck_statsd_msg msg; 22 | char buffer[128]; 23 | size_t len = strlen(msg_text); 24 | memcpy(buffer, msg_text, len); 25 | 26 | sput_fail_unless(brubeck_statsd_msg_parse(&msg, buffer, buffer + len) < 0, msg_text); 27 | } 28 | 29 | void test_statsd_msg__parse_strings(void) 30 | { 31 | must_parse("github.auth.fingerprint.sha1:1|c", 1, 1.0, 0); 32 | must_parse("github.auth.fingerprint.sha1:1|c|@0.1", 1, 10.0, 0); 33 | must_parse("github.auth.fingerprint.sha1:1|g", 1, 1.0, 0); 34 | must_parse("lol:1|ms", 1, 1.0, 0); 35 | must_parse("this.is.sparta:199812|C", 199812, 1.0, 0); 36 | must_parse("this.is.sparta:0012|h", 12, 1.0, 0); 37 | must_parse("this.is.sparta:23.23|g", 23.23, 1.0, 0); 38 | must_parse("this.is.sparta:0.232030|g", 0.23203, 1.0, 0); 39 | must_parse("this.are.some.floats:1234567.89|g", 1234567.89, 1.0, 0); 40 | must_parse("this.are.some.floats:1234567.89|g|@0.025", 1234567.89, 40.0, 0); 41 | must_parse("this.are.some.floats:1234567.89|g|@0.25", 1234567.89, 4.0, 0); 42 | must_parse("this.are.some.floats:1234567.89|g|@0.01", 1234567.89, 100.0, 0); 43 | must_parse("this.are.some.floats:1234567.89|g|@000.0100", 1234567.89, 100.0, 0); 44 | must_parse("this.are.some.floats:1234567.89|g|@1.0", 1234567.89, 1.0, 0); 45 | must_parse("this.are.some.floats:1234567.89|g|@1", 1234567.89, 1.0, 0); 46 | must_parse("this.are.some.floats:1234567.89|g|@1.", 1234567.89, 1.0, 0); 47 | must_parse("this.are.some.floats:|g", 0.0, 1.0, 0); 48 | must_parse("this.are.some.floats:1234567.89|g", 1234567.89, 1.0, 0); 49 | must_parse("gauge.increment:+1|g", 1, 1.0, BRUBECK_MOD_RELATIVE_VALUE); 50 | must_parse("gauge.decrement:-1|g", -1, 1.0, BRUBECK_MOD_RELATIVE_VALUE); 51 | 52 | must_not_parse("this.are.some.floats:12.89.23|g"); 53 | must_not_parse("this.are.some.floats:12.89|a"); 54 | must_not_parse("this.are.some.floats:12.89|msdos"); 55 | must_not_parse("this.are.some.floats:12.89g|g"); 56 | must_not_parse("this.are.some.floats:12.89|"); 57 | must_not_parse("this.are.some.floats:12.89"); 58 | must_not_parse("this.are.some.floats:12.89 |g"); 59 | must_not_parse("this.are.some.floats|g"); 60 | must_not_parse("this.are.some.floats:1.0|g|1.0"); 61 | must_not_parse("this.are.some.floats:1.0|g|0.1"); 62 | must_not_parse("this.are.some.floats:1.0|g|@0.1.1"); 63 | must_not_parse("this.are.some.floats:1.0|g|@0.1@"); 64 | must_not_parse("this.are.some.floats:1.0|g|@0.1125.2"); 65 | must_not_parse("this.are.some.floats:1.0|g|@0.1125.2"); 66 | must_not_parse("this.are.some.floats:1.0|g|@1.23"); 67 | must_not_parse("this.are.some.floats:1.0|g|@3.0"); 68 | must_not_parse("this.are.some.floats:1.0|g|@-3.0"); 69 | must_not_parse("this.are.some.floats:1.0|g|@-1.0"); 70 | must_not_parse("this.are.some.floats:1.0|g|@-0.23"); 71 | must_not_parse("this.are.some.floats:1.0|g|@0.0"); 72 | must_not_parse("this.are.some.floats:1.0|g|@0"); 73 | } 74 | -------------------------------------------------------------------------------- /src/city.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #ifdef _MSC_VER 5 | # include 6 | # define bswap_32(x) _byteswap_ulong(x) 7 | # define bswap_64(x) _byteswap_uint64(x) 8 | #elif defined(__APPLE__) 9 | # include 10 | # define bswap_32(x) OSSwapInt32(x) 11 | # define bswap_64(x) OSSwapInt64(x) 12 | #else 13 | # include 14 | #endif 15 | 16 | static uint32_t read32(const char *p) 17 | { 18 | uint32_t result; 19 | memcpy(&result, p, sizeof(result)); 20 | return result; 21 | } 22 | 23 | // Magic numbers for 32-bit hashing. Copied from murmur3. 24 | static const uint32_t c1 = 0xcc9e2d51; 25 | static const uint32_t c2 = 0x1b873593; 26 | 27 | // A 32-bit to 32-bit integer hash copied from murmur3. 28 | static uint32_t fmix(uint32_t h) 29 | { 30 | h ^= h >> 16; 31 | h *= 0x85ebca6b; 32 | h ^= h >> 13; 33 | h *= 0xc2b2ae35; 34 | h ^= h >> 16; 35 | return h; 36 | } 37 | 38 | static uint32_t ror32(uint32_t val, int shift) 39 | { 40 | return shift == 0 ? val : ((val >> shift) | (val << (32 - shift))); 41 | } 42 | 43 | #define swap(x, y) do { typeof(x) aux = x; x = y; y = aux; } while (0) 44 | 45 | #define PERMUTE3(a, b, c) do { swap(a, b); swap(a, c); } while (0) 46 | 47 | static uint32_t mur(uint32_t a, uint32_t h) 48 | { 49 | // Helper from murmur3 for combining two 32-bit values. 50 | a *= c1; 51 | a = ror32(a, 17); 52 | a *= c2; 53 | h ^= a; 54 | h = ror32(h, 19); 55 | return h * 5 + 0xe6546b64; 56 | } 57 | 58 | static uint32_t Hash32Len13to24(const char *s, size_t len) 59 | { 60 | uint32_t a = read32(s - 4 + (len >> 1)); 61 | uint32_t b = read32(s + 4); 62 | uint32_t c = read32(s + len - 8); 63 | uint32_t d = read32(s + (len >> 1)); 64 | uint32_t e = read32(s); 65 | uint32_t f = read32(s + len - 4); 66 | uint32_t h = len; 67 | 68 | return fmix(mur(f, mur(e, mur(d, mur(c, mur(b, mur(a, h))))))); 69 | } 70 | 71 | static uint32_t Hash32Len0to4(const char *s, size_t len) 72 | { 73 | uint32_t b = 0; 74 | uint32_t c = 9; 75 | int i; 76 | 77 | for (i = 0; i < len; i++) { 78 | b = b * c1 + s[i]; 79 | c ^= b; 80 | } 81 | return fmix(mur(b, mur(len, c))); 82 | } 83 | 84 | static uint32_t Hash32Len5to12(const char *s, size_t len) 85 | { 86 | uint32_t a = len, b = len * 5, c = 9, d = b; 87 | a += read32(s); 88 | b += read32(s + len - 4); 89 | c += read32(s + ((len >> 1) & 4)); 90 | return fmix(mur(c, mur(b, mur(a, d)))); 91 | } 92 | 93 | uint32_t CityHash32(const char *s, size_t len) 94 | { 95 | size_t iters; 96 | uint32_t a0, a1, a2, a3, a4; 97 | uint32_t h, g, f; 98 | 99 | if (len <= 24) { 100 | return len <= 12 ? 101 | (len <= 4 ? Hash32Len0to4(s, len) : Hash32Len5to12(s, len)) : 102 | Hash32Len13to24(s, len); 103 | } 104 | 105 | h = len; 106 | g = c1 * len; 107 | f = g; 108 | 109 | a0 = ror32(read32(s + len - 4) * c1, 17) * c2; 110 | a1 = ror32(read32(s + len - 8) * c1, 17) * c2; 111 | a2 = ror32(read32(s + len - 16) * c1, 17) * c2; 112 | a3 = ror32(read32(s + len - 12) * c1, 17) * c2; 113 | a4 = ror32(read32(s + len - 20) * c1, 17) * c2; 114 | 115 | h ^= a0; 116 | h = ror32(h, 19); 117 | h = h * 5 + 0xe6546b64; 118 | h ^= a2; 119 | h = ror32(h, 19); 120 | h = h * 5 + 0xe6546b64; 121 | g ^= a1; 122 | g = ror32(g, 19); 123 | g = g * 5 + 0xe6546b64; 124 | g ^= a3; 125 | g = ror32(g, 19); 126 | g = g * 5 + 0xe6546b64; 127 | f += a4; 128 | f = ror32(f, 19); 129 | f = f * 5 + 0xe6546b64; 130 | 131 | iters = (len - 1) / 20; 132 | do { 133 | uint32_t a0 = ror32(read32(s) * c1, 17) * c2; 134 | uint32_t a1 = read32(s + 4); 135 | uint32_t a2 = ror32(read32(s + 8) * c1, 17) * c2; 136 | uint32_t a3 = ror32(read32(s + 12) * c1, 17) * c2; 137 | uint32_t a4 = read32(s + 16); 138 | h ^= a0; 139 | h = ror32(h, 18); 140 | h = h * 5 + 0xe6546b64; 141 | f += a1; 142 | f = ror32(f, 19); 143 | f = f * c1; 144 | g += a2; 145 | g = ror32(g, 18); 146 | g = g * 5 + 0xe6546b64; 147 | h ^= a3 + a1; 148 | h = ror32(h, 19); 149 | h = h * 5 + 0xe6546b64; 150 | g ^= a4; 151 | g = bswap_32(g) * 5; 152 | h += a4 * 5; 153 | h = bswap_32(h); 154 | f += a0; 155 | PERMUTE3(f, h, g); 156 | s += 20; 157 | } while (--iters != 0); 158 | g = ror32(g, 11) * c1; 159 | g = ror32(g, 17) * c1; 160 | f = ror32(f, 11) * c1; 161 | f = ror32(f, 17) * c1; 162 | h = ror32(h + g, 19); 163 | h = h * 5 + 0xe6546b64; 164 | h = ror32(h, 17) * c1; 165 | h = ror32(h + f, 19); 166 | h = h * 5 + 0xe6546b64; 167 | h = ror32(h, 17) * c1; 168 | return h; 169 | } 170 | -------------------------------------------------------------------------------- /src/utils.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include "brubeck.h" 8 | 9 | #ifdef __gnu_linux__ 10 | #define LARGE_SOCK_SIZE 33554431 11 | #else 12 | #define LARGE_SOCK_SIZE 4096 13 | #endif 14 | 15 | char *find_substr(const char *s, const char *find, size_t slen) 16 | { 17 | char c, sc; 18 | size_t len; 19 | 20 | if ((c = *find++) != '\0') { 21 | len = strlen(find); 22 | do { 23 | do { 24 | if ((sc = *s++) == '\0' || slen-- < 1) 25 | return NULL; 26 | } while (sc != c); 27 | 28 | if (len > slen) 29 | return NULL; 30 | 31 | } while (strncmp(s, find, len) != 0); 32 | s--; 33 | } 34 | return (char *)s; 35 | } 36 | 37 | void sock_setnonblock(int fd) 38 | { 39 | int flags; 40 | 41 | flags = fcntl(fd, F_GETFL); 42 | flags |= O_NONBLOCK; 43 | 44 | if (fcntl(fd, F_SETFL, flags) < 0) 45 | die("Failed to set O_NONBLOCK"); 46 | } 47 | 48 | void sock_setreuse(int fd, int reuse) 49 | { 50 | if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)) == -1) 51 | die("Failed to set SO_REUSEADDR"); 52 | } 53 | 54 | void sock_enlarge_in(int fd) 55 | { 56 | int bs = LARGE_SOCK_SIZE; 57 | 58 | if (setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &bs, sizeof(bs)) == -1) 59 | die("Failed to set SO_RCVBUF"); 60 | } 61 | 62 | void sock_enlarge_out(int fd) 63 | { 64 | int bs = LARGE_SOCK_SIZE; 65 | 66 | if (setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &bs, sizeof(bs)) == -1) 67 | die("Failed to set SO_SNDBUF"); 68 | } 69 | 70 | void sock_setreuse_port(int fd, int reuse) 71 | { 72 | #ifdef SO_REUSEPORT 73 | if (setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &reuse, sizeof(reuse)) == -1) 74 | die("failed to set SO_REUSEPORT"); 75 | #endif 76 | } 77 | 78 | void url_to_inaddr2(struct sockaddr_in *addr, const char *url, int port) 79 | { 80 | memset(addr, 0x0, sizeof(struct sockaddr_in)); 81 | 82 | if (url) { 83 | struct addrinfo hints; 84 | struct addrinfo *result, *rp; 85 | 86 | memset(&hints, 0, sizeof(struct addrinfo)); 87 | hints.ai_family = AF_INET; 88 | 89 | if (getaddrinfo(url, NULL, &hints, &result) != 0) 90 | die("failed to resolve address '%s'", url); 91 | 92 | /* Look for the first IPv4 address we can find */ 93 | for (rp = result; rp; rp = rp->ai_next) { 94 | if (result->ai_family == AF_INET && 95 | result->ai_addrlen == sizeof(struct sockaddr_in)) 96 | break; 97 | } 98 | 99 | if (!rp) 100 | die("address format not supported"); 101 | 102 | memcpy(addr, rp->ai_addr, rp->ai_addrlen); 103 | addr->sin_port = htons(port); 104 | 105 | freeaddrinfo(result); 106 | } else { 107 | addr->sin_family = AF_INET; 108 | addr->sin_port = htons(port); 109 | addr->sin_addr.s_addr = htonl(INADDR_ANY); 110 | } 111 | } 112 | 113 | #define FLOAT_PRECISION 4 114 | 115 | int brubeck_itoa(char *ptr, uint32_t number) 116 | { 117 | char *origin = ptr; 118 | int size; 119 | 120 | do { 121 | *ptr++ = '0' + (number % 10); 122 | number /= 10; 123 | } while (number); 124 | 125 | size = ptr - origin; 126 | ptr--; 127 | 128 | while (origin < ptr) { 129 | char t = *ptr; 130 | *ptr-- = *origin; 131 | *origin++ = t; 132 | } 133 | 134 | return size; 135 | } 136 | 137 | int brubeck_ftoa(char *outbuf, float f) 138 | { 139 | uint64_t mantissa, int_part, frac_part; 140 | int safe_shift; 141 | uint64_t safe_mask; 142 | short exp2; 143 | char *p; 144 | 145 | union { 146 | int L; 147 | float F; 148 | } x; 149 | 150 | x.F = f; 151 | p = outbuf; 152 | 153 | exp2 = (unsigned char)(x.L >> 23) - 127; 154 | mantissa = (x.L & 0xFFFFFF) | 0x800000; 155 | frac_part = 0; 156 | int_part = 0; 157 | 158 | if (x.L < 0) { 159 | *p++ = '-'; 160 | } 161 | 162 | if (exp2 < -36) { 163 | *p++ = '0'; 164 | goto END; 165 | } 166 | 167 | safe_shift = -(exp2 + 1); 168 | safe_mask = 0xFFFFFFFFFFFFFFFFULL >>(64 - 24 - safe_shift); 169 | 170 | if (exp2 >= 64) { 171 | int_part = ULONG_MAX; 172 | } else if (exp2 >= 23) { 173 | int_part = mantissa << (exp2 - 23); 174 | } else if (exp2 >= 0) { 175 | int_part = mantissa >> (23 - exp2); 176 | frac_part = (mantissa) & safe_mask; 177 | } else /* if (exp2 < 0) */ { 178 | frac_part = (mantissa & 0xFFFFFF); 179 | } 180 | 181 | if (int_part == 0) { 182 | *p++ = '0'; 183 | } else { 184 | p += brubeck_itoa(p, int_part); 185 | } 186 | 187 | if (frac_part != 0) { 188 | int m; 189 | 190 | *p++ = '.'; 191 | 192 | for (m = 0; m < FLOAT_PRECISION; m++) { 193 | frac_part = (frac_part << 3) + (frac_part << 1); 194 | *p++ = (frac_part >> (24 + safe_shift)) + '0'; 195 | frac_part &= safe_mask; 196 | } 197 | 198 | for (; p[-1] == '0'; --p) {} 199 | 200 | if (p[-1] == '.') { 201 | --p; 202 | } 203 | } 204 | 205 | END: 206 | *p = 0; 207 | return p - outbuf; 208 | } 209 | -------------------------------------------------------------------------------- /test-bin/balancer.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #define MAX_PACKET_SIZE 512 15 | #define MAX_THREADS 4 16 | #define MAX_FANOUT 4 17 | 18 | #ifdef __gnu_linux__ 19 | #define LARGE_SOCK_SIZE 33554431 20 | #else 21 | #define LARGE_SOCK_SIZE 4096 22 | #endif 23 | 24 | static struct { 25 | unsigned int fanout; 26 | int in_socket; 27 | struct sockaddr_in in_addr; 28 | struct { 29 | int socket; 30 | struct sockaddr_in addr; 31 | } out[MAX_FANOUT]; 32 | pthread_t threads[MAX_THREADS]; 33 | } _balancer; 34 | 35 | static void diep(char *s) 36 | { 37 | perror(s); 38 | exit(1); 39 | } 40 | 41 | int sock_enlarge_in(int fd) 42 | { 43 | int bs = LARGE_SOCK_SIZE; 44 | 45 | if (setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &bs, sizeof(bs)) == -1) 46 | diep("setsockopt"); 47 | 48 | return 0; 49 | } 50 | 51 | int sock_enlarge_out(int fd) 52 | { 53 | int bs = LARGE_SOCK_SIZE; 54 | 55 | if (setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &bs, sizeof(bs)) == -1) 56 | diep("setsockopt"); 57 | 58 | return 0; 59 | } 60 | 61 | #if 0 62 | static void *balancer__worker(void *_) 63 | { 64 | const socklen_t addrlen = sizeof(struct sockaddr_in); 65 | unsigned int i; 66 | 67 | struct iovec iovecs[SIMULTANEOUS_PACKETS]; 68 | struct mmsghdr msgs[SIMULTANEOUS_PACKETS]; 69 | 70 | memset(msgs, 0x0, sizeof(msgs)); 71 | 72 | for (i = 0; i < SIMULTANEOUS_PACKETS; ++i) { 73 | iovecs[i].iov_base = xmalloc(MAX_PACKET_SIZE); 74 | iovecs[i].iov_len = MAX_PACKET_SIZE; 75 | msgs[i].msg_hdr.msg_iov = &iovecs[i]; 76 | msgs[i].msg_hdr.msg_iovlen = 1; 77 | } 78 | 79 | for (;;) { 80 | int res = recvmmsg(_balancer.in_socket, msgs, SIMULTANEOUS_PACKETS, 0, NULL); 81 | 82 | if (res <= 0) 83 | continue; 84 | 85 | for (i = 0; i < SIMULTANEOUS_PACKETS; ++i) { 86 | const char *buf = msgs[i].msg_hdr.msg_iov->iov_base; 87 | size_t len = msgs[i].msg_len; 88 | 89 | for (j = 0; j < FANOUT; ++j) 90 | sendto(_balancer.out[j].socket, buf, len, 0, &_balancer.out[j].addr, addrlen); 91 | } 92 | } 93 | 94 | return NULL; 95 | } 96 | #endif 97 | 98 | static void *balancer__worker(void *_) 99 | { 100 | const socklen_t addrlen = sizeof(struct sockaddr_in); 101 | unsigned int j; 102 | 103 | char buffer[MAX_PACKET_SIZE]; 104 | 105 | for (;;) { 106 | int res = recvfrom(_balancer.in_socket, buffer, 107 | sizeof(buffer), 0, NULL, NULL); 108 | 109 | if (res <= 0) 110 | continue; 111 | 112 | for (j = 0; j < _balancer.fanout; ++j) 113 | sendto(_balancer.out[j].socket, buffer, (size_t)res, 0, 114 | (struct sockaddr *)&_balancer.out[j].addr, addrlen); 115 | } 116 | 117 | return NULL; 118 | } 119 | 120 | static void init_sock(struct sockaddr_in *addr, int *sock, char *address_s) 121 | { 122 | char *port_s = strchr(address_s, ':'); 123 | int port; 124 | 125 | if (port_s == NULL) { 126 | fprintf(stderr, "Invalid address: %s\n", address_s); 127 | exit(1); 128 | } 129 | 130 | *port_s = '\0'; 131 | port = atoi(port_s + 1); 132 | 133 | addr->sin_family = AF_INET; 134 | addr->sin_port = htons(port); 135 | addr->sin_addr.s_addr = (strcmp(address_s, "0.0.0.0")) ? 136 | inet_addr(address_s) : htonl(INADDR_ANY); 137 | 138 | *sock = socket(AF_INET, SOCK_DGRAM, 0); 139 | if (*sock < 0) 140 | diep("socket"); 141 | } 142 | 143 | static void usage(const char *progname) 144 | { 145 | printf("Usage: %s [--listen addr] addr...\n", progname); 146 | exit(-1); 147 | } 148 | 149 | int main(int argc, char *argv[]) 150 | { 151 | int i = 1; 152 | 153 | if (argc > 1 && strcmp(argv[1], "--listen") == 0) { 154 | if (argc == 2) 155 | usage(argv[0]); 156 | 157 | init_sock(&_balancer.in_addr, &_balancer.in_socket, argv[2]); 158 | sock_enlarge_in(_balancer.in_socket); 159 | 160 | if (bind(_balancer.in_socket, (struct sockaddr *)&_balancer.in_addr, 161 | sizeof(_balancer.in_addr)) < 0) 162 | diep("bind"); 163 | 164 | i = 3; 165 | } 166 | 167 | for (; i < argc; ++i) { 168 | if (_balancer.fanout == MAX_FANOUT) 169 | usage(argv[0]); 170 | 171 | init_sock( 172 | &_balancer.out[_balancer.fanout].addr, 173 | &_balancer.out[_balancer.fanout].socket, 174 | argv[i]); 175 | 176 | sock_enlarge_out(_balancer.out[_balancer.fanout].socket); 177 | _balancer.fanout++; 178 | } 179 | 180 | if (!_balancer.fanout) { 181 | fprintf(stderr, "No fanout addresses to proxy.\n"); 182 | exit(1); 183 | } 184 | 185 | for (i = 0; i < MAX_THREADS; ++i) 186 | pthread_create(&_balancer.threads[i], NULL, &balancer__worker, NULL); 187 | 188 | for (i =0; i < MAX_THREADS; ++i) 189 | pthread_join(_balancer.threads[i], NULL); 190 | 191 | return 0; 192 | } 193 | -------------------------------------------------------------------------------- /tests/histogram.c: -------------------------------------------------------------------------------- 1 | #include "brubeck.h" 2 | #include "sput.h" 3 | #include "thread_helper.h" 4 | #include 5 | 6 | #define ITERS (4096 * 8) 7 | 8 | struct histo_test { 9 | struct brubeck_histo h; 10 | pthread_spinlock_t lock; 11 | }; 12 | 13 | static void *thread_histo(void *ptr) 14 | { 15 | struct histo_test *t = ptr; 16 | size_t i; 17 | 18 | for (i = 0; i < ITERS; ++i) { 19 | if (rand() % 2 == 0) { 20 | struct brubeck_histo_sample hsample; 21 | pthread_spin_lock(&t->lock); 22 | { 23 | brubeck_histo_sample(&hsample, &t->h); 24 | } 25 | pthread_spin_unlock(&t->lock); 26 | } else { 27 | pthread_spin_lock(&t->lock); 28 | { 29 | brubeck_histo_push(&t->h, 0.42, 1.0); 30 | } 31 | pthread_spin_unlock(&t->lock); 32 | } 33 | } 34 | 35 | return NULL; 36 | } 37 | 38 | void test_histogram__sampling(void) 39 | { 40 | struct histo_test test; 41 | 42 | memset(&test.h, 0x0, sizeof(test.h)); 43 | pthread_spin_init(&test.lock, 0); 44 | spawn_threads(&thread_histo, &test); 45 | } 46 | 47 | void test_histogram__single_element(void) 48 | { 49 | struct brubeck_histo h; 50 | struct brubeck_histo_sample sample; 51 | 52 | memset(&h, 0x0, sizeof(h)); 53 | 54 | brubeck_histo_push(&h, 42.0, 1.0); 55 | sput_fail_unless(h.size == 1, "histogram size"); 56 | sput_fail_unless(h.count == 1, "histogram value count"); 57 | 58 | brubeck_histo_sample(&sample, &h); 59 | 60 | sput_fail_unless(sample.min == 42.0, "sample.min"); 61 | sput_fail_unless(sample.max == 42.0, "sample.max"); 62 | sput_fail_unless(sample.percentile[3] == 42.0, "sample.percentile[3]"); 63 | sput_fail_unless(sample.mean == 42.0, "sample.mean"); 64 | sput_fail_unless(sample.count == 1, "sample.count"); 65 | sput_fail_unless(sample.sum == 42.0, "sample.sum"); 66 | } 67 | 68 | void test_histogram__large_range(void) 69 | { 70 | struct brubeck_histo h; 71 | struct brubeck_histo_sample sample; 72 | 73 | memset(&h, 0x0, sizeof(h)); 74 | 75 | brubeck_histo_push(&h, 1.3e12, 1.0); 76 | brubeck_histo_push(&h, 42.0, 1.0); 77 | brubeck_histo_push(&h, 42.0, 1.0); 78 | 79 | brubeck_histo_sample(&sample, &h); 80 | 81 | sput_fail_unless(sample.min == 42.0, "sample.min"); 82 | sput_fail_unless(sample.max == 1.3e12, "sample.max"); 83 | sput_fail_unless(sample.median == 42.0, "sample.median"); 84 | } 85 | 86 | void test_histogram__multisamples(void) 87 | { 88 | struct brubeck_histo h; 89 | struct brubeck_histo_sample sample; 90 | size_t i, j; 91 | 92 | memset(&h, 0x0, sizeof(h)); 93 | 94 | for (i = 0; i < 8; ++i) { 95 | for (j = 0; j < 128; ++j) 96 | brubeck_histo_push(&h, (double)(j + 1), 1.0); 97 | 98 | sput_fail_unless(h.size == 128, "histogram size"); 99 | sput_fail_unless(h.count == 128, "histogram value count"); 100 | 101 | brubeck_histo_sample(&sample, &h); 102 | 103 | sput_fail_unless(sample.min == 1.0, "sample.min"); 104 | sput_fail_unless(sample.max == 128.0, "sample.max"); 105 | sput_fail_unless(sample.percentile[3] == 127.0, "sample.percentile[3]"); 106 | sput_fail_unless(sample.mean == 64.5, "sample.mean"); 107 | sput_fail_unless(sample.count == 128, "sample.count"); 108 | sput_fail_unless(sample.sum == 8256.0, "sample.sum"); 109 | } 110 | } 111 | 112 | void test_histogram__with_sample_rate(void) 113 | { 114 | struct brubeck_histo h; 115 | struct brubeck_histo_sample sample; 116 | size_t j; 117 | 118 | memset(&h, 0x0, sizeof(h)); 119 | 120 | for (j = 0; j < 128; ++j) 121 | brubeck_histo_push(&h, (double)(j + 1), 10.0); 122 | 123 | sput_fail_unless(h.size == 128, "histogram size"); 124 | sput_fail_unless(h.count == 1280, "histogram value count"); 125 | 126 | brubeck_histo_sample(&sample, &h); 127 | 128 | sput_fail_unless(sample.min == 1.0, "sample.min"); 129 | sput_fail_unless(sample.max == 128.0, "sample.max"); 130 | sput_fail_unless(sample.percentile[3] == 127.0, "sample.percentile[3]"); 131 | sput_fail_unless(sample.mean == 64.5, "sample.mean"); 132 | sput_fail_unless(sample.count == 1280, "sample.count"); 133 | sput_fail_unless(sample.sum == 8256.0, "sample.sum"); 134 | } 135 | 136 | void test_histogram__capacity(void) 137 | { 138 | static const size_t HISTO_CAP = USHRT_MAX; 139 | 140 | struct brubeck_histo h; 141 | struct brubeck_histo_sample sample; 142 | size_t j; 143 | 144 | memset(&h, 0x0, sizeof(h)); 145 | 146 | for (j = 0; j < HISTO_CAP + 500; ++j) 147 | brubeck_histo_push(&h, (double)(j + 1), 1.0); 148 | 149 | sput_fail_unless(h.size == HISTO_CAP, "histogram size"); 150 | sput_fail_unless(h.count == (HISTO_CAP + 500), "histogram value count"); 151 | 152 | brubeck_histo_sample(&sample, &h); 153 | 154 | sput_fail_unless(sample.min == 1.0, "sample.min"); 155 | sput_fail_unless(sample.max == (double)HISTO_CAP, "sample.max"); 156 | sput_fail_unless(sample.count == (HISTO_CAP + 500), "sample.count"); 157 | 158 | for (j = 0; j < HISTO_CAP + 500; ++j) 159 | brubeck_histo_push(&h, (double)(j + 1), 10.0); 160 | 161 | sput_fail_unless(h.size == HISTO_CAP, "histogram size"); 162 | sput_fail_unless(h.count == ((HISTO_CAP + 500) * 10), "histogram value count"); 163 | 164 | brubeck_histo_sample(&sample, &h); 165 | 166 | sput_fail_unless(sample.min == 1.0, "sample.min"); 167 | sput_fail_unless(sample.max == (double)HISTO_CAP, "sample.max"); 168 | sput_fail_unless(sample.count == ((HISTO_CAP + 500) * 10), "sample.count"); 169 | } 170 | 171 | -------------------------------------------------------------------------------- /src/samplers/statsd-secure.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "brubeck.h" 5 | 6 | #define SHA_SIZE 32 7 | #define SHA_FUNCTION EVP_sha256 8 | 9 | #define MAX_PACKET_SIZE 1024 10 | #define MIN_PACKET_SIZE (SHA_SIZE + 12) 11 | 12 | static int 13 | memcmpct(const void *_a, const void *_b, size_t len) 14 | { 15 | const unsigned char *a = _a; 16 | const unsigned char *b = _b; 17 | size_t i; 18 | int cmp = 0; 19 | 20 | for (i = 0; i < len; ++i) 21 | cmp |= a[i] ^ b[i]; 22 | 23 | return cmp; 24 | } 25 | 26 | static const char * 27 | hmactos(const char *buffer) 28 | { 29 | static const char hex_str[] = "0123456789abcdef"; 30 | static __thread char hex_hmac[SHA_SIZE * 2 + 1]; 31 | 32 | unsigned int i, j; 33 | 34 | for (i = 0, j = 0; i < SHA_SIZE; i++) { 35 | hex_hmac[j++] = hex_str[buffer[i] >> 4]; 36 | hex_hmac[j++] = hex_str[buffer[i] & 0xF]; 37 | } 38 | 39 | hex_hmac[j] = 0; 40 | 41 | return hex_hmac; 42 | } 43 | 44 | static int 45 | verify_token(struct brubeck_server *server, struct brubeck_statsd_secure *statsd, const char *buffer) 46 | { 47 | uint32_t ha, hb; 48 | uint64_t timestamp; 49 | struct timespec now; 50 | 51 | memcpy(×tamp, buffer + SHA_SIZE, 8); 52 | clock_gettime(CLOCK_REALTIME, &now); 53 | 54 | if (now.tv_sec != statsd->now) { 55 | statsd->now = now.tv_sec; 56 | multibloom_reset(statsd->replays, statsd->now % statsd->drift); 57 | } 58 | 59 | /* token from the future? */ 60 | if (statsd->now < timestamp) { 61 | log_splunk( 62 | "sampler=statsd-secure event=fail_future now=%llu timestamp=%llu", 63 | (long long unsigned int)statsd->now, 64 | (long long unsigned int)timestamp 65 | ); 66 | brubeck_stats_inc(server, secure.from_future); 67 | return -1; 68 | } 69 | 70 | /* delayed */ 71 | if (statsd->now - timestamp > statsd->drift) { 72 | log_splunk( 73 | "sampler=statsd-secure event=fail_delayed now=%llu timestamp=%llu drift=%d", 74 | (long long unsigned int)statsd->now, 75 | (long long unsigned int)timestamp, 76 | (int)(statsd->now - timestamp) 77 | ); 78 | brubeck_stats_inc(server, secure.delayed); 79 | return -1; 80 | } 81 | 82 | memcpy(&ha, buffer, sizeof(ha)); 83 | memcpy(&hb, buffer + 4, sizeof(hb)); 84 | 85 | if (multibloom_check(statsd->replays, timestamp % statsd->drift, ha, hb)) { 86 | log_splunk("sampler=statsd-secure event=fail_replayed hmac=%s", hmactos(buffer)); 87 | brubeck_stats_inc(server, secure.replayed); 88 | return -1; 89 | } 90 | 91 | return 0; 92 | } 93 | 94 | static void *statsd_secure__thread(void *_in) 95 | { 96 | struct brubeck_statsd_secure *statsd = _in; 97 | struct brubeck_server *server = statsd->sampler.server; 98 | 99 | char buffer[MAX_PACKET_SIZE]; 100 | 101 | HMAC_CTX ctx; 102 | unsigned char hmac_buffer[SHA_SIZE]; 103 | unsigned int hmac_len; 104 | 105 | struct sockaddr_in reporter; 106 | socklen_t reporter_len = sizeof(reporter); 107 | memset(&reporter, 0, reporter_len); 108 | 109 | log_splunk("sampler=statsd-secure event=worker_online"); 110 | 111 | HMAC_CTX_init(&ctx); 112 | HMAC_Init_ex(&ctx, statsd->hmac_key, strlen(statsd->hmac_key), SHA_FUNCTION(), NULL); 113 | 114 | for (;;) { 115 | int res = recvfrom(statsd->sampler.in_sock, buffer, 116 | sizeof(buffer) - 1, 0, (struct sockaddr *)&reporter, &reporter_len); 117 | 118 | if (res < 0) { 119 | if (errno == EAGAIN || errno == EINTR) 120 | continue; 121 | 122 | log_splunk_errno("sampler=statsd-secure event=failed_read from=%s", 123 | inet_ntoa(reporter.sin_addr)); 124 | brubeck_stats_inc(server, errors); 125 | continue; 126 | } 127 | 128 | brubeck_atomic_inc(&statsd->sampler.inflow); 129 | 130 | if (res < MIN_PACKET_SIZE) { 131 | log_splunk("sampler=statsd-secure event=short_pkt len=%d", res); 132 | brubeck_stats_inc(server, secure.failed); 133 | continue; 134 | } 135 | 136 | HMAC_Init_ex(&ctx, NULL, 0, NULL, NULL); 137 | HMAC_Update(&ctx, (unsigned char *)buffer + SHA_SIZE, res - SHA_SIZE); 138 | HMAC_Final(&ctx, hmac_buffer, &hmac_len); 139 | 140 | if (memcmpct(buffer, hmac_buffer, SHA_SIZE) != 0) { 141 | log_splunk("sampler=statsd-secure event=fail_auth hmac=%s", hmactos(buffer)); 142 | brubeck_stats_inc(server, secure.failed); 143 | continue; 144 | } 145 | 146 | if (verify_token(server, statsd, buffer) < 0) 147 | continue; 148 | 149 | brubeck_statsd_packet_parse(server, buffer + MIN_PACKET_SIZE, buffer + res); 150 | } 151 | 152 | HMAC_CTX_cleanup(&ctx); 153 | return NULL; 154 | } 155 | 156 | static void shutdown_sampler(struct brubeck_sampler *sampler) 157 | { 158 | struct brubeck_statsd_secure *statsd = (struct brubeck_statsd_secure *)sampler; 159 | pthread_cancel(statsd->thread); 160 | } 161 | 162 | struct brubeck_sampler * 163 | brubeck_statsd_secure_new(struct brubeck_server *server, json_t *settings) 164 | { 165 | struct brubeck_statsd_secure *std = xmalloc(sizeof(struct brubeck_statsd_secure)); 166 | char *address; 167 | int port, replay_len, drift; 168 | 169 | std->sampler.shutdown = &shutdown_sampler; 170 | std->sampler.type = BRUBECK_SAMPLER_STATSD_SECURE; 171 | std->now = 0; 172 | 173 | json_unpack_or_die(settings, 174 | "{s:s, s:i, s:s, s:i, s:i}", 175 | "address", &address, 176 | "port", &port, 177 | "hmac_key", &std->hmac_key, 178 | "max_drift", &drift, 179 | "replay_len", &replay_len); 180 | 181 | brubeck_sampler_init_inet((struct brubeck_sampler *)std, server, address, port); 182 | std->drift = (time_t)drift; 183 | std->replays = multibloom_new(std->drift, replay_len, 0.001); 184 | std->sampler.in_sock = brubeck_sampler_socket(&std->sampler, 0); 185 | 186 | if (pthread_create(&std->thread, NULL, &statsd_secure__thread, std) != 0) 187 | die("failed to start sampler thread"); 188 | 189 | return (struct brubeck_sampler *)std; 190 | } 191 | -------------------------------------------------------------------------------- /src/backends/carbon.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "brubeck.h" 4 | 5 | static bool carbon_is_connected(void *backend) 6 | { 7 | struct brubeck_carbon *self = (struct brubeck_carbon *)backend; 8 | return (self->out_sock >= 0); 9 | } 10 | 11 | static int carbon_connect(void *backend) 12 | { 13 | struct brubeck_carbon *self = (struct brubeck_carbon *)backend; 14 | 15 | if (carbon_is_connected(self)) 16 | return 0; 17 | 18 | self->out_sock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); 19 | 20 | if (self->out_sock >= 0) { 21 | int rc = connect(self->out_sock, 22 | (struct sockaddr *)&self->out_sockaddr, 23 | sizeof(self->out_sockaddr)); 24 | 25 | if (rc == 0) { 26 | log_splunk("backend=carbon event=connected"); 27 | sock_enlarge_out(self->out_sock); 28 | return 0; 29 | } 30 | 31 | close(self->out_sock); 32 | self->out_sock = -1; 33 | } 34 | 35 | log_splunk_errno("backend=carbon event=failed_to_connect"); 36 | return -1; 37 | } 38 | 39 | static void carbon_disconnect(struct brubeck_carbon *self) 40 | { 41 | log_splunk_errno("backend=carbon event=disconnected"); 42 | 43 | close(self->out_sock); 44 | self->out_sock = -1; 45 | } 46 | 47 | static void plaintext_each( 48 | const char *key, 49 | value_t value, 50 | void *backend) 51 | { 52 | struct brubeck_carbon *carbon = (struct brubeck_carbon *)backend; 53 | char buffer[1024]; 54 | char *ptr = buffer; 55 | size_t key_len = strlen(key); 56 | ssize_t wr; 57 | 58 | if (!carbon_is_connected(carbon)) 59 | return; 60 | 61 | memcpy(ptr, key, key_len); 62 | ptr += key_len; 63 | *ptr++ = ' '; 64 | 65 | ptr += brubeck_ftoa(ptr, value); 66 | *ptr++ = ' '; 67 | 68 | ptr += brubeck_itoa(ptr, carbon->backend.tick_time); 69 | *ptr++ = '\n'; 70 | 71 | wr = write_in_full(carbon->out_sock, buffer, ptr - buffer); 72 | if (wr < 0) { 73 | carbon_disconnect(carbon); 74 | return; 75 | } 76 | 77 | carbon->sent += wr; 78 | } 79 | 80 | static inline size_t pickle1_int32(char *ptr, void *_src) 81 | { 82 | *ptr = 'J'; 83 | memcpy(ptr + 1, _src, 4); 84 | return 5; 85 | } 86 | 87 | static inline size_t pickle1_double(char *ptr, void *_src) 88 | { 89 | uint8_t *source = _src; 90 | 91 | *ptr++ = 'G'; 92 | 93 | ptr[0] = source[7]; 94 | ptr[1] = source[6]; 95 | ptr[2] = source[5]; 96 | ptr[3] = source[4]; 97 | ptr[4] = source[3]; 98 | ptr[5] = source[2]; 99 | ptr[6] = source[1]; 100 | ptr[7] = source[0]; 101 | 102 | return 9; 103 | } 104 | 105 | static void pickle1_push( 106 | struct pickler *buf, 107 | const char *key, 108 | uint8_t key_len, 109 | uint32_t timestamp, 110 | value_t value) 111 | { 112 | char *ptr = buf->ptr + buf->pos; 113 | 114 | *ptr++ = '('; 115 | 116 | *ptr++ = 'U'; 117 | *ptr++ = key_len; 118 | memcpy(ptr, key, key_len); 119 | ptr += key_len; 120 | 121 | *ptr++ = 'q'; 122 | *ptr++ = buf->pt++; 123 | 124 | *ptr++ = '('; 125 | 126 | ptr += pickle1_int32(ptr, ×tamp); 127 | ptr += pickle1_double(ptr, &value); 128 | 129 | *ptr++ = 't'; 130 | *ptr++ = 'q'; 131 | *ptr++ = buf->pt++; 132 | 133 | *ptr++ = 't'; 134 | *ptr++ = 'q'; 135 | *ptr++ = buf->pt++; 136 | 137 | buf->pos = (ptr - buf->ptr); 138 | } 139 | 140 | static inline void pickle1_init(struct pickler *buf) 141 | { 142 | static const uint8_t lead[] = { ']', 'q', 0, '(' }; 143 | 144 | memcpy(buf->ptr + 4, lead, sizeof(lead)); 145 | buf->pos = 4 + sizeof(lead); 146 | buf->pt = 1; 147 | } 148 | 149 | static void pickle1_flush(void *backend) 150 | { 151 | static const uint8_t trail[] = {'e', '.'}; 152 | 153 | struct brubeck_carbon *carbon = (struct brubeck_carbon *)backend; 154 | struct pickler *buf = &carbon->pickler; 155 | 156 | uint32_t *buf_lead; 157 | ssize_t wr; 158 | 159 | if (buf->pt == 1 || !carbon_is_connected(carbon)) 160 | return; 161 | 162 | memcpy(buf->ptr + buf->pos, trail, sizeof(trail)); 163 | buf->pos += sizeof(trail); 164 | 165 | buf_lead = (uint32_t *)buf->ptr; 166 | *buf_lead = htonl((uint32_t)buf->pos - 4); 167 | 168 | wr = write_in_full(carbon->out_sock, buf->ptr, buf->pos); 169 | 170 | pickle1_init(&carbon->pickler); 171 | if (wr < 0) { 172 | carbon_disconnect(carbon); 173 | return; 174 | } 175 | 176 | carbon->sent += wr; 177 | } 178 | 179 | static void pickle1_each( 180 | const char *key, 181 | value_t value, 182 | void *backend) 183 | { 184 | struct brubeck_carbon *carbon = (struct brubeck_carbon *)backend; 185 | uint8_t key_len = (uint8_t)strlen(key); 186 | 187 | if (carbon->pickler.pos + PICKLE1_SIZE(key_len) 188 | >= PICKLE_BUFFER_SIZE) { 189 | pickle1_flush(carbon); 190 | } 191 | 192 | if (!carbon_is_connected(carbon)) 193 | return; 194 | 195 | pickle1_push(&carbon->pickler, key, key_len, 196 | carbon->backend.tick_time, value); 197 | } 198 | 199 | struct brubeck_backend * 200 | brubeck_carbon_new(struct brubeck_server *server, json_t *settings, int shard_n) 201 | { 202 | struct brubeck_carbon *carbon = xcalloc(1, sizeof(struct brubeck_carbon)); 203 | char *address; 204 | int port, frequency, pickle = 0; 205 | 206 | json_unpack_or_die(settings, 207 | "{s:s, s:i, s?:b, s:i}", 208 | "address", &address, 209 | "port", &port, 210 | "pickle", &pickle, 211 | "frequency", &frequency); 212 | 213 | carbon->backend.type = BRUBECK_BACKEND_CARBON; 214 | carbon->backend.shard_n = shard_n; 215 | carbon->backend.connect = &carbon_connect; 216 | carbon->backend.is_connected = &carbon_is_connected; 217 | 218 | if (pickle) { 219 | carbon->backend.sample = &pickle1_each; 220 | carbon->backend.flush = &pickle1_flush; 221 | carbon->pickler.ptr = malloc(PICKLE_BUFFER_SIZE); 222 | pickle1_init(&carbon->pickler); 223 | } else { 224 | carbon->backend.sample = &plaintext_each; 225 | carbon->backend.flush = NULL; 226 | } 227 | 228 | carbon->backend.sample_freq = frequency; 229 | carbon->backend.server = server; 230 | carbon->out_sock = -1; 231 | url_to_inaddr2(&carbon->out_sockaddr, address, port); 232 | 233 | brubeck_backend_run_threaded((struct brubeck_backend *)carbon); 234 | log_splunk("backend=carbon event=started"); 235 | 236 | return (struct brubeck_backend *)carbon; 237 | } 238 | -------------------------------------------------------------------------------- /src/http.c: -------------------------------------------------------------------------------- 1 | #include "brubeck.h" 2 | 3 | #ifdef BRUBECK_HAVE_MICROHTTPD 4 | #include "microhttpd.h" 5 | #include "jansson.h" 6 | 7 | #ifdef BRUBECK_METRICS_FLOW 8 | 9 | static int flow_cmp(const void *a, const void *b) 10 | { 11 | const struct brubeck_metric *ma = *(const struct brubeck_metric **)a; 12 | const struct brubeck_metric *mb = *(const struct brubeck_metric **)b; 13 | if (ma->flow < mb->flow) return 1; 14 | if (ma->flow > mb->flow) return -1; 15 | return 0; 16 | } 17 | 18 | static struct MHD_Response * 19 | flow_stats(struct brubeck_server *server) 20 | { 21 | size_t metric_count, i, topn; 22 | struct brubeck_metric **metrics; 23 | json_t *top_metrics_j; 24 | char *jsonr; 25 | 26 | metrics = brubeck_hashtable_to_a(server->metrics, &metric_count); 27 | qsort(metrics, metric_count, sizeof(struct brubeck_metric *), &flow_cmp); 28 | 29 | topn = (metric_count < 32) ? metric_count : 32; 30 | top_metrics_j = json_object(); 31 | 32 | for (i = 0; i < topn; ++i) { 33 | struct brubeck_metric *m = metrics[i]; 34 | json_object_set_new(top_metrics_j, m->key, json_integer(m->flow)); 35 | } 36 | 37 | free(metrics); 38 | jsonr = json_dumps(top_metrics_j, JSON_INDENT(4) | JSON_PRESERVE_ORDER); 39 | json_decref(top_metrics_j); 40 | 41 | return MHD_create_response_from_data(strlen(jsonr), jsonr, 1, 0); 42 | } 43 | 44 | #else 45 | 46 | static struct MHD_Response * 47 | flow_stats(struct brubeck_server *server) 48 | { 49 | return NULL; 50 | } 51 | 52 | #endif 53 | 54 | static struct brubeck_metric *safe_lookup_metric(struct brubeck_server *server, const char *key) 55 | { 56 | return brubeck_hashtable_find(server->metrics, key, (uint16_t)strlen(key)); 57 | } 58 | 59 | static struct MHD_Response * 60 | expire_metric(struct brubeck_server *server, const char *url) 61 | { 62 | struct brubeck_metric *metric = safe_lookup_metric( 63 | server, url + strlen("/expire/")); 64 | 65 | if (metric) { 66 | metric->expire = BRUBECK_EXPIRE_DISABLED; 67 | return MHD_create_response_from_data( 68 | 0, "", 0, 0); 69 | } 70 | return NULL; 71 | } 72 | 73 | static struct MHD_Response * 74 | send_metric(struct brubeck_server *server, const char *url) 75 | { 76 | static const char *metric_types[] = { 77 | "gauge", "meter", "counter", "histogram", "timer", "internal" 78 | }; 79 | static const char *expire_status[] = { 80 | "disabled", "inactive", "active" 81 | }; 82 | 83 | struct brubeck_metric *metric = safe_lookup_metric( 84 | server, url + strlen("/metric/")); 85 | 86 | if (metric) { 87 | json_t *mj = json_pack("{s:s, s:s, s:i, s:s}", 88 | "key", metric->key, 89 | "type", metric_types[metric->type], 90 | #if METRIC_SHARD_SPECIFIER 91 | "shard", (int)metric->shard, 92 | #else 93 | "shard", 0, 94 | #endif 95 | "expire", expire_status[metric->expire] 96 | ); 97 | 98 | char *jsonr = json_dumps(mj, JSON_INDENT(4) | JSON_PRESERVE_ORDER); 99 | json_decref(mj); 100 | return MHD_create_response_from_data( 101 | strlen(jsonr), jsonr, 1, 0); 102 | } 103 | 104 | return NULL; 105 | } 106 | 107 | static struct MHD_Response * 108 | send_stats(struct brubeck_server *brubeck) 109 | { 110 | char *jsonr; 111 | json_t *stats, *secure, *backends, *samplers; 112 | int i; 113 | 114 | backends = json_array(); 115 | 116 | for (i = 0; i < brubeck->active_backends; ++i) { 117 | struct brubeck_backend *backend = brubeck->backends[i]; 118 | 119 | if (backend->type == BRUBECK_BACKEND_CARBON) { 120 | struct brubeck_carbon *carbon = (struct brubeck_carbon *)backend; 121 | struct sockaddr_in *address = &carbon->out_sockaddr; 122 | char addr[INET_ADDRSTRLEN]; 123 | 124 | json_array_append_new(backends, 125 | json_pack("{s:s, s:i, s:b, s:s, s:i, s:I}", 126 | "type", "carbon", 127 | "sample_freq", (int)carbon->backend.sample_freq, 128 | "connected", (carbon->out_sock >= 0), 129 | "address", inet_ntop(AF_INET, &address->sin_addr.s_addr, addr, INET_ADDRSTRLEN), 130 | "port", (int)ntohs(address->sin_port), 131 | "sent", (json_int_t)carbon->sent 132 | )); 133 | } 134 | } 135 | 136 | samplers = json_array(); 137 | 138 | for (i = 0; i < brubeck->active_samplers; ++i) { 139 | struct brubeck_sampler *sampler = brubeck->samplers[i]; 140 | struct sockaddr_in *address = &sampler->addr; 141 | char addr[INET_ADDRSTRLEN]; 142 | const char *sampler_name = NULL; 143 | 144 | switch (sampler->type) { 145 | case BRUBECK_SAMPLER_STATSD: sampler_name = "statsd"; break; 146 | case BRUBECK_SAMPLER_STATSD_SECURE: sampler_name = "statsd_secure"; break; 147 | default: assert(0); 148 | } 149 | 150 | json_array_append_new(samplers, 151 | json_pack("{s:s, s:f, s:s, s:i}", 152 | "type", sampler_name, 153 | "sample_freq", (double)sampler->current_flow, 154 | "address", inet_ntop(AF_INET, &address->sin_addr.s_addr, addr, INET_ADDRSTRLEN), 155 | "port", (int)ntohs(address->sin_port) 156 | )); 157 | } 158 | 159 | secure = json_pack("{s:i, s:i, s:i, s:i}", 160 | "failed", brubeck_stats_sample(brubeck, secure.failed), 161 | "from_future", brubeck_stats_sample(brubeck, secure.from_future), 162 | "delayed", brubeck_stats_sample(brubeck, secure.delayed), 163 | "replayed", brubeck_stats_sample(brubeck, secure.replayed) 164 | ); 165 | 166 | stats = json_pack("{s:s, s:i, s:i, s:i, s:o, s:o, s:o}", 167 | "version", "brubeck " GIT_SHA, 168 | "metrics", brubeck_stats_sample(brubeck, metrics), 169 | "errors", brubeck_stats_sample(brubeck, errors), 170 | "unique_keys", brubeck_stats_sample(brubeck, unique_keys), 171 | "secure", secure, 172 | "backends", backends, 173 | "samplers", samplers); 174 | 175 | jsonr = json_dumps(stats, JSON_INDENT(4) | JSON_PRESERVE_ORDER); 176 | json_decref(stats); 177 | return MHD_create_response_from_data( 178 | strlen(jsonr), jsonr, 1, 0); 179 | } 180 | 181 | static struct MHD_Response * 182 | send_ping(struct brubeck_server *brubeck) 183 | { 184 | const value_t frequency = (double)brubeck->internal_stats.sample_freq; 185 | const char *status = "OK"; 186 | 187 | char *jsonr; 188 | json_t *stats; 189 | int i; 190 | 191 | for (i = 0; i < brubeck->active_backends; ++i) { 192 | struct brubeck_backend *backend = brubeck->backends[i]; 193 | if (!backend->is_connected(backend)) { 194 | status = "ERROR (backend disconnected)"; 195 | break; 196 | } 197 | } 198 | 199 | stats = json_pack("{s:s, s:i, s:s, s:f, s:f, s:i}", 200 | "version", "brubeck " GIT_SHA, 201 | "pid", (int)getpid(), 202 | "status", status, 203 | "metrics_per_second", (value_t)brubeck_stats_sample(brubeck, metrics) / frequency, 204 | "errors_per_second", (value_t)brubeck_stats_sample(brubeck, errors) / frequency, 205 | "unique_keys", brubeck_stats_sample(brubeck, unique_keys) 206 | ); 207 | 208 | jsonr = json_dumps(stats, JSON_INDENT(4) | JSON_PRESERVE_ORDER); 209 | json_decref(stats); 210 | return MHD_create_response_from_data( 211 | strlen(jsonr), jsonr, 1, 0); 212 | } 213 | 214 | static int 215 | handle_request(void *cls, struct MHD_Connection *connection, 216 | const char *url, const char *method, 217 | const char *version, const char *upload_data, 218 | size_t *upload_data_size, void **con_cls) 219 | { 220 | int ret; 221 | struct MHD_Response *response = NULL; 222 | struct brubeck_server *brubeck = cls; 223 | 224 | if (!strcmp(method, "GET")) { 225 | if (!strcmp(url, "/_ping") || !strcmp(url, "/ping")) 226 | response = send_ping(brubeck); 227 | 228 | else if (!strcmp(url, "/stats")) 229 | response = send_stats(brubeck); 230 | 231 | else if (!strcmp(url, "/flow_stats")) 232 | response = flow_stats(brubeck); 233 | 234 | else if (starts_with(url, "/metric/")) 235 | response = send_metric(brubeck, url); 236 | } 237 | else if (!strcmp(method, "POST")) { 238 | if (starts_with(url, "/expire/")) 239 | response = expire_metric(brubeck, url); 240 | } 241 | 242 | if (!response) { 243 | static const char *NOT_FOUND = "404 not found"; 244 | response = MHD_create_response_from_data( 245 | strlen(NOT_FOUND), (void *)NOT_FOUND, 0, 0); 246 | MHD_add_response_header(response, "Connection", "close"); 247 | ret = MHD_queue_response(connection, 404, response); 248 | } else { 249 | MHD_add_response_header(response, "Connection", "close"); 250 | MHD_add_response_header(response, "Content-Type", "application/json"); 251 | ret = MHD_queue_response(connection, 200, response); 252 | } 253 | 254 | MHD_destroy_response(response); 255 | return ret; 256 | } 257 | 258 | void brubeck_http_endpoint_init(struct brubeck_server *server, const char *listen) 259 | { 260 | struct MHD_Daemon *daemon; 261 | 262 | const char *port = strrchr(listen, ':'); 263 | port = port ? port + 1 : listen; 264 | 265 | daemon = MHD_start_daemon( 266 | MHD_USE_SELECT_INTERNALLY, 267 | atoi(port), 268 | NULL, NULL, 269 | &handle_request, server, 270 | MHD_OPTION_CONNECTION_TIMEOUT, (unsigned int)10, 271 | MHD_OPTION_END); 272 | 273 | if (!daemon) 274 | die("failed to start HTTP endpoint"); 275 | log_splunk("event=http_server listen=%s", port); 276 | } 277 | 278 | #else 279 | 280 | void brubeck_http_endpoint_init(struct brubeck_server *server, const char *listen) 281 | { 282 | die("http support has not been compiled in Brubeck"); 283 | } 284 | 285 | #endif 286 | 287 | -------------------------------------------------------------------------------- /src/samplers/statsd.c: -------------------------------------------------------------------------------- 1 | #include 2 | #define _GNU_SOURCE 3 | #include 4 | #include 5 | #include "brubeck.h" 6 | 7 | #ifdef __GLIBC__ 8 | # if ((__GLIBC__ > 2) || ((__GLIBC__ == 2) && (__GLIBC_MINOR__ >= 12))) 9 | # define HAVE_RECVMMSG 1 10 | # endif 11 | #endif 12 | 13 | #define MAX_PACKET_SIZE 8192 14 | 15 | #ifdef HAVE_RECVMMSG 16 | 17 | #ifndef MSG_WAITFORONE 18 | # define MSG_WAITFORONE 0x0 19 | #endif 20 | 21 | static void statsd_run_recvmmsg(struct brubeck_statsd *statsd, int sock) 22 | { 23 | const unsigned int SIM_PACKETS = statsd->mmsg_count; 24 | struct brubeck_server *server = statsd->sampler.server; 25 | 26 | unsigned int i; 27 | struct iovec iovecs[SIM_PACKETS]; 28 | struct mmsghdr msgs[SIM_PACKETS]; 29 | 30 | memset(msgs, 0x0, sizeof(msgs)); 31 | 32 | for (i = 0; i < SIM_PACKETS; ++i) { 33 | iovecs[i].iov_base = xmalloc(MAX_PACKET_SIZE); 34 | iovecs[i].iov_len = MAX_PACKET_SIZE - 1; 35 | msgs[i].msg_hdr.msg_iov = &iovecs[i]; 36 | msgs[i].msg_hdr.msg_iovlen = 1; 37 | } 38 | 39 | log_splunk("sampler=statsd event=worker_online syscall=recvmmsg socket=%d", sock); 40 | 41 | for (;;) { 42 | int res = recvmmsg(sock, msgs, SIM_PACKETS, MSG_WAITFORONE, NULL); 43 | 44 | if (res < 0) { 45 | if (errno == EAGAIN || errno == EINTR) 46 | continue; 47 | 48 | log_splunk_errno("sampler=statsd event=failed_read"); 49 | brubeck_stats_inc(server, errors); 50 | continue; 51 | } 52 | 53 | /* store stats */ 54 | brubeck_atomic_add(&statsd->sampler.inflow, SIM_PACKETS); 55 | 56 | for (i = 0; i < SIM_PACKETS; ++i) { 57 | char *buf = msgs[i].msg_hdr.msg_iov->iov_base; 58 | char *end = buf + msgs[i].msg_len; 59 | brubeck_statsd_packet_parse(server, buf, end); 60 | } 61 | } 62 | } 63 | #endif 64 | 65 | static void statsd_run_recvmsg(struct brubeck_statsd *statsd, int sock) 66 | { 67 | struct brubeck_server *server = statsd->sampler.server; 68 | 69 | char *buffer = xmalloc(MAX_PACKET_SIZE); 70 | struct sockaddr_in reporter; 71 | socklen_t reporter_len = sizeof(reporter); 72 | memset(&reporter, 0, reporter_len); 73 | 74 | log_splunk("sampler=statsd event=worker_online syscall=recvmsg socket=%d", sock); 75 | 76 | for (;;) { 77 | int res = recvfrom(sock, buffer, MAX_PACKET_SIZE - 1, 0, 78 | (struct sockaddr *)&reporter, &reporter_len); 79 | 80 | if (res < 0) { 81 | if (errno == EAGAIN || errno == EINTR) 82 | continue; 83 | 84 | log_splunk_errno("sampler=statsd event=failed_read from=%s", 85 | inet_ntoa(reporter.sin_addr)); 86 | brubeck_stats_inc(server, errors); 87 | continue; 88 | } 89 | 90 | brubeck_atomic_inc(&statsd->sampler.inflow); 91 | brubeck_statsd_packet_parse(server, buffer, buffer + res); 92 | } 93 | } 94 | 95 | static inline char * 96 | parse_float(char *buffer, value_t *result, uint8_t *mods) 97 | { 98 | int negative = 0; 99 | char *start = buffer; 100 | value_t value = 0.0; 101 | 102 | if (*buffer == '-') { 103 | ++buffer; 104 | negative = 1; 105 | *mods |= BRUBECK_MOD_RELATIVE_VALUE; 106 | } else if (*buffer == '+') { 107 | ++buffer; 108 | *mods |= BRUBECK_MOD_RELATIVE_VALUE; 109 | } 110 | 111 | while (*buffer >= '0' && *buffer <= '9') { 112 | value = (value * 10.0) + (*buffer - '0'); 113 | ++buffer; 114 | } 115 | 116 | if (*buffer == '.') { 117 | double f = 0.0; 118 | int n = 0; 119 | ++buffer; 120 | 121 | while (*buffer >= '0' && *buffer <= '9') { 122 | f = (f * 10.0) + (*buffer - '0'); 123 | buffer++; 124 | n++; 125 | } 126 | 127 | value += f / pow(10.0, n); 128 | } 129 | 130 | if (negative) 131 | value = -value; 132 | 133 | if (unlikely(*buffer == 'e' || *buffer == 'E')) 134 | value = strtod(start, &buffer); 135 | 136 | *result = value; 137 | return buffer; 138 | } 139 | 140 | int brubeck_statsd_msg_parse(struct brubeck_statsd_msg *msg, char *buffer, char *end) 141 | { 142 | *end = '\0'; 143 | 144 | /** 145 | * Message key: all the string until the first ':' 146 | * 147 | * gaugor:333|g 148 | * ^^^^^^ 149 | */ 150 | { 151 | msg->key = buffer; 152 | msg->key_len = 0; 153 | while (*buffer != ':' && *buffer != '\0') { 154 | /* Invalid metric, can't have a space */ 155 | if (*buffer == ' ') 156 | return -1; 157 | ++buffer; 158 | } 159 | if (*buffer == '\0') 160 | return -1; 161 | 162 | msg->key_len = buffer - msg->key; 163 | *buffer++ = '\0'; 164 | 165 | /* Corrupted metric. Graphite won't swallow this */ 166 | if (msg->key[msg->key_len - 1] == '.') 167 | return -1; 168 | } 169 | 170 | /** 171 | * Message value: the numeric value between ':' and '|'. 172 | * This is already converted to an integer. 173 | * 174 | * gaugor:333|g 175 | * ^^^ 176 | */ 177 | { 178 | msg->modifiers = 0; 179 | buffer = parse_float(buffer, &msg->value, &msg->modifiers); 180 | 181 | if (*buffer != '|') 182 | return -1; 183 | 184 | buffer++; 185 | } 186 | 187 | /** 188 | * Message type: one or two char identifier with the 189 | * message type. Valid values: g, c, C, h, ms 190 | * 191 | * gaugor:333|g 192 | * ^ 193 | */ 194 | { 195 | switch (*buffer) { 196 | case 'g': msg->type = BRUBECK_MT_GAUGE; break; 197 | case 'c': msg->type = BRUBECK_MT_METER; break; 198 | case 'C': msg->type = BRUBECK_MT_COUNTER; break; 199 | case 'h': msg->type = BRUBECK_MT_HISTO; break; 200 | case 'm': 201 | ++buffer; 202 | if (*buffer == 's') { 203 | msg->type = BRUBECK_MT_TIMER; 204 | break; 205 | } 206 | 207 | default: 208 | return -1; 209 | } 210 | 211 | buffer++; 212 | } 213 | 214 | /** 215 | * Sample rate: parse the sample rate trailer if it exists. 216 | * It must be a floating point number between 0.0 and 1.0 217 | * 218 | * gorets:1|c|@0.1 219 | * ^^^^---- 220 | */ 221 | { 222 | if (buffer[0] == '|' && buffer[1] == '@') { 223 | double sample_rate; 224 | uint8_t dummy; 225 | 226 | buffer = parse_float(buffer + 2, &sample_rate, &dummy); 227 | if (sample_rate <= 0.0 || sample_rate > 1.0) 228 | return -1; 229 | 230 | msg->sample_freq = (1.0 / sample_rate); 231 | } else { 232 | msg->sample_freq = 1.0; 233 | } 234 | 235 | 236 | if (buffer[0] == '\0' || (buffer[0] == '\n' && buffer[1] == '\0')) 237 | return 0; 238 | 239 | return -1; 240 | } 241 | } 242 | 243 | void brubeck_statsd_packet_parse(struct brubeck_server *server, char *buffer, char *end) 244 | { 245 | struct brubeck_statsd_msg msg; 246 | struct brubeck_metric *metric; 247 | 248 | while (buffer < end) { 249 | char *stat_end = memchr(buffer, '\n', end - buffer); 250 | if (!stat_end) 251 | stat_end = end; 252 | 253 | if (brubeck_statsd_msg_parse(&msg, buffer, stat_end) < 0) { 254 | brubeck_stats_inc(server, errors); 255 | log_splunk("sampler=statsd event=packet_drop"); 256 | } else { 257 | brubeck_stats_inc(server, metrics); 258 | metric = brubeck_metric_find(server, msg.key, msg.key_len, msg.type); 259 | if (metric != NULL) 260 | brubeck_metric_record(metric, msg.value, msg.sample_freq, msg.modifiers); 261 | } 262 | 263 | /* move buf past this stat */ 264 | buffer = stat_end + 1; 265 | } 266 | } 267 | 268 | static void *statsd__thread(void *_in) 269 | { 270 | struct brubeck_statsd *statsd = _in; 271 | int sock = statsd->sampler.in_sock; 272 | 273 | #ifdef SO_REUSEPORT 274 | if (sock < 0) { 275 | sock = brubeck_sampler_socket(&statsd->sampler, 1); 276 | } 277 | #endif 278 | 279 | assert(sock >= 0); 280 | 281 | #ifdef HAVE_RECVMMSG 282 | if (statsd->mmsg_count > 1) { 283 | statsd_run_recvmmsg(statsd, sock); 284 | return NULL; 285 | } 286 | #endif 287 | 288 | statsd_run_recvmsg(statsd, sock); 289 | return NULL; 290 | } 291 | 292 | static void run_worker_threads(struct brubeck_statsd *statsd) 293 | { 294 | unsigned int i; 295 | statsd->workers = xmalloc(statsd->worker_count * sizeof(pthread_t)); 296 | 297 | for (i = 0; i < statsd->worker_count; ++i) { 298 | if (pthread_create(&statsd->workers[i], NULL, &statsd__thread, statsd) != 0) 299 | die("failed to start sampler thread"); 300 | } 301 | } 302 | 303 | static void shutdown_sampler(struct brubeck_sampler *sampler) 304 | { 305 | struct brubeck_statsd *statsd = (struct brubeck_statsd *)sampler; 306 | size_t i; 307 | 308 | for (i = 0; i < statsd->worker_count; ++i) { 309 | pthread_cancel(statsd->workers[i]); 310 | } 311 | } 312 | 313 | struct brubeck_sampler * 314 | brubeck_statsd_new(struct brubeck_server *server, json_t *settings) 315 | { 316 | struct brubeck_statsd *std = xmalloc(sizeof(struct brubeck_statsd)); 317 | 318 | char *address; 319 | int port; 320 | int multisock = 0; 321 | 322 | std->sampler.type = BRUBECK_SAMPLER_STATSD; 323 | std->sampler.shutdown = &shutdown_sampler; 324 | std->sampler.in_sock = -1; 325 | std->worker_count = 4; 326 | std->mmsg_count = 1; 327 | 328 | json_unpack_or_die(settings, 329 | "{s:s, s:i, s?:i, s?:i, s?:b}", 330 | "address", &address, 331 | "port", &port, 332 | "workers", &std->worker_count, 333 | "multimsg", &std->mmsg_count, 334 | "multisock", &multisock); 335 | 336 | brubeck_sampler_init_inet(&std->sampler, server, address, port); 337 | 338 | #ifndef SO_REUSEPORT 339 | multisock = 0; 340 | #endif 341 | 342 | if (!multisock) 343 | std->sampler.in_sock = brubeck_sampler_socket(&std->sampler, 0); 344 | 345 | run_worker_threads(std); 346 | return &std->sampler; 347 | } 348 | -------------------------------------------------------------------------------- /src/metric.c: -------------------------------------------------------------------------------- 1 | #include "brubeck.h" 2 | 3 | static inline struct brubeck_metric * 4 | new_metric(struct brubeck_server *server, const char *key, size_t key_len, uint8_t type) 5 | { 6 | struct brubeck_metric *metric; 7 | 8 | /* slab allocation cannot fail */ 9 | metric = brubeck_slab_alloc(&server->slab, 10 | sizeof(struct brubeck_metric) + key_len + 1); 11 | 12 | memset(metric, 0x0, sizeof(struct brubeck_metric)); 13 | 14 | memcpy(metric->key, key, key_len); 15 | metric->key[key_len] = '\0'; 16 | metric->key_len = (uint16_t)key_len; 17 | 18 | metric->expire = BRUBECK_EXPIRE_ACTIVE; 19 | metric->type = type; 20 | pthread_spin_init(&metric->lock, PTHREAD_PROCESS_PRIVATE); 21 | 22 | #ifdef BRUBECK_METRICS_FLOW 23 | metric->flow = 0; 24 | #else 25 | /* Compile time assert: ensure that the metric struct can be packed 26 | * in a single slab */ 27 | ct_assert(sizeof(struct brubeck_metric) <= (SLAB_SIZE)); 28 | #endif 29 | 30 | return metric; 31 | } 32 | 33 | typedef void (*mt_prototype_record)(struct brubeck_metric *, value_t, value_t, uint8_t); 34 | typedef void (*mt_prototype_sample)(struct brubeck_metric *, brubeck_sample_cb, void *); 35 | 36 | 37 | /********************************************* 38 | * Gauge 39 | * 40 | * ALLOC: mt + 4 bytes 41 | *********************************************/ 42 | static void 43 | gauge__record(struct brubeck_metric *metric, value_t value, value_t sample_freq, uint8_t modifiers) 44 | { 45 | pthread_spin_lock(&metric->lock); 46 | { 47 | if (modifiers & BRUBECK_MOD_RELATIVE_VALUE) { 48 | metric->as.gauge.value += value; 49 | } else { 50 | metric->as.gauge.value = value; 51 | } 52 | } 53 | pthread_spin_unlock(&metric->lock); 54 | } 55 | 56 | static void 57 | gauge__sample(struct brubeck_metric *metric, brubeck_sample_cb sample, void *opaque) 58 | { 59 | value_t value; 60 | 61 | pthread_spin_lock(&metric->lock); 62 | { 63 | value = metric->as.gauge.value; 64 | } 65 | pthread_spin_unlock(&metric->lock); 66 | 67 | sample(metric->key, value, opaque); 68 | } 69 | 70 | 71 | /********************************************* 72 | * Meter 73 | * 74 | * ALLOC: mt + 4 75 | *********************************************/ 76 | static void 77 | meter__record(struct brubeck_metric *metric, value_t value, value_t sample_freq, uint8_t modifiers) 78 | { 79 | /* upsample */ 80 | value *= sample_freq; 81 | 82 | pthread_spin_lock(&metric->lock); 83 | { 84 | metric->as.meter.value += value; 85 | } 86 | pthread_spin_unlock(&metric->lock); 87 | } 88 | 89 | static void 90 | meter__sample(struct brubeck_metric *metric, brubeck_sample_cb sample, void *opaque) 91 | { 92 | value_t value; 93 | 94 | pthread_spin_lock(&metric->lock); 95 | { 96 | value = metric->as.meter.value; 97 | metric->as.meter.value = 0.0; 98 | } 99 | pthread_spin_unlock(&metric->lock); 100 | 101 | sample(metric->key, value, opaque); 102 | } 103 | 104 | 105 | /********************************************* 106 | * Counter 107 | * 108 | * ALLOC: mt + 4 + 4 + 4 109 | *********************************************/ 110 | static void 111 | counter__record(struct brubeck_metric *metric, value_t value, value_t sample_freq, uint8_t modifiers) 112 | { 113 | /* upsample */ 114 | value *= sample_freq; 115 | 116 | pthread_spin_lock(&metric->lock); 117 | { 118 | if (metric->as.counter.previous > 0.0) { 119 | value_t diff = (value >= metric->as.counter.previous) ? 120 | (value - metric->as.counter.previous) : 121 | (value); 122 | 123 | metric->as.counter.value += diff; 124 | } 125 | 126 | metric->as.counter.previous = value; 127 | } 128 | pthread_spin_unlock(&metric->lock); 129 | } 130 | 131 | static void 132 | counter__sample(struct brubeck_metric *metric, brubeck_sample_cb sample, void *opaque) 133 | { 134 | value_t value; 135 | 136 | pthread_spin_lock(&metric->lock); 137 | { 138 | value = metric->as.counter.value; 139 | metric->as.counter.value = 0.0; 140 | } 141 | pthread_spin_unlock(&metric->lock); 142 | 143 | sample(metric->key, value, opaque); 144 | } 145 | 146 | 147 | /********************************************* 148 | * Histogram / Timer 149 | * 150 | * ALLOC: mt + 16 + 4 151 | *********************************************/ 152 | static void 153 | histogram__record(struct brubeck_metric *metric, value_t value, value_t sample_freq, uint8_t modifiers) 154 | { 155 | pthread_spin_lock(&metric->lock); 156 | { 157 | brubeck_histo_push(&metric->as.histogram, value, sample_freq); 158 | } 159 | pthread_spin_unlock(&metric->lock); 160 | } 161 | 162 | static void 163 | histogram__sample(struct brubeck_metric *metric, brubeck_sample_cb sample, void *opaque) 164 | { 165 | struct brubeck_histo_sample hsample; 166 | char *key; 167 | 168 | pthread_spin_lock(&metric->lock); 169 | { 170 | brubeck_histo_sample(&hsample, &metric->as.histogram); 171 | } 172 | pthread_spin_unlock(&metric->lock); 173 | 174 | /* alloc space for this on the stack. we need enough for: 175 | * key_length + longest_suffix + null terminator 176 | */ 177 | key = alloca(metric->key_len + strlen(".percentile.999") + 1); 178 | memcpy(key, metric->key, metric->key_len); 179 | 180 | 181 | WITH_SUFFIX(".count") { 182 | sample(key, hsample.count, opaque); 183 | } 184 | 185 | WITH_SUFFIX(".count_ps") { 186 | struct brubeck_backend *backend = opaque; 187 | sample(key, hsample.count / (double)backend->sample_freq, opaque); 188 | } 189 | 190 | /* if there have been no metrics during this sampling period, 191 | * we don't need to report any of the histogram samples */ 192 | if (hsample.count == 0.0) 193 | return; 194 | 195 | WITH_SUFFIX(".min") { 196 | sample(key, hsample.min, opaque); 197 | } 198 | 199 | WITH_SUFFIX(".max") { 200 | sample(key, hsample.max, opaque); 201 | } 202 | 203 | WITH_SUFFIX(".sum") { 204 | sample(key, hsample.sum, opaque); 205 | } 206 | 207 | WITH_SUFFIX(".mean") { 208 | sample(key, hsample.mean, opaque); 209 | } 210 | 211 | WITH_SUFFIX(".median") { 212 | sample(key, hsample.median, opaque); 213 | } 214 | 215 | WITH_SUFFIX(".percentile.75") { 216 | sample(key, hsample.percentile[PC_75], opaque); 217 | } 218 | 219 | WITH_SUFFIX(".percentile.95") { 220 | sample(key, hsample.percentile[PC_95], opaque); 221 | } 222 | 223 | WITH_SUFFIX(".percentile.98") { 224 | sample(key, hsample.percentile[PC_98], opaque); 225 | } 226 | 227 | WITH_SUFFIX(".percentile.99") { 228 | sample(key, hsample.percentile[PC_99], opaque); 229 | } 230 | 231 | WITH_SUFFIX(".percentile.999") { 232 | sample(key, hsample.percentile[PC_999], opaque); 233 | } 234 | } 235 | 236 | /********************************************************/ 237 | 238 | static struct brubeck_metric__proto { 239 | mt_prototype_record record; 240 | mt_prototype_sample sample; 241 | } _prototypes[] = { 242 | /* Gauge */ 243 | { 244 | &gauge__record, 245 | &gauge__sample 246 | }, 247 | 248 | /* Meter */ 249 | { 250 | &meter__record, 251 | &meter__sample 252 | }, 253 | 254 | /* Counter */ 255 | { 256 | &counter__record, 257 | &counter__sample 258 | }, 259 | 260 | /* Histogram */ 261 | { 262 | &histogram__record, 263 | &histogram__sample 264 | }, 265 | 266 | /* Timer -- uses same implementation as histogram */ 267 | { 268 | &histogram__record, 269 | &histogram__sample 270 | }, 271 | 272 | /* Internal -- used for sampling brubeck itself */ 273 | { 274 | NULL, /* recorded manually */ 275 | brubeck_internal__sample 276 | } 277 | }; 278 | 279 | void brubeck_metric_sample(struct brubeck_metric *metric, brubeck_sample_cb cb, void *backend) 280 | { 281 | _prototypes[metric->type].sample(metric, cb, backend); 282 | } 283 | 284 | void brubeck_metric_record(struct brubeck_metric *metric, value_t value, value_t sample_freq, uint8_t modifiers) 285 | { 286 | _prototypes[metric->type].record(metric, value, sample_freq, modifiers); 287 | } 288 | 289 | struct brubeck_backend * 290 | brubeck_metric_shard(struct brubeck_server *server, struct brubeck_metric *metric) 291 | { 292 | int shard = 0; 293 | if (server->active_backends > 1) 294 | shard = CityHash32(metric->key, metric->key_len) % server->active_backends; 295 | return server->backends[shard]; 296 | } 297 | 298 | struct brubeck_metric * 299 | brubeck_metric_new(struct brubeck_server *server, const char *key, size_t key_len, uint8_t type) 300 | { 301 | struct brubeck_metric *metric; 302 | 303 | metric = new_metric(server, key, key_len, type); 304 | if (!metric) 305 | return NULL; 306 | 307 | if (!brubeck_hashtable_insert(server->metrics, metric->key, metric->key_len, metric)) 308 | return brubeck_hashtable_find(server->metrics, key, key_len); 309 | 310 | brubeck_backend_register_metric(brubeck_metric_shard(server, metric), metric); 311 | 312 | /* Record internal stats */ 313 | brubeck_stats_inc(server, unique_keys); 314 | return metric; 315 | } 316 | 317 | struct brubeck_metric * 318 | brubeck_metric_find(struct brubeck_server *server, const char *key, size_t key_len, uint8_t type) 319 | { 320 | struct brubeck_metric *metric; 321 | 322 | assert(key[key_len] == '\0'); 323 | metric = brubeck_hashtable_find(server->metrics, key, (uint16_t)key_len); 324 | 325 | if (unlikely(metric == NULL)) { 326 | if (server->at_capacity) 327 | return NULL; 328 | 329 | return brubeck_metric_new(server, key, key_len, type); 330 | } 331 | 332 | #ifdef BRUBECK_METRICS_FLOW 333 | brubeck_atomic_inc(&metric->flow); 334 | #endif 335 | 336 | metric->expire = BRUBECK_EXPIRE_ACTIVE; 337 | return metric; 338 | } 339 | -------------------------------------------------------------------------------- /src/server.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include "brubeck.h" 7 | 8 | #if JANSSON_VERSION_HEX < 0x020500 9 | # error "libjansson-dev is too old (2.5+ required)" 10 | #endif 11 | 12 | static void 13 | update_flows(struct brubeck_server *server) 14 | { 15 | int i; 16 | for (i = 0; i < server->active_samplers; ++i) { 17 | struct brubeck_sampler *sampler = server->samplers[i]; 18 | sampler->current_flow = sampler->inflow; 19 | sampler->inflow = 0; 20 | } 21 | } 22 | 23 | #define UTF8_UPARROW "\xE2\x86\x91" 24 | #define UTF8_DOWNARROW "\xE2\x86\x93" 25 | 26 | static void 27 | update_proctitle(struct brubeck_server *server) 28 | { 29 | static const char *size_suffix[] = { "b", "kb", "mb", "gb", "tb", "pb", "eb" }; 30 | #define PUTS(...) pos += snprintf(buf + pos, sizeof(buf) - pos, __VA_ARGS__) 31 | char buf[2048]; 32 | int i, j, pos = 0; 33 | 34 | PUTS("[%s] [ " UTF8_UPARROW, server->config_name); 35 | 36 | for (i = 0; i < server->active_backends; ++i) { 37 | struct brubeck_backend *backend = server->backends[i]; 38 | if (backend->type == BRUBECK_BACKEND_CARBON) { 39 | struct brubeck_carbon *carbon = (struct brubeck_carbon *)backend; 40 | double sent = carbon->sent; 41 | 42 | for (j = 0; j < 7 && sent >= 1024.0; ++j) 43 | sent /= 1024.0; 44 | 45 | PUTS("%s #%d %.1f%s%s", 46 | (i > 0) ? "," : "", 47 | i + 1, sent, size_suffix[j], 48 | (carbon->out_sock >= 0) ? "" : " (dc)"); 49 | } 50 | } 51 | 52 | PUTS(" ] [ " UTF8_DOWNARROW); 53 | 54 | for (i = 0; i < server->active_samplers; ++i) { 55 | struct brubeck_sampler *sampler = server->samplers[i]; 56 | PUTS("%s :%d %d/s", 57 | (i > 0) ? "," : "", 58 | (int)ntohs(sampler->addr.sin_port), 59 | (int)sampler->current_flow); 60 | } 61 | 62 | PUTS(" ]"); 63 | setproctitle("brubeck", buf); 64 | } 65 | 66 | static void 67 | expire_metric(struct brubeck_metric *mt, void *_) 68 | { 69 | /* If this metric is not disabled, turn "inactive" 70 | * into "disabled" and "active" into "inactive" 71 | */ 72 | if (mt->expire > BRUBECK_EXPIRE_DISABLED) 73 | mt->expire = mt->expire - 1; 74 | } 75 | 76 | static void 77 | dump_metric(struct brubeck_metric *mt, void *out_file) 78 | { 79 | static const char *METRIC_NAMES[] = {"g", "c", "C", "h", "ms", "internal"}; 80 | fprintf((FILE *)out_file, "%s|%s\n", mt->key, METRIC_NAMES[mt->type]); 81 | } 82 | 83 | static void 84 | dump_all_metrics(struct brubeck_server *server) 85 | { 86 | FILE *dump = NULL; 87 | 88 | log_splunk("event=dump_metrics"); 89 | 90 | if (server->dump_path) 91 | dump = fopen(server->dump_path, "w+"); 92 | 93 | if (!dump) { 94 | log_splunk_errno("event=dump_failed"); 95 | return; 96 | } 97 | 98 | brubeck_hashtable_foreach(server->metrics, &dump_metric, dump); 99 | fclose(dump); 100 | } 101 | 102 | static void load_backends(struct brubeck_server *server, json_t *backends) 103 | { 104 | size_t idx; 105 | json_t *b; 106 | 107 | json_array_foreach(backends, idx, b) { 108 | const char *type = json_string_value(json_object_get(b, "type")); 109 | struct brubeck_backend *backend = NULL; 110 | 111 | if (type && !strcmp(type, "carbon")) { 112 | backend = brubeck_carbon_new(server, b, server->active_backends); 113 | server->backends[server->active_backends++] = backend; 114 | } else { 115 | log_splunk("backend=%s event=invalid_backend", type); 116 | } 117 | } 118 | } 119 | 120 | static void load_samplers(struct brubeck_server *server, json_t *samplers) 121 | { 122 | size_t idx; 123 | json_t *s; 124 | 125 | json_array_foreach(samplers, idx, s) { 126 | const char *type = json_string_value(json_object_get(s, "type")); 127 | 128 | if (type && !strcmp(type, "statsd")) { 129 | server->samplers[server->active_samplers++] = brubeck_statsd_new(server, s); 130 | } else if (type && !strcmp(type, "statsd-secure")) { 131 | server->samplers[server->active_samplers++] = brubeck_statsd_secure_new(server, s); 132 | } else { 133 | log_splunk("sampler=%s event=invalid_sampler", type); 134 | } 135 | } 136 | } 137 | 138 | static int load_timerfd(int interval) 139 | { 140 | struct itimerspec timer; 141 | int timerfd = timerfd_create(CLOCK_MONOTONIC, 0); 142 | 143 | if (timerfd < 0) 144 | die("failed to create timer"); 145 | 146 | memset(&timer, 0x0, sizeof(timer)); 147 | timer.it_value.tv_sec = interval; 148 | timer.it_value.tv_nsec = 0; 149 | timer.it_interval.tv_sec = interval; 150 | timer.it_interval.tv_nsec = 0; 151 | 152 | if (timerfd_settime(timerfd, 0, &timer, NULL) < 0) 153 | die("failed to set system timer"); 154 | 155 | return timerfd; 156 | } 157 | 158 | static int load_signalfd(void) 159 | { 160 | sigset_t mask; 161 | 162 | sigemptyset(&mask); 163 | sigaddset(&mask, SIGINT); 164 | sigaddset(&mask, SIGTERM); 165 | sigaddset(&mask, SIGHUP); 166 | sigaddset(&mask, SIGUSR2); 167 | 168 | if (sigprocmask(SIG_BLOCK, &mask, NULL) == -1) 169 | die("failed to sigprocmask the needed signals"); 170 | 171 | return signalfd(-1, &mask, 0); 172 | } 173 | 174 | static char *get_config_name(const char *full_path) 175 | { 176 | const char *filename = strrchr(full_path, '/'); 177 | char *config_name = strdup(filename ? filename + 1 : full_path); 178 | char *ext = strrchr(config_name, '.'); 179 | 180 | if (ext) 181 | *ext = '\0'; 182 | 183 | return config_name; 184 | } 185 | 186 | static void load_config(struct brubeck_server *server, const char *path) 187 | { 188 | json_error_t error; 189 | 190 | /* required */ 191 | int capacity; 192 | json_t *backends, *samplers; 193 | 194 | /* optional */ 195 | int expire = 0; 196 | char *http = NULL; 197 | 198 | server->name = "brubeck"; 199 | server->config_name = get_config_name(path); 200 | server->dump_path = NULL; 201 | server->config = json_load_file(path, 0, &error); 202 | if (!server->config) { 203 | die("failed to load config file, %s (%s:%d:%d)", 204 | error.text, error.source, error.line, error.column); 205 | } 206 | 207 | json_unpack_or_die(server->config, 208 | "{s?:s, s:s, s:i, s:o, s:o, s?:s, s?:i}", 209 | "server_name", &server->name, 210 | "dumpfile", &server->dump_path, 211 | "capacity", &capacity, 212 | "backends", &backends, 213 | "samplers", &samplers, 214 | "http", &http, 215 | "expire", &expire); 216 | 217 | gh_log_set_instance(server->name); 218 | 219 | server->metrics = brubeck_hashtable_new(1 << capacity); 220 | if (!server->metrics) 221 | die("failed to initialize hash table (size: %lu)", 1ul << capacity); 222 | 223 | load_backends(server, backends); 224 | load_samplers(server, samplers); 225 | 226 | if (http) brubeck_http_endpoint_init(server, http); 227 | if (expire) server->fd_expire = load_timerfd(expire); 228 | } 229 | 230 | void brubeck_server_init(struct brubeck_server *server, const char *config) 231 | { 232 | memset(server, 0x0, sizeof(struct brubeck_server)); 233 | 234 | /* ignore SIGPIPE here so we don't crash when 235 | * backends get disconnected */ 236 | signal(SIGPIPE, SIG_IGN); 237 | 238 | server->fd_signal = load_signalfd(); 239 | server->fd_update = load_timerfd(1); 240 | server->fd_expire = -1; 241 | 242 | /* init the memory allocator */ 243 | brubeck_slab_init(&server->slab); 244 | 245 | /* init the samplers and backends */ 246 | load_config(server, config); 247 | 248 | /* Init the internal stats */ 249 | brubeck_internal__init(server); 250 | } 251 | 252 | static int timer_elapsed(struct pollfd *fd) 253 | { 254 | if (fd->revents & POLLIN) { 255 | uint64_t timer; 256 | int s = read(fd->fd, &timer, sizeof(timer)); 257 | return (s == sizeof(timer)); 258 | } 259 | return 0; 260 | } 261 | 262 | static int signal_triggered(struct pollfd *fd) 263 | { 264 | if (fd->revents & POLLIN) { 265 | struct signalfd_siginfo fdsi; 266 | int s = read(fd->fd, &fdsi, sizeof(fdsi)); 267 | if (s == sizeof(fdsi)) 268 | return fdsi.ssi_signo; 269 | } 270 | return -1; 271 | } 272 | 273 | int brubeck_server_run(struct brubeck_server *server) 274 | { 275 | struct pollfd fds[3]; 276 | int nfd = 2; 277 | size_t i; 278 | 279 | memset(fds, 0x0, sizeof(fds)); 280 | 281 | fds[0].fd = server->fd_signal; 282 | fds[0].events = POLLIN; 283 | 284 | fds[1].fd = server->fd_update; 285 | fds[1].events = POLLIN; 286 | 287 | if (server->fd_expire >= 0) { 288 | fds[2].fd = server->fd_expire; 289 | fds[2].events = POLLIN; 290 | nfd++; 291 | } 292 | 293 | server->running = 1; 294 | log_splunk("event=listening"); 295 | 296 | while (server->running) { 297 | if (poll(fds, nfd, -1) < 0) 298 | continue; 299 | 300 | switch (signal_triggered(&fds[0])) { 301 | case SIGHUP: 302 | gh_log_reopen(); 303 | log_splunk("event=reload_log"); 304 | break; 305 | case SIGUSR2: 306 | dump_all_metrics(server); 307 | break; 308 | case SIGINT: 309 | case SIGTERM: 310 | server->running = 0; 311 | break; 312 | } 313 | 314 | if (timer_elapsed(&fds[1])) { 315 | update_flows(server); 316 | update_proctitle(server); 317 | } 318 | 319 | if (timer_elapsed(&fds[2])) { 320 | log_splunk("event=expire_metrics"); 321 | brubeck_hashtable_foreach(server->metrics, &expire_metric, NULL); 322 | } 323 | } 324 | 325 | for (i = 0; i < server->active_backends; ++i) 326 | pthread_cancel(server->backends[i]->thread); 327 | 328 | for (i = 0; i < server->active_samplers; ++i) { 329 | struct brubeck_sampler *sampler = server->samplers[i]; 330 | if (sampler->shutdown) 331 | sampler->shutdown(sampler); 332 | } 333 | 334 | log_splunk("event=shutdown"); 335 | return 0; 336 | } 337 | 338 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Brubeck (unmaintained) 2 | 3 | Brubeck is a [statsd](https://github.com/etsy/statsd)-compatible stats 4 | aggregator written in C. Brubeck is currently unmaintained. 5 | 6 | ## List of known maintained forks 7 | 8 | - https://github.com/lukepalmer/brubeck-new 9 | 10 | ## What is statsd? 11 | 12 | Statsd is a metrics aggregator for Graphite (and other data storage backends). This 13 | technical documentation assumes working knowledge of what statsd is and how it works; 14 | please read the [statsd documentation](https://github.com/etsy/statsd#statsd-) for 15 | more details. 16 | 17 | Statsd is a good idea, and if you're using Graphite for metrics collection in your 18 | infrastructure, you probably want a statsd-compatible aggregator in front of it. 19 | 20 | ## Tradeoffs 21 | 22 | - Brubeck is missing many of the features of the original StatsD. We've only implemented what we felt was necessary for our metrics stack. 23 | 24 | - Brubeck only runs on Linux. It won't even build on Mac OS X. 25 | 26 | - Some of the performance features require a (moderately) recent version of the kernel that you may not have. 27 | 28 | ## Building 29 | 30 | Brubeck has the following dependencies: 31 | 32 | - A Turing-complete computing device running a modern version of the Linux kernel 33 | (the kernel needs to be at least 2.6.33 in order to use multiple recvmsg support) 34 | 35 | - A compiler for the C programming language 36 | 37 | - Jansson (`libjansson-dev` on Debian) to load the configuration (version 2.5+ is required) 38 | 39 | - OpenSSL (`libcrypto`) if you're building StatsD-Secure support 40 | 41 | - libmicrohttpd (`libmicrohttpd-dev`) to have an internal HTTP stats endpoint. Build with `BRUBECK_NO_HTTP` to disable this. 42 | 43 | Build brubeck by typing: 44 | 45 | ./script/bootstrap 46 | 47 | Other operating systems or kernels can probably build Brubeck too. More specifically, 48 | Brubeck has been seen to work under FreeBSD and OpenBSD, but this is not supported. 49 | 50 | ## Supported Metric Types 51 | 52 | Brubeck supports most of the metric types from statsd and many other implementations. 53 | 54 | - `g` - Gauges 55 | - `c` - Meters 56 | - `C` - Counters 57 | - `h` - Histograms 58 | - `ms` - Timers (in milliseconds) 59 | 60 | Client-sent sampling rates are ignored. 61 | 62 | Visit the [statsd docs](https://github.com/etsy/statsd/blob/master/docs/metric_types.md) for more information on metric types. 63 | 64 | ## Interfacing 65 | 66 | The are several ways to interact with a running Brubeck daemon. 67 | 68 | ### Signals 69 | 70 | Brubeck answers to the following signals: 71 | 72 | - `SIGINT`, `SIGTERM`: shutdown cleanly 73 | - `SIGHUP`: reopen the log files (in case you're using logrotate or an equivalent) 74 | - `SIGUSR2`: dump a newline-separated list of all the metrics currently aggregated by the 75 | daemon and their types. 76 | 77 | ### HTTP Endpoint 78 | 79 | If enabled on the config file, Brubeck can provide an HTTP API to poll its status. The following routes are available: 80 | 81 | - `GET /ping`: return a short JSON payload with the current status of the daemon (just to check it's up) 82 | - `GET /stats`: get a large JSON payload with full statistics, including active endpoints and throughputs 83 | - `GET /metric/{{metric_name}}`: get the current status of a metric, if it's being aggregated 84 | - `POST /expire/{{metric_name}}`: expire a metric that is no longer being reported to stop it from being aggregated to the backend 85 | 86 | ## Configuration 87 | 88 | The configuration for Brubeck is loaded through a JSON file, passed on the commandline. 89 | 90 | ./brubeck --config=my.config.json 91 | 92 | If no configuration file is passed to the daemon, it will load `config.default.json`, which 93 | contains useful defaults for local development/testing. 94 | 95 | The JSON file can contain the following sections: 96 | 97 | - `server_name`: a string identifying the name for this specific Brubeck instance. This will 98 | be used by the daemon when reporting its internal metrics. 99 | 100 | - `dumpfile`: a path where to store the metrics list when triggering a dump (see the section on 101 | Interfacing with the daemon) 102 | 103 | - `http`: if existing, this string sets the listen address and port for the HTTP API 104 | 105 | - `backends`: an array of the different backends to load. If more than one backend is loaded, 106 | brubeck will function in sharding mode, distributing aggregation load evenly through all 107 | the different backends through constant-hashing. 108 | 109 | - `carbon`: a backend that aggregates data into a Carbon cache. The backend sends all the 110 | aggregated data once every `frequency` seconds. By default the data is sent to the port 2003 111 | of the Carbon cache (plain text protocol), but the pickle wire protocol can be enabled by 112 | setting `pickle` to `true` and changing the port accordingly. 113 | 114 | ``` 115 | { 116 | "type" : "carbon", 117 | "address" : "0.0.0.0", 118 | "port" : 2003, 119 | "frequency" : 10, 120 | "pickle: true 121 | } 122 | ``` 123 | 124 | We strongly encourage you to use the pickle wire protocol instead of plaintext, 125 | because carbon-relay.py is not very performant and will choke when parsing plaintext 126 | under enough load. Pickles are much softer CPU-wise on the Carbon relays, 127 | aggregators and caches. 128 | 129 | Hmmmm pickles. Now I'm hungry. Lincoln when's lunch? 130 | 131 | - `samplers`: an array of the different samplers to load. Samplers run on parallel and gather 132 | incoming metrics from the network. 133 | 134 | - `statsd`: the default statsd-compatible sampler. It listens on an UDP port for metrics 135 | packets. You can have more than one statsd sampler on the same daemon, but Brubeck was 136 | designed to support a single sampler taking the full metrics load on a single port. 137 | 138 | ``` 139 | { 140 | "type" : "statsd", 141 | "address" : "0.0.0.0", 142 | "port" : 8126, 143 | } 144 | ``` 145 | 146 | The StatsD sampler has the following options (and default values) for performance tuning: 147 | 148 | - `"workers" : 4` number of worker threads that will service the StatsD socket endpoint. More threads means emptying the socket faster, but the context switching and cache smashing will affect performance. In general, you can saturate your NIC as long as you have enough worker threads (one per core) and a fast enough CPU. Set this to 1 if you want to run the daemon in event-loop mode. But that'd be silly. This is not Node. 149 | 150 | - `"multisock" : false` if set to true, Brubeck will use the `SO_REUSEPORT` flag available since Linux 3.9 to create one socket per worker thread and bind it to the same address/port. The kernel will then round-robin between the threads without forcing them to race for the socket. This improves performance by up to 30%, try benchmarking this if your Kernel is recent enough. 151 | 152 | - `"multimsg" : 1` if set to greater than one, Brubeck will use the `recvmmsg` syscall (available since Linux 2.6.33) to read several UDP packets (the specified amount) in a single call and reduce the amount of context switches. This doesn't improve performance much with several worker threads, but may have an effect in a limited configuration with only one thread. Make it a power of two for better results. As always, benchmark. YMMV. 153 | 154 | - `statsd-secure`: like StatsD, but each packet has a HMAC that verifies its integrity. This is hella useful if you're running infrastructure in The Cloud (TM) (C) and you want to send back packets back to your VPN without them being tampered by third parties. 155 | 156 | ``` 157 | { 158 | "type" : "statsd-secure", 159 | "address" : "0.0.0.0", 160 | "port" : 9126, 161 | "max_drift" : 3, 162 | "hmac_key" : "750c783e6ab0b503eaa86e310a5db738", 163 | "replay_len" : 8000 164 | } 165 | ``` 166 | 167 | The `address` and `port` parts are obviously the same as in statsd. 168 | 169 | - `max_drift` defines the maximum time (in seconds) that packets can be delayed 170 | since they were sent from the origin. All metrics come with a timestamp, so metrics 171 | that drift more than this value will silently be discared. 172 | 173 | - `hmac_key` is the shared HMAC secret. The client sending the metrics must also know 174 | this in order to sign them. 175 | 176 | - `replay_len` is the size of the bloom filter that will be used to prevent replay 177 | attacks. We use a rolling bloom filter (one for every drift second), so `replay_len` 178 | should roughly be the amount of **unique** metrics you expect to receive in a 1s 179 | interval. 180 | 181 | **NOTE**: StatsD-secure doesn't run with multiple worker threads because verifying 182 | signatures is already slow enough. Don't use this in performance critical scenarios. 183 | 184 | **NOTE**: StatsD-secure uses a bloom filter to prevent replay attacks, so a small 185 | percentage of metrics *will* be dropped because of false positives. Take this into 186 | consideration. 187 | 188 | **NOTE**: An HMAC does *not* encrypt the packets, it just verifies its integrity. 189 | If you need to protect the content of the packets from eavesdropping, get those 190 | external machines in your VPN. 191 | 192 | **NOTE**: StatsD-secure may or may not be a good idea. If you have the chance to 193 | send all your metrics inside a VPN, I suggest you do that instead. 194 | 195 | ## Testing 196 | 197 | There's some tests in the `test` folder for key parts of the system (such as packet parsing, 198 | and all concurrent data access); besides that we test the behavior of the daemon live on staging 199 | and production systems. 200 | 201 | - Small changes are deployed into production as-is, straight from their feature branch. 202 | Deployment happens in 3 seconds for all the Brubeck instances in our infrastructure, so 203 | we can roll back into the master branch immediately if something fails. 204 | 205 | - For critical changes, we multiplex a copy of the metrics stream into an Unix domain socket, 206 | so we can have two instances of the daemon (old and new) aggregating to the production 207 | cluster and a staging cluster, and verify that the metrics flow into the two clusters is equivalent. 208 | 209 | - Benchmarking is performed on real hardware in our datacenter. The daemon is spammed with fake 210 | metrics across the network and we ensure that there are no regressions (particularly in the linear 211 | scaling between cores for the statsd sampler). 212 | 213 | When in doubt, please refer to the part of the MIT license that says *"THE SOFTWARE IS PROVIDED 214 | 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED"*. We use Brubeck in production and 215 | have been doing so for years, but we cannot make any promises regarding availability or 216 | performance. 217 | 218 | ## FAQ 219 | 220 | - **I cannot hit 4 million UDP metrics per second. I want my money back.** 221 | 222 | Make sure receiver-side scaling is properly configured in your kernel and that IRQs 223 | are being serviced by different cores, and that the daemon's threads are not 224 | pinned to a specific core. Make sure you're running the daemon in a physical machine 225 | and not a cheap cloud VPS. Make sure your NIC has the right drivers and it's not 226 | bottlenecking. Install a newer kernel and try running with `SO_REUSEPORT`. 227 | 228 | If nothing works, refunds are available upon request. Just get mad at me on Twitter. 229 | -------------------------------------------------------------------------------- /tests/sput.h: -------------------------------------------------------------------------------- 1 | /* 2 | * sput - Simple, Portable Unit Testing Framework for C/C++ v1.3.0 3 | * 4 | * http://www.lingua-systems.com/unit-testing/ 5 | * 6 | * 7 | * Copyright (c) 2011-2014 Lingua-Systems Software GmbH 8 | * 9 | * All rights reserved. 10 | * 11 | * 12 | * Redistribution and use in source and binary forms, with or without 13 | * modification, are permitted provided that the following conditions are 14 | * met: 15 | * 16 | * * Redistributions of source code must retain the above copyright notice, 17 | * this list of conditions and the following disclaimer. 18 | * * Redistributions in binary form must reproduce the above copyright 19 | * notice, this list of conditions and the following disclaimer in the 20 | * documentation and/or other materials provided with the distribution. 21 | * 22 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 23 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 24 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 25 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 26 | * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 27 | * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 28 | * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 29 | * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 30 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 31 | * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 32 | * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | 35 | 36 | #ifndef HAVE_SPUT_H 37 | #define HAVE_SPUT_H 38 | 39 | 40 | #ifdef __cplusplus 41 | extern "C" { 42 | #endif 43 | 44 | 45 | #include 46 | #include 47 | #include 48 | #include 49 | 50 | 51 | /* =================================================================== 52 | * definitions 53 | * =================================================================== */ 54 | 55 | #define SPUT_VERSION_MAJOR 1 56 | #define SPUT_VERSION_MINOR 3 57 | #define SPUT_VERSION_PATCH 0 58 | #define SPUT_VERSION_STRING "1.3.0" 59 | 60 | #define SPUT_DEFAULT_SUITE_NAME "Unlabeled Suite" 61 | #define SPUT_DEFAULT_CHECK_NAME "Unlabeled Check" 62 | 63 | #define SPUT_INITIALIZED 0x06 /* ACK */ 64 | 65 | 66 | /* =================================================================== 67 | * sput global variable 68 | * =================================================================== */ 69 | 70 | struct sput 71 | { 72 | FILE *out; 73 | char initialized; 74 | 75 | struct sput_overall 76 | { 77 | unsigned long checks; 78 | unsigned long suites; 79 | unsigned long ok; 80 | unsigned long nok; 81 | } overall; 82 | 83 | struct sput_suite 84 | { 85 | const char *name; 86 | unsigned long nr; 87 | unsigned long checks; 88 | unsigned long ok; 89 | unsigned long nok; 90 | } suite; 91 | 92 | struct sput_test 93 | { 94 | const char *name; 95 | unsigned long nr; 96 | } test; 97 | 98 | struct sput_check 99 | { 100 | const char *name; 101 | const char *cond; 102 | const char *type; 103 | unsigned long line; 104 | } check; 105 | 106 | struct sput_time 107 | { 108 | time_t start; 109 | time_t end; 110 | } time; 111 | }; 112 | 113 | extern struct sput __sput; 114 | 115 | 116 | /* ================================================================== 117 | * sput internal macros 118 | * ================================================================== */ 119 | 120 | #define _sput_die_unless_initialized() \ 121 | if (__sput.initialized != SPUT_INITIALIZED) \ 122 | { \ 123 | fputs("sput_start_testing() omitted\n", stderr); \ 124 | exit(EXIT_FAILURE); \ 125 | } 126 | 127 | 128 | #define _sput_die_unless_suite_set() \ 129 | if (! __sput.suite.name) \ 130 | { \ 131 | fputs("sput_enter_suite() omitted\n", __sput.out); \ 132 | exit(EXIT_FAILURE); \ 133 | } 134 | 135 | 136 | #define _sput_die_unless_test_set() \ 137 | if (! __sput.test.name) \ 138 | { \ 139 | fputs("sput_run_test() omitted\n", __sput.out); \ 140 | exit(EXIT_FAILURE); \ 141 | } 142 | 143 | 144 | #define _sput_check_failed() \ 145 | { \ 146 | _sput_die_unless_initialized(); \ 147 | _sput_die_unless_suite_set(); \ 148 | __sput.suite.nok++; \ 149 | fprintf(__sput.out, \ 150 | "[%lu:%lu] %s:#%lu \"%s\" FAIL\n" \ 151 | "! Type: %s\n" \ 152 | "! Condition: %s\n" \ 153 | "! Line: %lu\n", \ 154 | __sput.suite.nr, __sput.suite.checks, __sput.test.name, \ 155 | __sput.test.nr, __sput.check.name, __sput.check.type, \ 156 | __sput.check.cond, __sput.check.line); \ 157 | } 158 | 159 | 160 | #define _sput_check_succeeded() \ 161 | { \ 162 | _sput_die_unless_initialized(); \ 163 | _sput_die_unless_suite_set(); \ 164 | __sput.suite.ok++; \ 165 | fprintf(__sput.out, \ 166 | "[%lu:%lu] %s:#%lu \"%s\" pass\n", \ 167 | __sput.suite.nr, __sput.suite.checks, \ 168 | __sput.test.name, \ 169 | __sput.test.nr, \ 170 | __sput.check.name); \ 171 | } 172 | 173 | 174 | /* ================================================================== 175 | * user macros 176 | * ================================================================== */ 177 | 178 | #define sput_start_testing() \ 179 | do { \ 180 | memset(&__sput, 0, sizeof(__sput)); \ 181 | __sput.out = stdout; \ 182 | __sput.time.start = time(NULL); \ 183 | __sput.initialized = SPUT_INITIALIZED; \ 184 | } while (0) 185 | 186 | 187 | #define sput_leave_suite() \ 188 | do { \ 189 | float failp = 0.0f; \ 190 | _sput_die_unless_initialized(); \ 191 | _sput_die_unless_suite_set(); \ 192 | failp = __sput.suite.checks ? (float) \ 193 | ((__sput.suite.nok * 100.0) / __sput.suite.checks) : \ 194 | 0.0f; \ 195 | fprintf(__sput.out, \ 196 | "\n--> %lu check(s), %lu ok, %lu failed (%.2f%%)\n", \ 197 | __sput.suite.checks, __sput.suite.ok, __sput.suite.nok, \ 198 | failp); \ 199 | __sput.overall.checks += __sput.suite.checks; \ 200 | __sput.overall.ok += __sput.suite.ok; \ 201 | __sput.overall.nok += __sput.suite.nok; \ 202 | memset(&__sput.suite, 0, sizeof(__sput.suite)); \ 203 | } while (0) 204 | 205 | 206 | #define sput_get_return_value() \ 207 | (__sput.overall.nok > 0 ? EXIT_FAILURE : EXIT_SUCCESS) 208 | 209 | 210 | #define sput_enter_suite(_name) \ 211 | do { \ 212 | _sput_die_unless_initialized(); \ 213 | if (__sput.suite.name) \ 214 | { \ 215 | sput_leave_suite(); \ 216 | } \ 217 | __sput.suite.name = _name != NULL ? \ 218 | _name : SPUT_DEFAULT_SUITE_NAME; \ 219 | __sput.suite.nr = ++__sput.overall.suites; \ 220 | fprintf(__sput.out, "\n== Entering suite #%lu, \"%s\" ==\n\n", \ 221 | __sput.suite.nr, __sput.suite.name); \ 222 | } while (0) 223 | 224 | 225 | #define sput_finish_testing() \ 226 | do { \ 227 | float failp = 0.0f; \ 228 | _sput_die_unless_initialized(); \ 229 | if (__sput.suite.name) \ 230 | { \ 231 | sput_leave_suite(); \ 232 | } \ 233 | failp = __sput.overall.checks ? (float) \ 234 | ((__sput.overall.nok * 100.0) / __sput.overall.checks) : \ 235 | 0.0f; \ 236 | __sput.time.end = time(NULL); \ 237 | fprintf(__sput.out, \ 238 | "\n==> %lu check(s) in %lu suite(s) finished after %.2f " \ 239 | "second(s),\n" \ 240 | " %lu succeeded, %lu failed (%.2f%%)\n" \ 241 | "\n[%s]\n", \ 242 | __sput.overall.checks, __sput.overall.suites, \ 243 | difftime(__sput.time.end, __sput.time.start), \ 244 | __sput.overall.ok, __sput.overall.nok, failp, \ 245 | (sput_get_return_value() == EXIT_SUCCESS) ? \ 246 | "SUCCESS" : "FAILURE"); \ 247 | } while (0) 248 | 249 | 250 | #define sput_set_output_stream(_fp) \ 251 | do { \ 252 | __sput.out = _fp != NULL ? _fp : stdout; \ 253 | } while (0) 254 | 255 | 256 | #define sput_fail_if(_cond, _name) \ 257 | do { \ 258 | _sput_die_unless_initialized(); \ 259 | _sput_die_unless_suite_set(); \ 260 | _sput_die_unless_test_set(); \ 261 | __sput.check.name = _name != NULL ? \ 262 | _name : SPUT_DEFAULT_CHECK_NAME; \ 263 | __sput.check.line = __LINE__; \ 264 | __sput.check.cond = #_cond; \ 265 | __sput.check.type = "fail-if"; \ 266 | __sput.test.nr++; \ 267 | __sput.suite.checks++; \ 268 | if ((_cond)) \ 269 | { \ 270 | _sput_check_failed(); \ 271 | } \ 272 | else \ 273 | { \ 274 | _sput_check_succeeded(); \ 275 | } \ 276 | } while (0) 277 | 278 | 279 | #define sput_fail_unless(_cond, _name) \ 280 | do { \ 281 | _sput_die_unless_initialized(); \ 282 | _sput_die_unless_suite_set(); \ 283 | _sput_die_unless_test_set(); \ 284 | __sput.check.name = _name != NULL ? \ 285 | _name : SPUT_DEFAULT_CHECK_NAME; \ 286 | __sput.check.line = __LINE__; \ 287 | __sput.check.cond = #_cond; \ 288 | __sput.check.type = "fail-unless"; \ 289 | __sput.test.nr++; \ 290 | __sput.suite.checks++; \ 291 | if (! (_cond)) \ 292 | { \ 293 | _sput_check_failed(); \ 294 | } \ 295 | else \ 296 | { \ 297 | _sput_check_succeeded(); \ 298 | } \ 299 | } while (0) 300 | 301 | 302 | #define sput_run_test(_func) \ 303 | do { \ 304 | _sput_die_unless_initialized(); \ 305 | _sput_die_unless_suite_set(); \ 306 | memset(&__sput.test, 0, sizeof(__sput.test)); \ 307 | __sput.test.name = #_func; \ 308 | _func(); \ 309 | } while (0) 310 | 311 | 312 | #ifdef __cplusplus 313 | } 314 | #endif 315 | 316 | 317 | #endif /* HAVE_SPUT_H */ 318 | 319 | 320 | /* vim: set sts=4 sw=4 ts=4 ai et ft=c: */ 321 | --------------------------------------------------------------------------------