├── .gitignore ├── test_core.h ├── test_core.c ├── tsdb_bitmap.h ├── README ├── Makefile ├── tsdb_bitmap.c ├── tsdb_trace.h ├── tsdb_info.c ├── tsdb_trace.c ├── test_simple.c ├── tsdb_create.c ├── tsdb_set.c ├── test_bitmaps.c ├── tsdb_api.h ├── quicklz.h ├── test_tags.c ├── tsdb_get.c ├── seatest.h ├── test_advanced.c ├── seatest.c ├── LICENSE ├── NOTES ├── quicklz.c └── tsdb_api.c /.gitignore: -------------------------------------------------------------------------------- 1 | *.a 2 | *.o 3 | test-* 4 | tsdb-* 5 | -------------------------------------------------------------------------------- /test_core.h: -------------------------------------------------------------------------------- 1 | #include "seatest.h" 2 | #include "tsdb_api.h" 3 | 4 | extern int file_exists(char *filename); 5 | -------------------------------------------------------------------------------- /test_core.c: -------------------------------------------------------------------------------- 1 | #include "test_core.h" 2 | 3 | int file_exists(char *filename) { 4 | FILE *file; 5 | if ((file = fopen(filename, "r"))) { 6 | fclose(file); 7 | return 1; 8 | } else { 9 | return 0; 10 | } 11 | } 12 | 13 | -------------------------------------------------------------------------------- /tsdb_bitmap.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | enum { BITS_PER_WORD = sizeof(u_int32_t) * CHAR_BIT }; 5 | 6 | #define WORD_OFFSET(b) ((b) / BITS_PER_WORD) 7 | #define BIT_OFFSET(b) ((b) % BITS_PER_WORD) 8 | 9 | void set_bit(u_int32_t *words, int n); 10 | 11 | void clear_bit(u_int32_t *words, int n); 12 | 13 | int get_bit(u_int32_t *words, int n); 14 | 15 | void scan_result(u_int32_t *result, 16 | u_int32_t max_index, 17 | void(*handler)(u_int32_t *index)); 18 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | TSDB (TimeSeries DataBase) 2 | -------------------------- 3 | 4 | Timeseries (http://en.wikipedia.org/wiki/Time_series) are used to represent events at specific times. Popular tools such as rrdtool (http://www.rrdtool.org) made it possible to handle time series easily. Unfortunately RRD falls short when 5 | - handling thousand/million time series due to high update time and disk-space used 6 | - data have to be exported on a web 2.0 GUI 7 | 8 | For this reason we developed TSDB. 9 | 10 | test.c 11 | Allows you to play with the API and see how it works 12 | 13 | tsdbExport 14 | Allows you to export data in both text and Json format 15 | 16 | 17 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | CC = gcc -g 2 | CFLAGS = -Wall -I. -DSEATEST_EXIT_ON_FAIL 3 | LDFLAGS = -L /opt/local/lib 4 | SYSLIBS = -lrrd -ldb 5 | 6 | TSDB_LIB = libtsdb.a 7 | TSDB_LIB_O = tsdb_api.o tsdb_trace.o tsdb_bitmap.o quicklz.o 8 | 9 | TEST_LIBS = $(TSDB_LIB) test_core.o seatest.o 10 | 11 | TARGETS = $(TSDB_LIB) \ 12 | tsdb-create \ 13 | tsdb-info \ 14 | tsdb-set \ 15 | tsdb-get \ 16 | test-simple \ 17 | test-advanced \ 18 | test-bitmaps \ 19 | test-tags 20 | 21 | all: $(TARGETS) 22 | 23 | %.o: %.c %.h 24 | ${CC} ${CFLAGS} ${INCLUDE} -c $< -o $@ 25 | 26 | $(TSDB_LIB): $(TSDB_LIB_O) 27 | ar rs $@ ${TSDB_LIB_O} 28 | ranlib $@ 29 | 30 | tsdb-%: tsdb_%.o $(TSDB_LIB) 31 | $(CC) $(LDFLAGS) tsdb_$*.o $(TSDB_LIB) $(SYSLIBS) -o $@ 32 | 33 | test-%: test_%.o $(TEST_LIBS) 34 | $(CC) $(LDFLAGS) test_$*.o $(TEST_LIBS) $(SYSLIBS) -o $@ 35 | 36 | clean: 37 | rm -f ${TARGETS} *.o *~ 38 | 39 | .SECONDARY: $(TEST_LIBS) 40 | -------------------------------------------------------------------------------- /tsdb_bitmap.c: -------------------------------------------------------------------------------- 1 | #include "tsdb_bitmap.h" 2 | 3 | void set_bit(u_int32_t *words, int n) { 4 | words[WORD_OFFSET(n)] |= (1 << BIT_OFFSET(n)); 5 | } 6 | 7 | void clear_bit(u_int32_t *words, int n) { 8 | words[WORD_OFFSET(n)] &= ~(1 << BIT_OFFSET(n)); 9 | } 10 | 11 | int get_bit(u_int32_t *words, int n) { 12 | u_int32_t bit = words[WORD_OFFSET(n)] & (1 << BIT_OFFSET(n)); 13 | return bit != 0; 14 | } 15 | 16 | void scan_result(u_int32_t *result, u_int32_t max_index, 17 | void(*handler)(u_int32_t *index)) { 18 | u_int32_t i, j, index; 19 | u_int32_t max_word = max_index / BITS_PER_WORD; 20 | 21 | for (i = 0; i <= max_word; i++) { 22 | if (result[i] == 0) { 23 | continue; 24 | } 25 | for (j = 0; j < BITS_PER_WORD; j++) { 26 | index = i * BITS_PER_WORD + j; 27 | if (index > max_index) { 28 | break; 29 | } 30 | if (get_bit(result, index) && handler) { 31 | handler(&index); 32 | } 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /tsdb_trace.h: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright (C) 2011 IIT/CNR (http://www.iit.cnr.it/en) 4 | * Luca Deri 5 | * 6 | * This program is free software; you can redistribute it and/or modify 7 | * it under the terms of the GNU General Public License as published by 8 | * the Free Software Foundation; either version 2 of the License, or 9 | * (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License along 17 | * with this program; if not, write to the Free Software Foundation, Inc., 18 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 19 | */ 20 | 21 | #define TRACE_ERROR 1, __FILE__, __LINE__ 22 | #define TRACE_WARNING 2, __FILE__, __LINE__ 23 | #define TRACE_INFO 3, __FILE__, __LINE__ 24 | 25 | #define trace_error(...) trace_event(TRACE_ERROR, __VA_ARGS__) 26 | #define trace_warning(...) trace_event(TRACE_WARNING, __VA_ARGS__) 27 | #define trace_info(...) trace_event(TRACE_INFO, __VA_ARGS__) 28 | 29 | #define set_trace_level(level) __trace_level = level 30 | 31 | extern int __trace_level; 32 | extern void trace_event(int level, char* file, int line, char * format, ...); 33 | -------------------------------------------------------------------------------- /tsdb_info.c: -------------------------------------------------------------------------------- 1 | #include "tsdb_api.h" 2 | 3 | typedef struct { 4 | char *file; 5 | } info_args; 6 | 7 | static void help(int code) { 8 | printf("tsdb-info FILE\n"); 9 | exit(code); 10 | } 11 | 12 | static void process_args(int argc, char *argv[], info_args *args) { 13 | int c; 14 | while ((c = getopt(argc, argv, "h")) != -1) { 15 | switch (c) { 16 | case 'h': 17 | help(0); 18 | break; 19 | default: 20 | help(1); 21 | } 22 | } 23 | 24 | int remaining = argc - optind; 25 | if (remaining != 1) { 26 | help(1); 27 | } 28 | args->file = argv[optind]; 29 | } 30 | 31 | static void check_file_exists(const char *path) { 32 | FILE *file; 33 | if ((file = fopen(path, "r"))) { 34 | fclose(file); 35 | } else { 36 | printf("tsdb-info: %s doesn't exist\n", path); 37 | exit(1); 38 | } 39 | } 40 | 41 | static void print_db_info(char *file) { 42 | tsdb_handler db; 43 | int rc; 44 | u_int16_t unused16 = 0; 45 | u_int32_t unused32 = 0; 46 | struct stat info; 47 | 48 | rc = tsdb_open(file, &db, &unused16, unused32, 1); 49 | if (rc) { 50 | printf("tsdb-info: error opening %s\n", file); 51 | exit(1); 52 | } 53 | 54 | rc = stat(file, &info); 55 | if (rc) { 56 | printf("tsdb-info: error getting file info for %s\n", file); 57 | exit(1); 58 | } 59 | 60 | printf(" File: %s\n", file); 61 | printf(" Size: %zd\n", info.st_size); 62 | printf("Vals Per Entry: %u\n", db.values_per_entry); 63 | printf(" Slot Seconds: %u\n", db.slot_duration);; 64 | tsdb_close(&db); 65 | } 66 | 67 | int main(int argc, char *argv[]) { 68 | info_args args; 69 | 70 | set_trace_level(0); 71 | process_args(argc, argv, &args); 72 | check_file_exists(args.file); 73 | print_db_info(args.file); 74 | 75 | return 0; 76 | } 77 | -------------------------------------------------------------------------------- /tsdb_trace.c: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright (C) 2011 IIT/CNR (http://www.iit.cnr.it/en) 4 | * Luca Deri 5 | * 6 | * 7 | * This program is free software; you can redistribute it and/or modify 8 | * it under the terms of the GNU General Public License as published by 9 | * the Free Software Foundation; either version 2 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU General Public License along 18 | * with this program; if not, write to the Free Software Foundation, Inc., 19 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 20 | */ 21 | 22 | #include "tsdb_api.h" 23 | 24 | int __trace_level = 2; 25 | 26 | void trace_event(int level, char* file, int line, char *format, ...) { 27 | if (level > __trace_level) { 28 | return; 29 | } 30 | 31 | va_list va_ap; 32 | char buf[2048], out_buf[640]; 33 | char theDate[32], *extra_msg = ""; 34 | time_t theTime = time(NULL); 35 | 36 | va_start (va_ap, format); 37 | memset(buf, 0, sizeof(buf)); 38 | strftime(theDate, 32, "%d/%b/%Y %H:%M:%S", localtime(&theTime)); 39 | 40 | vsnprintf(buf, sizeof(buf) - 1, format, va_ap); 41 | 42 | if (level == 1) { 43 | extra_msg = "ERROR: "; 44 | } else if(level == 2) { 45 | extra_msg = "WARNING: "; 46 | } 47 | 48 | while (buf[strlen(buf) - 1] == '\n') { 49 | buf[strlen(buf) - 1] = '\0'; 50 | } 51 | 52 | snprintf(out_buf, sizeof(out_buf), "%s [%s:%d] %s%s", theDate, 53 | #ifdef WIN32 54 | strrchr(file, '\\') + 1, 55 | #else 56 | file, 57 | #endif 58 | line, extra_msg, buf); 59 | printf("%s\n", out_buf); 60 | 61 | fflush(stdout); 62 | va_end(va_ap); 63 | } 64 | 65 | -------------------------------------------------------------------------------- /test_simple.c: -------------------------------------------------------------------------------- 1 | #include "test_core.h" 2 | 3 | static char *get_file_arg(int argc, char *argv[]) { 4 | if (argc < 2) { 5 | fprintf(stderr, "usage: test-simple TEST-DB\n"); 6 | exit(1); 7 | } 8 | char *file = argv[1]; 9 | if (file_exists(file)) { 10 | fprintf(stderr, "%s exists\n", file); 11 | exit(1); 12 | } 13 | return file; 14 | } 15 | 16 | #define num_epochs 10 17 | #define slot_seconds 60 18 | #define num_keys 10001 19 | 20 | int main(int argc, char *argv[]) { 21 | 22 | char *file = get_file_arg(argc, argv); 23 | 24 | tsdb_handler db; 25 | int ret; 26 | int cur, start, stop; 27 | char key[255]; 28 | uint i; 29 | uint write_val; 30 | uint *read_val; 31 | 32 | set_trace_level(1); 33 | 34 | // Open (create) a new db. 35 | 36 | u_int16_t vals_per_entry = 1; 37 | ret = tsdb_open(file, &db, &vals_per_entry, slot_seconds, 0); 38 | assert_int_equal(0, ret); 39 | 40 | // Move through epochs for write. 41 | 42 | start = 1000000000; 43 | stop = start + (num_epochs - 1) * slot_seconds; 44 | 45 | for (cur = start; cur <= stop; cur += slot_seconds) { 46 | 47 | ret = tsdb_goto_epoch(&db, cur, 0, 1); 48 | assert_int_equal(0, ret); 49 | 50 | // Write keys. 51 | 52 | for (i = 1; i <= num_keys; i++) { 53 | sprintf(key, "key-%i", i); 54 | write_val = i * 1000; 55 | ret = tsdb_set(&db, key, &write_val); 56 | assert_int_equal(0, ret); 57 | } 58 | } 59 | 60 | tsdb_flush(&db); 61 | 62 | // Move through epochs for reads. 63 | 64 | for (cur = start; cur <= stop; cur += slot_seconds) { 65 | 66 | ret = tsdb_goto_epoch(&db, cur, 1, 0); 67 | assert_int_equal(0, ret); 68 | 69 | // Read keys. 70 | 71 | for (i = 1; i <= num_keys; i++) { 72 | sprintf(key, "key-%i", i); 73 | ret = tsdb_get_by_key(&db, key, &read_val); 74 | assert_int_equal(0, ret); 75 | write_val = i * 1000; 76 | assert_int_equal(write_val, *read_val); 77 | } 78 | } 79 | 80 | tsdb_close(&db); 81 | 82 | return 0; 83 | } 84 | -------------------------------------------------------------------------------- /tsdb_create.c: -------------------------------------------------------------------------------- 1 | #include "tsdb_api.h" 2 | 3 | typedef struct { 4 | char *file; 5 | u_int32_t slot_seconds; 6 | u_int16_t values_per_entry; 7 | int verbose; 8 | } create_args; 9 | 10 | static void init_trace(int verbose) { 11 | set_trace_level(verbose ? 99 : 0); 12 | } 13 | 14 | static void help(int code) { 15 | printf("tsdb-create [-v] file slot_seconds [values_per_entry]\n"); 16 | exit(code); 17 | } 18 | 19 | static void check_file_exists(const char *path) { 20 | FILE *file; 21 | if ((file = fopen(path, "r"))) { 22 | fclose(file); 23 | printf("tsdb-create: %s already exists\n", path); 24 | exit(1); 25 | } 26 | } 27 | 28 | static u_int32_t str_to_uint32(const char *str, const char *argname) { 29 | uint num = 0; 30 | if (sscanf(str, "%u", &num) == EOF) { 31 | printf("tsdb-create: invalid value for %s\n", argname); 32 | exit(1); 33 | } 34 | return num; 35 | } 36 | 37 | static void process_create_args(int argc, char *argv[], create_args *args) { 38 | int c; 39 | 40 | args->verbose = 0; 41 | 42 | while ((c = getopt(argc, argv, "hv")) != -1) { 43 | switch (c) { 44 | case 'h': 45 | help(0); 46 | break; 47 | case 'v': 48 | args->verbose = 1; 49 | break; 50 | default: 51 | help(1); 52 | } 53 | } 54 | 55 | int remaining = argc - optind; 56 | if (remaining < 2 || remaining > 3) { 57 | help(1); 58 | } 59 | args->file = argv[optind]; 60 | args->slot_seconds = str_to_uint32(argv[optind + 1], "slot_seconds"); 61 | if (remaining == 3) { 62 | args->values_per_entry = 63 | str_to_uint32(argv[optind + 2], "values_per_entry"); 64 | } else { 65 | args->values_per_entry = 1; 66 | } 67 | } 68 | 69 | static void validate_slot_seconds(u_int32_t slot_seconds) { 70 | if (slot_seconds == 0) { 71 | printf("tsdb-create: invalid value for slot_seconds\n"); 72 | exit(1); 73 | } 74 | } 75 | 76 | static void validate_values_per_entry(u_int16_t values_per_entry) { 77 | if (values_per_entry == 0) { 78 | printf("tsdb-create: invalid value for values_per_entry\n"); 79 | exit(1); 80 | } 81 | } 82 | 83 | static void create_db(char *file, 84 | u_int32_t slot_seconds, 85 | u_int16_t values_per_entry) { 86 | tsdb_handler handler; 87 | int rc; 88 | rc = tsdb_open(file, &handler, &values_per_entry, slot_seconds, 0); 89 | if (rc) { 90 | printf("tsdb-create: error creating database\n"); 91 | exit(1); 92 | } 93 | tsdb_close(&handler); 94 | } 95 | 96 | int main(int argc, char *argv[]) { 97 | 98 | create_args args; 99 | 100 | process_create_args(argc, argv, &args); 101 | init_trace(args.verbose); 102 | check_file_exists(args.file); 103 | validate_slot_seconds(args.slot_seconds); 104 | validate_values_per_entry(args.values_per_entry); 105 | create_db(args.file, args.slot_seconds, args.values_per_entry); 106 | 107 | return 0; 108 | } 109 | -------------------------------------------------------------------------------- /tsdb_set.c: -------------------------------------------------------------------------------- 1 | #include "tsdb_api.h" 2 | 3 | typedef struct { 4 | char *file; 5 | char *key; 6 | u_int32_t timestamp; 7 | int values_start; 8 | int values_stop; 9 | char **argv; 10 | int verbose; 11 | } set_args; 12 | 13 | static void help(int code) { 14 | printf("tsdb-set [-v] file key [-t timestamp] [values]\n"); 15 | exit(code); 16 | } 17 | 18 | static u_int32_t str_to_uint32(const char *str, const char *argname) { 19 | uint num = 0; 20 | if (sscanf(str, "%u", &num) == EOF) { 21 | printf("tsdb-set: invalid value for %s\n", argname); 22 | exit(1); 23 | } 24 | return num; 25 | } 26 | 27 | static void process_args(int argc, char *argv[], set_args *args) { 28 | int c; 29 | 30 | args->timestamp = 0; 31 | args->verbose = 0; 32 | 33 | while ((c = getopt(argc, argv, "hvt:")) != -1) { 34 | switch (c) { 35 | case 'h': 36 | help(0); 37 | break; 38 | case 'v': 39 | args->verbose = 1; 40 | break; 41 | case 't': 42 | args->timestamp = str_to_uint32(optarg, "timestamp"); 43 | break; 44 | default: 45 | help(1); 46 | } 47 | } 48 | 49 | int remaining = argc - optind; 50 | if (remaining < 2) { 51 | help(1); 52 | } 53 | 54 | args->file = argv[optind]; 55 | args->key = argv[optind + 1]; 56 | args->values_start = optind + 2; 57 | args->values_stop = argc - 1; 58 | args->argv = argv; 59 | } 60 | 61 | static void check_file_exists(const char *path) { 62 | FILE *file; 63 | if ((file = fopen(path, "r"))) { 64 | fclose(file); 65 | } else { 66 | printf("tsdb-set: %s doesn't exist (use tsdb-create)\n", path); 67 | exit(1); 68 | } 69 | } 70 | 71 | static void open_db(char *file, tsdb_handler *db) { 72 | u_int16_t unused16 = 0; 73 | u_int32_t unused32 = 0; 74 | if (tsdb_open(file, db, &unused16, unused32, 0)) { 75 | printf("tsdb-set: error opening db %s\n", file); 76 | exit(1); 77 | } 78 | } 79 | 80 | static void goto_epoch(tsdb_handler *db, u_int32_t epoch) { 81 | if (tsdb_goto_epoch(db, epoch, 0, 1) ) { 82 | printf("tsdb-set: error settting epoch\n"); 83 | exit(1); 84 | } 85 | } 86 | 87 | static void set_values(tsdb_handler *db, char* key, tsdb_value *values) { 88 | if (tsdb_set(db, key, values)) { 89 | printf("tsdb-set: error setting value\n"); 90 | exit(1); 91 | } 92 | } 93 | 94 | static tsdb_value *alloc_values(set_args *args, int value_count) { 95 | tsdb_value *values = calloc(value_count, sizeof(tsdb_value)); 96 | int cur_value = args->values_start; 97 | int i = 0; 98 | while (i < value_count && cur_value <= args->values_stop) { 99 | values[i++] = str_to_uint32(args->argv[cur_value++], "value"); 100 | } 101 | return values; 102 | } 103 | 104 | static void set_tsdb_values(set_args *args) { 105 | tsdb_handler db; 106 | tsdb_value *values; 107 | u_int32_t epoch = (args->timestamp ? args->timestamp : time(NULL)); 108 | 109 | open_db(args->file, &db); 110 | values = alloc_values(args, db.values_per_entry); 111 | goto_epoch(&db, epoch); 112 | set_values(&db, args->key, values); 113 | free(values); 114 | tsdb_close(&db); 115 | } 116 | 117 | static void init_trace(int verbose) { 118 | set_trace_level(verbose ? 99 : 0); 119 | } 120 | 121 | int main(int argc, char *argv[]) { 122 | set_args args; 123 | 124 | process_args(argc, argv, &args); 125 | init_trace(args.verbose); 126 | check_file_exists(args.file); 127 | set_tsdb_values(&args); 128 | 129 | return 0; 130 | } 131 | -------------------------------------------------------------------------------- /test_bitmaps.c: -------------------------------------------------------------------------------- 1 | #include "test_core.h" 2 | #include "tsdb_bitmap.h" 3 | 4 | void print_title(char *title) { 5 | u_int32_t i, padding = 18 - strlen(title); 6 | printf("%s:", title); 7 | for (i = 0; i < padding; i++) { 8 | printf(" "); 9 | } 10 | } 11 | 12 | void print_index(u_int32_t *index) { 13 | printf(" %u", *index); 14 | } 15 | 16 | void print_result(u_int32_t *result, u_int32_t max_index, char *title) { 17 | print_title(title); 18 | scan_result(result, max_index, print_index); 19 | printf("\n"); 20 | } 21 | 22 | void compress_array(u_int32_t *array, u_int32_t array_len, 23 | char *compressed, u_int32_t *compressed_len) { 24 | u_int32_t new_len = array_len + 400; 25 | qlz_state_compress state_compress; 26 | memset(&state_compress, 0, sizeof(state_compress)); 27 | compressed = (char*)malloc(new_len); 28 | *compressed_len = qlz_compress(array, compressed, array_len, 29 | &state_compress); 30 | } 31 | 32 | int main(int argc, char *argv[]) { 33 | u_int32_t i, j; 34 | u_int8_t word_size = sizeof(u_int32_t); 35 | u_int32_t array_len = CHUNK_GROWTH / word_size; 36 | 37 | // three arrays, each representing a tag 38 | 39 | u_int32_t tag1[array_len]; memset(tag1, 0, sizeof(tag1)); 40 | u_int32_t tag2[array_len]; memset(tag2, 0, sizeof(tag2)); 41 | u_int32_t tag3[array_len]; memset(tag3, 0, sizeof(tag3)); 42 | 43 | // array for operation results 44 | 45 | u_int32_t result[array_len]; 46 | 47 | // to tag an index, we set the bit on the applicable array 48 | 49 | set_bit(tag1, 0); 50 | set_bit(tag3, 0); 51 | 52 | set_bit(tag2, 1); 53 | 54 | set_bit(tag2, 2); 55 | set_bit(tag3, 2); 56 | 57 | set_bit(tag1, 3); 58 | set_bit(tag2, 3); 59 | set_bit(tag3, 3); 60 | 61 | set_bit(tag3, 4); 62 | 63 | set_bit(tag1, 9999); 64 | 65 | u_int32_t max_index = 9999; 66 | u_int32_t array_max_used = max_index / word_size; 67 | 68 | printf("Size of tag array uncompressed: %lu\n", sizeof(tag1)); 69 | 70 | char *compressed = 0; 71 | u_int32_t compressed_len; 72 | compress_array(tag1, sizeof(tag1), compressed, &compressed_len); 73 | printf("Size of tag array compressed: %u\n", compressed_len); 74 | free(compressed); 75 | 76 | printf("\n"); 77 | 78 | // tag1 79 | // 80 | for (i = 0; i < array_max_used; i++) { 81 | result[i] = tag1[i]; 82 | } 83 | print_result(result, max_index, "tag1"); 84 | 85 | // tag1 and tag2 86 | // 87 | for (i = 0; i < array_max_used; i++) { 88 | result[i] = tag1[i] & tag2[i]; 89 | } 90 | print_result(result, max_index, "tag1 and tag2"); 91 | 92 | // tag2 and tag3 93 | // 94 | for (i = 0; i < array_max_used; i++) { 95 | result[i] = tag2[i] & tag3[i]; 96 | } 97 | print_result(result, max_index, "tag2 and tag3"); 98 | 99 | // tag1 or tag2 100 | // 101 | for (i = 0; i < array_max_used; i++) { 102 | result[i] = tag1[i] | tag2[i]; 103 | } 104 | print_result(result, max_index, "tag1 or tag2"); 105 | 106 | // tag2 or tag3 107 | // 108 | for (i = 0; i < array_max_used; i++) { 109 | result[i] = tag2[i] | tag3[i]; 110 | } 111 | print_result(result, max_index, "tag2 or tag3"); 112 | 113 | // tag1 and not tag2 114 | // 115 | for (i = 0; i < array_max_used; i++) { 116 | result[i] = tag1[i] & ~tag2[i]; 117 | } 118 | print_result(result, max_index, "tag1 and not tag2"); 119 | 120 | // calculate result and scan multiple times (demonstrate load) 121 | // 122 | for (i = 0; i < 100000; i++) { 123 | for (j = 0; j < array_max_used; j++) { 124 | result[j] = (tag1[j] & ~tag2[j]) | tag3[j]; 125 | } 126 | scan_result(result, max_index, NULL); 127 | } 128 | 129 | return 0; 130 | } 131 | -------------------------------------------------------------------------------- /tsdb_api.h: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright (C) 2011 IIT/CNR (http://www.iit.cnr.it/en) 4 | * Luca Deri 5 | * 6 | * This program is free software; you can redistribute it and/or modify 7 | * it under the terms of the GNU General Public License as published by 8 | * the Free Software Foundation; either version 2 of the License, or 9 | * (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License along 17 | * with this program; if not, write to the Free Software Foundation, Inc., 18 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 19 | */ 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | #include "tsdb_trace.h" 29 | #include "quicklz.h" 30 | 31 | #define CHUNK_GROWTH 10000 32 | #define CHUNK_LEN_PADDING 400 33 | #define MAX_NUM_FRAGMENTS 16384 34 | 35 | typedef struct { 36 | u_int8_t *data; 37 | u_int32_t data_len; 38 | u_int32_t epoch; 39 | u_int8_t growable; 40 | u_int8_t fragment_changed[MAX_NUM_FRAGMENTS]; 41 | u_int32_t base_index; 42 | } tsdb_chunk; 43 | 44 | typedef struct { 45 | u_int32_t *array; 46 | u_int32_t array_len; 47 | } tsdb_tag; 48 | 49 | typedef u_int32_t tsdb_value; 50 | 51 | typedef struct { 52 | u_int8_t alive; 53 | u_int8_t read_only; 54 | u_int16_t values_per_entry; 55 | u_int16_t values_len; 56 | u_int32_t unknown_value; 57 | u_int32_t lowest_free_index; 58 | u_int32_t slot_duration; 59 | qlz_state_compress state_compress; 60 | qlz_state_decompress state_decompress; 61 | tsdb_chunk chunk; 62 | DB *db; 63 | } tsdb_handler; 64 | 65 | extern int tsdb_open(char *tsdb_path, tsdb_handler *handler, 66 | u_int16_t *values_per_entry, 67 | u_int32_t slot_duration, 68 | u_int8_t read_only); 69 | 70 | extern void tsdb_close(tsdb_handler *handler); 71 | 72 | extern void normalize_epoch(tsdb_handler *handler, u_int32_t *epoch); 73 | 74 | extern int tsdb_goto_epoch(tsdb_handler *handler, 75 | u_int32_t epoch_value, 76 | u_int8_t fail_if_missing, 77 | u_int8_t growable); 78 | 79 | extern int tsdb_set(tsdb_handler *handler, char *key, tsdb_value *value); 80 | 81 | extern int tsdb_set_with_index(tsdb_handler *handler, char *key, 82 | tsdb_value *value, u_int32_t *index); 83 | 84 | extern int tsdb_get_by_key(tsdb_handler *handler, 85 | char *key, 86 | tsdb_value **value); 87 | 88 | extern int tsdb_get_key_index(tsdb_handler *handler, 89 | char *key, 90 | u_int32_t *index); 91 | 92 | extern int tsdb_get_by_index(tsdb_handler *handler, 93 | u_int32_t *index, 94 | tsdb_value **value); 95 | 96 | extern void tsdb_flush(tsdb_handler *handler); 97 | 98 | extern int tsdb_tag_key(tsdb_handler *handler, char* key, char* tag_name); 99 | 100 | extern int tsdb_get_tag_indexes(tsdb_handler *handler, 101 | char *tag_name, 102 | u_int32_t *indexes, 103 | u_int32_t indexes_len, 104 | u_int32_t *count); 105 | 106 | #define TSDB_AND 1 107 | #define TSDB_OR 2 108 | 109 | extern int tsdb_get_consolidated_tag_indexes(tsdb_handler *handler, 110 | char **tag_names, 111 | u_int16_t tag_names_len, 112 | int consolidator, 113 | u_int32_t *indexes, 114 | u_int32_t indexes_len, 115 | u_int32_t *count); 116 | -------------------------------------------------------------------------------- /quicklz.h: -------------------------------------------------------------------------------- 1 | #ifndef QLZ_HEADER 2 | #define QLZ_HEADER 3 | 4 | // Fast data compression library 5 | // Copyright (C) 2006-2011 Lasse Mikkel Reinhold 6 | // lar@quicklz.com 7 | // 8 | // QuickLZ can be used for free under the GPL 1, 2 or 3 license (where anything 9 | // released into public must be open source) or under a commercial license if such 10 | // has been acquired (see http://www.quicklz.com/order.html). The commercial license 11 | // does not cover derived or ported versions created by third parties under GPL. 12 | 13 | // You can edit following user settings. Data must be decompressed with the same 14 | // setting of QLZ_COMPRESSION_LEVEL and QLZ_STREAMING_BUFFER as it was compressed 15 | // (see manual). If QLZ_STREAMING_BUFFER > 0, scratch buffers must be initially 16 | // zeroed out (see manual). First #ifndef makes it possible to define settings from 17 | // the outside like the compiler command line. 18 | 19 | // 1.5.0 final 20 | 21 | #ifndef QLZ_COMPRESSION_LEVEL 22 | 23 | // 1 gives fastest compression speed. 3 gives fastest decompression speed and best 24 | // compression ratio. 25 | #define QLZ_COMPRESSION_LEVEL 1 26 | //#define QLZ_COMPRESSION_LEVEL 2 27 | //#define QLZ_COMPRESSION_LEVEL 3 28 | 29 | // If > 0, zero out both states prior to first call to qlz_compress() or qlz_decompress() 30 | // and decompress packets in the same order as they were compressed 31 | #define QLZ_STREAMING_BUFFER 0 32 | //#define QLZ_STREAMING_BUFFER 100000 33 | //#define QLZ_STREAMING_BUFFER 1000000 34 | 35 | // Guarantees that decompression of corrupted data cannot crash. Decreases decompression 36 | // speed 10-20%. Compression speed not affected. 37 | //#define QLZ_MEMORY_SAFE 38 | #endif 39 | 40 | #define QLZ_VERSION_MAJOR 1 41 | #define QLZ_VERSION_MINOR 5 42 | #define QLZ_VERSION_REVISION 0 43 | 44 | // Using size_t, memset() and memcpy() 45 | #include 46 | 47 | // Verify compression level 48 | #if QLZ_COMPRESSION_LEVEL != 1 && QLZ_COMPRESSION_LEVEL != 2 && QLZ_COMPRESSION_LEVEL != 3 49 | #error QLZ_COMPRESSION_LEVEL must be 1, 2 or 3 50 | #endif 51 | 52 | typedef unsigned int ui32; 53 | typedef unsigned short int ui16; 54 | 55 | // Decrease QLZ_POINTERS for level 3 to increase compression speed. Do not touch any other values! 56 | #if QLZ_COMPRESSION_LEVEL == 1 57 | #define QLZ_POINTERS 1 58 | #define QLZ_HASH_VALUES 4096 59 | #elif QLZ_COMPRESSION_LEVEL == 2 60 | #define QLZ_POINTERS 4 61 | #define QLZ_HASH_VALUES 2048 62 | #elif QLZ_COMPRESSION_LEVEL == 3 63 | #define QLZ_POINTERS 16 64 | #define QLZ_HASH_VALUES 4096 65 | #endif 66 | 67 | // Detect if pointer size is 64-bit. It's not fatal if some 64-bit target is not detected because this is only for adding an optional 64-bit optimization. 68 | #if defined _LP64 || defined __LP64__ || defined __64BIT__ || _ADDR64 || defined _WIN64 || defined __arch64__ || __WORDSIZE == 64 || (defined __sparc && defined __sparcv9) || defined __x86_64 || defined __amd64 || defined __x86_64__ || defined _M_X64 || defined _M_IA64 || defined __ia64 || defined __IA64__ 69 | #define QLZ_PTR_64 70 | #endif 71 | 72 | // hash entry 73 | typedef struct 74 | { 75 | #if QLZ_COMPRESSION_LEVEL == 1 76 | ui32 cache; 77 | #if defined QLZ_PTR_64 && QLZ_STREAMING_BUFFER == 0 78 | unsigned int offset; 79 | #else 80 | const unsigned char *offset; 81 | #endif 82 | #else 83 | const unsigned char *offset[QLZ_POINTERS]; 84 | #endif 85 | 86 | } qlz_hash_compress; 87 | 88 | typedef struct 89 | { 90 | #if QLZ_COMPRESSION_LEVEL == 1 91 | const unsigned char *offset; 92 | #else 93 | const unsigned char *offset[QLZ_POINTERS]; 94 | #endif 95 | } qlz_hash_decompress; 96 | 97 | 98 | // states 99 | typedef struct 100 | { 101 | #if QLZ_STREAMING_BUFFER > 0 102 | unsigned char stream_buffer[QLZ_STREAMING_BUFFER]; 103 | #endif 104 | size_t stream_counter; 105 | qlz_hash_compress hash[QLZ_HASH_VALUES]; 106 | unsigned char hash_counter[QLZ_HASH_VALUES]; 107 | } qlz_state_compress; 108 | 109 | 110 | #if QLZ_COMPRESSION_LEVEL == 1 || QLZ_COMPRESSION_LEVEL == 2 111 | typedef struct 112 | { 113 | #if QLZ_STREAMING_BUFFER > 0 114 | unsigned char stream_buffer[QLZ_STREAMING_BUFFER]; 115 | #endif 116 | qlz_hash_decompress hash[QLZ_HASH_VALUES]; 117 | unsigned char hash_counter[QLZ_HASH_VALUES]; 118 | size_t stream_counter; 119 | } qlz_state_decompress; 120 | #elif QLZ_COMPRESSION_LEVEL == 3 121 | typedef struct 122 | { 123 | #if QLZ_STREAMING_BUFFER > 0 124 | unsigned char stream_buffer[QLZ_STREAMING_BUFFER]; 125 | #endif 126 | #if QLZ_COMPRESSION_LEVEL <= 2 127 | qlz_hash_decompress hash[QLZ_HASH_VALUES]; 128 | #endif 129 | size_t stream_counter; 130 | } qlz_state_decompress; 131 | #endif 132 | 133 | 134 | #if defined (__cplusplus) 135 | extern "C" { 136 | #endif 137 | 138 | // Public functions of QuickLZ 139 | size_t qlz_size_decompressed(const char *source); 140 | size_t qlz_size_compressed(const char *source); 141 | size_t qlz_compress(const void *source, char *destination, size_t size, qlz_state_compress *state); 142 | size_t qlz_decompress(const char *source, void *destination, qlz_state_decompress *state); 143 | int qlz_get_setting(int setting); 144 | 145 | #if defined (__cplusplus) 146 | } 147 | #endif 148 | 149 | #endif 150 | 151 | -------------------------------------------------------------------------------- /test_tags.c: -------------------------------------------------------------------------------- 1 | #include "test_core.h" 2 | 3 | static char *get_file_arg(int argc, char *argv[]) { 4 | if (argc < 2) { 5 | fprintf(stderr, "usage: test-advanced TEST-DB\n"); 6 | exit(1); 7 | } 8 | char *file = argv[1]; 9 | if (file_exists(file)) { 10 | fprintf(stderr, "%s exists\n", file); 11 | exit(1); 12 | } 13 | return file; 14 | } 15 | 16 | int main(int argc, char *argv[]) { 17 | 18 | char *file = get_file_arg(argc, argv); 19 | set_trace_level(0); 20 | 21 | tsdb_handler db; 22 | int ret; 23 | tsdb_value write_val; 24 | tsdb_value *read_val; 25 | 26 | // Open (create) a new db. 27 | 28 | u_int16_t vals_per_entry = 1; 29 | ret = tsdb_open(file, &db, &vals_per_entry, 60, 0); 30 | assert_int_equal(0, ret); 31 | 32 | //=================================================================== 33 | // Basic use 34 | //=================================================================== 35 | 36 | // We can't add a tag for a key that doesn't exist. 37 | // 38 | ret = tsdb_tag_key(&db, "key-1", "test"); 39 | assert_int_equal(-1, ret); 40 | 41 | // Let's add a key. 42 | // 43 | ret = tsdb_goto_epoch(&db, 60, 0, 0); 44 | assert_int_equal(0, ret); 45 | write_val = 111; 46 | ret = tsdb_set(&db, "key-1", &write_val); 47 | assert_int_equal(0, ret); 48 | 49 | // Now we can tag it. 50 | // 51 | ret = tsdb_tag_key(&db, "key-1", "blue"); 52 | assert_int_equal(0, ret); 53 | 54 | // Let's add key-2, but for a different tag. 55 | // 56 | write_val = 222; 57 | ret = tsdb_set(&db, "key-2", &write_val); 58 | assert_int_equal(0, ret); 59 | ret = tsdb_tag_key(&db, "key-2", "red"); 60 | assert_int_equal(0, ret); 61 | 62 | // And tag our second key using the first tag. 63 | // 64 | ret = tsdb_tag_key(&db, "key-2", "blue"); 65 | assert_int_equal(0, ret); 66 | 67 | //=================================================================== 68 | // Single tag queries 69 | //=================================================================== 70 | 71 | // The primary purpose of a tag is to allow efficient lookup of 72 | // associated key indexes. 73 | // 74 | // Indexes for a tag can be collected using tsdb_get_tag_indexes. 75 | // 76 | u_int32_t matches[100]; 77 | u_int32_t match_count; 78 | ret = tsdb_get_tag_indexes(&db, "blue", matches, 100, &match_count); 79 | assert_int_equal(0, ret); 80 | assert_int_equal(2, match_count); 81 | 82 | // We have two matches. Let's use the indexes to lookup values. 83 | // 84 | ret = tsdb_get_by_index(&db, &matches[0], &read_val); 85 | assert_int_equal(0, ret); 86 | assert_int_equal(111, *read_val); 87 | ret = tsdb_get_by_index(&db, &matches[1], &read_val); 88 | assert_int_equal(0, ret); 89 | assert_int_equal(222, *read_val); 90 | 91 | // Let's check for indexes tagged wth "red". 92 | // 93 | ret = tsdb_get_tag_indexes(&db, "red", matches, 100, &match_count); 94 | assert_int_equal(0, ret); 95 | assert_int_equal(1, match_count); 96 | 97 | // And their value. 98 | // 99 | ret = tsdb_get_by_index(&db, &matches[0], &read_val); 100 | assert_int_equal(0, ret); 101 | assert_int_equal(222, *read_val); 102 | 103 | //=================================================================== 104 | // Complex tag queries 105 | //=================================================================== 106 | 107 | char *red_and_blue[2] = { "red", "blue" }; 108 | 109 | ret = tsdb_get_consolidated_tag_indexes(&db, red_and_blue, 2, TSDB_AND, 110 | matches, 100, &match_count); 111 | assert_int_equal(0, ret); 112 | assert_int_equal(1, match_count); 113 | 114 | // And the value for the matched index. 115 | // 116 | ret = tsdb_get_by_index(&db, &matches[0], &read_val); 117 | assert_int_equal(0, ret); 118 | assert_int_equal(222, *read_val); 119 | 120 | // Let's add a few more indexes with tags. 121 | // 122 | write_val = 333; 123 | ret = tsdb_set(&db, "key-3", &write_val); 124 | assert_int_equal(0, ret); 125 | tsdb_tag_key(&db, "key-3", "red"); 126 | tsdb_tag_key(&db, "key-3", "blue"); 127 | tsdb_tag_key(&db, "key-3", "green"); 128 | 129 | write_val = 444; 130 | ret = tsdb_set(&db, "key-4", &write_val); 131 | assert_int_equal(0, ret); 132 | tsdb_tag_key(&db, "key-4", "red"); 133 | tsdb_tag_key(&db, "key-4", "green"); 134 | 135 | // Let's get indexes that are tagged with both blue and green. 136 | // 137 | char *blue_and_green[2] = { "blue", "green" }; 138 | ret = tsdb_get_consolidated_tag_indexes(&db, blue_and_green, 2, TSDB_AND, 139 | matches, 100, &match_count); 140 | assert_int_equal(0, ret); 141 | assert_int_equal(1, match_count); 142 | ret = tsdb_get_by_index(&db, &matches[0], &read_val); 143 | assert_int_equal(0, ret); 144 | assert_int_equal(333, *read_val); 145 | 146 | // And now the indexes tagged with blue or green. 147 | // 148 | ret = tsdb_get_consolidated_tag_indexes(&db, blue_and_green, 2, TSDB_OR, 149 | matches, 100, &match_count); 150 | assert_int_equal(0, ret); 151 | assert_int_equal(4, match_count); 152 | tsdb_get_by_index(&db, &matches[0], &read_val); 153 | assert_int_equal(111, *read_val); 154 | tsdb_get_by_index(&db, &matches[1], &read_val); 155 | assert_int_equal(222, *read_val); 156 | tsdb_get_by_index(&db, &matches[2], &read_val); 157 | assert_int_equal(333, *read_val); 158 | tsdb_get_by_index(&db, &matches[3], &read_val); 159 | assert_int_equal(444, *read_val); 160 | 161 | tsdb_close(&db); 162 | 163 | return 0; 164 | } 165 | -------------------------------------------------------------------------------- /tsdb_get.c: -------------------------------------------------------------------------------- 1 | #include "tsdb_api.h" 2 | 3 | typedef struct { 4 | char *file; 5 | char *key; 6 | u_int32_t start; 7 | u_int32_t end; 8 | u_int16_t interval; 9 | int verbose; 10 | } get_args; 11 | 12 | static void help(int code) { 13 | printf("tsdb-get [-v] file key [-s start] [-e end] [-i interval]\n"); 14 | exit(code); 15 | } 16 | 17 | static void check_strtol_error(int no_digits, long val, int err, 18 | const char *argname) { 19 | if (no_digits 20 | || (err == ERANGE && (val == LONG_MAX || val == LONG_MIN)) 21 | || (err != 0 && val == 0)) { 22 | printf("tsdb-get: invalid value for %s\n", argname); 23 | exit(1); 24 | } 25 | } 26 | 27 | static int unit_seconds_val(const char *units, const char *argname) { 28 | if (*units == '\0') { 29 | return 0; 30 | } else if (strcmp(units, "s") == 0) { 31 | return 1; 32 | } else if (strcmp(units, "m") == 0) { 33 | return 60; 34 | } else if (strcmp(units, "h") == 0) { 35 | return 3600; 36 | } else if (strcmp(units, "d") == 0) { 37 | return 86400; 38 | } else { 39 | printf("tsdb-get: unknown unit for %s\n", argname); 40 | exit(1); 41 | } 42 | } 43 | 44 | static u_int32_t epoch_val(const char *str, u_int32_t now, 45 | const char *argname) { 46 | char *units; 47 | long numval; 48 | int unit_seconds; 49 | 50 | errno = 0; 51 | numval = strtol(str, &units, 10); 52 | check_strtol_error(str == units, numval, errno, argname); 53 | unit_seconds = unit_seconds_val(units, argname); 54 | if (unit_seconds == 0) { 55 | return numval; 56 | } else { 57 | return now + numval * unit_seconds; 58 | } 59 | } 60 | 61 | static u_int16_t interval_val(const char *str, const char *argname) { 62 | char *units; 63 | long numval; 64 | int unit_seconds; 65 | 66 | errno = 0; 67 | numval = strtol(str, &units, 10); 68 | check_strtol_error(str == units, numval, errno, argname); 69 | unit_seconds = unit_seconds_val(units, argname); 70 | if (unit_seconds == 0) { 71 | return numval; 72 | } else { 73 | return numval * unit_seconds; 74 | } 75 | } 76 | 77 | static void process_args(int argc, char *argv[], get_args *args) { 78 | int c; 79 | u_int32_t now = time(NULL); 80 | 81 | args->start = now; 82 | args->end = now; 83 | args->verbose = 0; 84 | args->interval = 0; 85 | 86 | while ((c = getopt(argc, argv, "hvs:e:i:")) != -1) { 87 | switch (c) { 88 | case 's': 89 | args->start = epoch_val(optarg, now, "start"); 90 | break; 91 | case 'e': 92 | args->end = epoch_val(optarg, now, "end"); 93 | break; 94 | case 'i': 95 | args->interval = interval_val(optarg, "interval"); 96 | break; 97 | case 'v': 98 | args->verbose = 1; 99 | break; 100 | case 'h': 101 | help(0); 102 | break; 103 | default: 104 | help(1); 105 | } 106 | } 107 | 108 | int remaining = argc - optind; 109 | if (remaining != 2) { 110 | help(1); 111 | } 112 | args->file = argv[optind]; 113 | args->key = argv[optind + 1]; 114 | } 115 | 116 | static void check_file_exists(const char *path) { 117 | FILE *file; 118 | if ((file = fopen(path, "r"))) { 119 | fclose(file); 120 | } else { 121 | printf("tsdb-get: %s doesn't exist\n", path); 122 | exit(1); 123 | } 124 | } 125 | 126 | static void init_trace(int verbose) { 127 | set_trace_level(verbose ? 99 : 0); 128 | } 129 | 130 | static void open_db(char *file, tsdb_handler *db) { 131 | u_int16_t unused16 = 0; 132 | u_int32_t unused32 = 0; 133 | if (tsdb_open(file, db, &unused16, unused32, 0)) { 134 | printf("tsdb-get: error opening db %s\n", file); 135 | exit(1); 136 | } 137 | } 138 | 139 | static int goto_epoch(tsdb_handler *db, u_int32_t epoch) { 140 | if (tsdb_goto_epoch(db, epoch, 1, 0)) { 141 | return 0; 142 | } else { 143 | return 1; 144 | } 145 | } 146 | 147 | static int get_vals(tsdb_handler *db, char *key, tsdb_value **vals) { 148 | return tsdb_get_by_key(db, key, vals); 149 | } 150 | 151 | static void print_missing(u_int32_t epoch, int value_count) { 152 | int i; 153 | printf("%u", epoch); 154 | for (i = 0; i < value_count; i++) { 155 | printf(" -"); 156 | } 157 | printf("\n"); 158 | } 159 | 160 | static void print_vals(u_int32_t epoch, int count, tsdb_value *vals) { 161 | int i; 162 | printf("%u", epoch); 163 | for (i = 0; i < count; i++) { 164 | printf(" %u", vals[i]); 165 | } 166 | printf("\n"); 167 | } 168 | 169 | static void print_key_vals(tsdb_handler *db, char *key, u_int32_t epoch) { 170 | tsdb_value *vals; 171 | if (get_vals(db, key, &vals) >= 0) { 172 | print_vals(epoch, db->values_per_entry, vals); 173 | } else { 174 | print_missing(epoch, db->values_per_entry); 175 | } 176 | } 177 | 178 | static void print_epoch_vals(tsdb_handler *db, u_int32_t epoch, char *key) { 179 | if (goto_epoch(db, epoch)) { 180 | print_key_vals(db, key, epoch); 181 | } else { 182 | print_missing(epoch, db->values_per_entry); 183 | } 184 | } 185 | 186 | static void print_tsdb_values(char *file, char *key, u_int32_t start, 187 | u_int32_t end, u_int16_t interval) { 188 | tsdb_handler db; 189 | u_int32_t cur_epoch = start; 190 | 191 | open_db(file, &db); 192 | 193 | normalize_epoch(&db, &cur_epoch); 194 | normalize_epoch(&db, &end); 195 | if (interval <= 0) { 196 | interval = db.slot_duration; 197 | } 198 | 199 | while (cur_epoch <= end) { 200 | print_epoch_vals(&db, cur_epoch, key); 201 | cur_epoch += interval; 202 | } 203 | } 204 | 205 | int main(int argc, char *argv[]) { 206 | get_args args; 207 | 208 | process_args(argc, argv, &args); 209 | init_trace(args.verbose); 210 | check_file_exists(args.file); 211 | print_tsdb_values(args.file, args.key, args.start, args.end, 212 | args.interval); 213 | return 0; 214 | } 215 | -------------------------------------------------------------------------------- /seatest.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2010 Keith Nicholas 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to deal 6 | in the Software without restriction, including without limitation the rights 7 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | THE SOFTWARE. 21 | */ 22 | 23 | #ifndef SEATEST_H 24 | #define SEATEST_H 25 | #include 26 | 27 | /* 28 | Defines 29 | */ 30 | 31 | #define SEATEST_VERSION "0.5" 32 | #define SEATEST_PROJECT_HOME "http://code.google.com/p/seatest/" 33 | #define SEATEST_PRINT_BUFFER_SIZE 100000 34 | 35 | /* 36 | Typedefs 37 | */ 38 | 39 | typedef void (*seatest_void_void)(void); 40 | typedef void (*seatest_void_string)(char*); 41 | 42 | /* 43 | Declarations 44 | */ 45 | void (*seatest_simple_test_result)(int passed, char* reason, const char* function, unsigned int line); 46 | void seatest_test_fixture_start(char* filepath); 47 | void seatest_test_fixture_end( void ); 48 | void seatest_simple_test_result_log(int passed, char* reason, const char* function, unsigned int line); 49 | void seatest_assert_true(int test, const char* function, unsigned int line); 50 | void seatest_assert_false(int test, const char* function, unsigned int line); 51 | void seatest_assert_int_equal(int expected, int actual, const char* function, unsigned int line); 52 | void seatest_assert_ulong_equal(unsigned long expected, unsigned long actual, const char* function, unsigned int line); 53 | void seatest_assert_float_equal(float expected, float actual, float delta, const char* function, unsigned int line); 54 | void seatest_assert_double_equal(double expected, double actual, double delta, const char* function, unsigned int line); 55 | void seatest_assert_string_equal(char* expected, char* actual, const char* function, unsigned int line); 56 | void seatest_assert_string_ends_with(char* expected, char* actual, const char* function, unsigned int line); 57 | void seatest_assert_string_starts_with(char* expected, char* actual, const char* function, unsigned int line); 58 | void seatest_assert_string_contains(char* expected, char* actual, const char* function, unsigned int line); 59 | void seatest_assert_string_doesnt_contain(char* expected, char* actual, const char* function, unsigned int line); 60 | int seatest_should_run( char* fixture, char* test); 61 | void seatest_before_run( char* fixture, char* test); 62 | void seatest_run_test(char* fixture, char* test); 63 | void seatest_setup( void ); 64 | void seatest_teardown( void ); 65 | void seatest_suite_teardown( void ); 66 | void seatest_suite_setup( void ); 67 | 68 | /* 69 | Assert Macros 70 | */ 71 | 72 | #define assert_true(test) do { seatest_assert_true(test, __FUNCTION__, __LINE__); } while (0) 73 | #define assert_false(test) do { seatest_assert_false(test, __FUNCTION__, __LINE__); } while (0) 74 | #define assert_int_equal(expected, actual) do { seatest_assert_int_equal(expected, actual, __FUNCTION__, __LINE__); } while (0) 75 | #define assert_ulong_equal(expected, actual) do { seatest_assert_ulong_equal(expected, actual, __FUNCTION__, __LINE__); } while (0) 76 | #define assert_string_equal(expected, actual) do { seatest_assert_string_equal(expected, actual, __FUNCTION__, __LINE__); } while (0) 77 | #define assert_n_array_equal(expected, actual, n) do { int seatest_count; for(seatest_count=0; seatest_count 0); 101 | 102 | // Let's move to another epoch and try some more tests. 103 | // 104 | ret = tsdb_goto_epoch(&db, 120, 0, 0); 105 | assert_int_equal(0, ret); 106 | assert_true(db.chunk.data == NULL); 107 | assert_int_equal(0, db.chunk.data_len); 108 | 109 | // Reading a key that doesn't exit. 110 | // 111 | ret = tsdb_get_by_key(&db, "key-1", &read_val); 112 | assert_int_equal(-1, ret); 113 | 114 | // Writing the key. 115 | // 116 | wide_val[0] = 22222; 117 | wide_val[1] = 33333; 118 | ret = tsdb_set(&db, "key-1", wide_val); 119 | assert_int_equal(0, ret); 120 | 121 | // Let's move back to the previous epoch, and indiate that we want to 122 | // fail if the epoch doesn't exist. Because we've already created it, 123 | // however, we should succeed. 124 | // 125 | ret = tsdb_goto_epoch(&db, 60, 1, 0); 126 | assert_int_equal(0, ret); 127 | 128 | // And read some values. 129 | // 130 | ret = tsdb_get_by_key(&db, "key-1", &read_val); 131 | assert_int_equal(0, ret); 132 | assert_int_equal(111, *read_val); 133 | ret = tsdb_get_by_key(&db, "key-2", &read_val); 134 | assert_int_equal(0, ret); 135 | assert_int_equal(111, read_val[0]); 136 | assert_int_equal(222, read_val[1]); 137 | 138 | // And back to the next epoch. 139 | // 140 | ret = tsdb_goto_epoch(&db, 120, 1, 0); 141 | assert_int_equal(0, ret); 142 | ret = tsdb_get_by_key(&db, "key-1", &read_val); 143 | assert_int_equal(0, ret); 144 | assert_int_equal(22222, read_val[0]); 145 | assert_int_equal(33333, read_val[1]); 146 | 147 | //=================================================================== 148 | // Epoch growth 149 | //=================================================================== 150 | 151 | // tsdb initially allocates a chunk (array) to store keys. When 152 | // the chunk is full and additional space is needed because to 153 | // accommodate new keys, tsdb will grow the chunk size. 154 | // 155 | // Let's illustrate by filling up a chunk, measuring the chunk 156 | // size, and then adding new keys to verify that the chunk size 157 | // grows. 158 | // 159 | // We'll move to the next epoch to start. We'll indicate that we 160 | // don't want to fail if the epoch is missing and that we want the 161 | // epoch to grow as needed to accommodate keys. 162 | // 163 | ret = tsdb_goto_epoch(&db, 180, 0, 1); 164 | assert_int_equal(0, ret); 165 | 166 | // As we'd expect, the epoch data is empty to start. 167 | // 168 | assert_int_equal(0, db.chunk.data_len); 169 | 170 | // Let's add one key so we can measure how much data is allocated for 171 | // a single chunk. 172 | // 173 | wide_val[0] = 444444; 174 | wide_val[1] = 555555; 175 | ret = tsdb_set(&db, "key-1", wide_val); 176 | assert_int_equal(0, ret); 177 | 178 | // Here's our initial chunk size and the corresponding number of keys 179 | // we can expect to fit into a single chunk. 180 | // 181 | uint chunk_size = db.chunk.data_len; 182 | uint keys_per_chunk = chunk_size / db.values_len; 183 | 184 | // Let's fill up the remaining chunk slots. 185 | // 186 | uint i; 187 | char key[32]; 188 | for (i = 1; i < keys_per_chunk; i++) { 189 | sprintf(key, "key-%i", i + 1); 190 | ret = tsdb_set(&db, key, wide_val); 191 | assert_int_equal(0, ret); 192 | } 193 | 194 | // At this point our chunk data should be full, but we haven't yet 195 | // allocated more data. 196 | // 197 | assert_int_equal(chunk_size, db.chunk.data_len); 198 | 199 | // Let's add one more key -- this will cause more data to be allocated 200 | // for the chunk. 201 | // 202 | sprintf(key, "key-%i", i + 1); 203 | ret = tsdb_set(&db, key, wide_val); 204 | assert_int_equal(0, ret); 205 | assert_int_equal(chunk_size * 2, db.chunk.data_len); 206 | 207 | // And read the values back. 208 | // 209 | for (i = 0; i < keys_per_chunk; i++) { 210 | sprintf(key, "key-%i", i + 1); 211 | ret = tsdb_get_by_key(&db, key, &read_val); 212 | assert_int_equal(0, ret); 213 | assert_int_equal(444444, read_val[0]); 214 | assert_int_equal(555555, read_val[1]); 215 | } 216 | 217 | //=================================================================== 218 | // Lookup by index 219 | //=================================================================== 220 | 221 | // tsdb supports lookup by key, however, it also provides an 222 | // optimization that lets you retrieve a key's index and use it 223 | // to read values more efficiently from an epoch. This works because, 224 | // once assigned, a key is always associated with the same index 225 | // for the life of the tsdb database. 226 | // 227 | // This is useful when scanning a series of epochs for a one or more 228 | // keys -- rather than lookup the associated key index for each 229 | // epoch, the caller can save and reuse the applicable indexes. 230 | // 231 | // Let's retrieve the index for "key-1" (one of the keys we've written 232 | // to this database) and use it to read a value from the current 233 | // epoch. 234 | // 235 | u_int32_t index; 236 | ret = tsdb_get_key_index(&db, "key-1", &index); 237 | assert_int_equal(0, ret); 238 | ret = tsdb_get_by_index(&db, &index, &read_val); 239 | assert_int_equal(0, ret); 240 | assert_int_equal(444444, read_val[0]); 241 | assert_int_equal(555555, read_val[1]); 242 | 243 | // There an alternative form of tsdb_set that updates an index 244 | // argument with the key's index. This can be used to save a 245 | // call to tsdb_get_key_index if a key's index is needed 246 | // immediately after a set. 247 | // 248 | wide_val[0] = 66666666; 249 | wide_val[1] = 77777777; 250 | ret = tsdb_set_with_index(&db, "key-1", wide_val, &index); 251 | assert_int_equal(0, ret); 252 | assert_true(index >= 0); 253 | 254 | // And as above, we can use the index to read the values back 255 | // directly. 256 | // 257 | ret = tsdb_get_by_index(&db, &index, &read_val); 258 | assert_int_equal(0, ret); 259 | assert_int_equal(66666666, read_val[0]); 260 | assert_int_equal(77777777, read_val[1]); 261 | 262 | // We can read up to the maximum index (chunk slots - 1) but no 263 | // further. We've already expanded the chunk size, so can currently 264 | // read up to two chunk's worth of values using indexes. 265 | // 266 | index = (keys_per_chunk * 2) - 1; 267 | ret = tsdb_get_by_index(&db, &index, &read_val); 268 | assert_int_equal(0, ret); 269 | assert_int_equal(0, read_val[0]); 270 | assert_int_equal(0, read_val[1]); 271 | 272 | // Reading passed the last item fails. 273 | // 274 | index++; 275 | ret = tsdb_get_by_index(&db, &index, &read_val); 276 | assert_int_equal(-1, ret); 277 | 278 | tsdb_close(&db); 279 | 280 | return 0; 281 | } 282 | -------------------------------------------------------------------------------- /seatest.c: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2010 Keith Nicholas 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to deal 6 | in the Software without restriction, including without limitation the rights 7 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | THE SOFTWARE. 21 | */ 22 | 23 | #include "seatest.h" 24 | #include 25 | #include 26 | #ifdef WIN32 27 | #include "windows.h" 28 | int seatest_is_string_equal_i(const char* s1, const char* s2) 29 | { 30 | #pragma warning(disable: 4996) 31 | return stricmp(s1, s2) == 0; 32 | } 33 | 34 | #else 35 | #include 36 | unsigned int GetTickCount() { return 0;} 37 | void _getch( void ) { } 38 | int seatest_is_string_equal_i(const char* s1, const char* s2) 39 | { 40 | return strcasecmp(s1, s2) == 0; 41 | } 42 | #endif 43 | 44 | #ifdef SEATEST_INTERNAL_TESTS 45 | static int sea_test_last_passed = 0; 46 | #endif 47 | 48 | 49 | typedef enum 50 | { 51 | SEATEST_DISPLAY_TESTS, 52 | SEATEST_RUN_TESTS, 53 | SEATEST_DO_NOTHING, 54 | SEATEST_DO_ABORT 55 | } seatest_action_t; 56 | 57 | typedef struct 58 | { 59 | int argc; 60 | char** argv; 61 | seatest_action_t action; 62 | } seatest_testrunner_t; 63 | static int seatest_screen_width = 70; 64 | static int sea_tests_run = 0; 65 | static int sea_tests_passed = 0; 66 | static int sea_tests_failed = 0; 67 | static int seatest_display_only = 0; 68 | static int seatest_verbose = 0; 69 | static int seatest_machine_readable = 0; 70 | static char* seatest_current_fixture; 71 | static char* seatest_current_fixture_path; 72 | static char seatest_magic_marker[20] = ""; 73 | 74 | static seatest_void_void seatest_suite_setup_func = 0; 75 | static seatest_void_void seatest_suite_teardown_func = 0; 76 | static seatest_void_void seatest_fixture_setup = 0; 77 | static seatest_void_void seatest_fixture_teardown = 0; 78 | 79 | void (*seatest_simple_test_result)(int passed, char* reason, const char* function, unsigned int line) = seatest_simple_test_result_log; 80 | 81 | void suite_setup(seatest_void_void setup) 82 | { 83 | seatest_suite_setup_func = setup; 84 | } 85 | void suite_teardown(seatest_void_void teardown) 86 | { 87 | seatest_suite_teardown_func = teardown; 88 | } 89 | 90 | int seatest_is_display_only() 91 | { 92 | return seatest_display_only; 93 | } 94 | 95 | void seatest_suite_setup( void ) 96 | { 97 | if(seatest_suite_setup_func != 0) seatest_suite_setup_func(); 98 | } 99 | 100 | void seatest_suite_teardown( void ) 101 | { 102 | if(seatest_suite_teardown_func != 0) seatest_suite_teardown_func(); 103 | } 104 | 105 | void fixture_setup(void (*setup)( void )) 106 | { 107 | seatest_fixture_setup = setup; 108 | } 109 | void fixture_teardown(void (*teardown)( void )) 110 | { 111 | seatest_fixture_teardown = teardown; 112 | } 113 | 114 | void seatest_setup( void ) 115 | { 116 | if(seatest_fixture_setup != 0) seatest_fixture_setup(); 117 | } 118 | 119 | void seatest_teardown( void ) 120 | { 121 | if(seatest_fixture_teardown != 0) seatest_fixture_teardown(); 122 | } 123 | 124 | char* test_file_name(char* path) 125 | { 126 | char* file = path + strlen(path); 127 | while(file != path && *file!= '\\' ) file--; 128 | if(*file == '\\') file++; 129 | return file; 130 | } 131 | 132 | static int seatest_fixture_tests_run; 133 | static int seatest_fixture_tests_failed; 134 | 135 | void seatest_simple_test_result_log(int passed, char* reason, const char* function, unsigned int line) 136 | { 137 | if (!passed) 138 | { 139 | 140 | if(seatest_machine_readable) 141 | { 142 | printf("%s%s,%s,%u,%s\r\n", seatest_magic_marker, seatest_current_fixture_path, function, line, reason ); 143 | } 144 | else 145 | { 146 | printf("%-30s Line %-5d %s\r\n", function, line, reason ); 147 | } 148 | #ifdef SEATEST_EXIT_ON_FAIL 149 | exit(1); 150 | #endif 151 | 152 | 153 | sea_tests_failed++; 154 | } 155 | else 156 | { 157 | if(seatest_verbose) 158 | { 159 | if(seatest_machine_readable) 160 | { 161 | printf("%s%s,%s,%u,Passed\r\n", seatest_magic_marker, seatest_current_fixture_path, function, line ); 162 | } 163 | else 164 | { 165 | printf("%-30s Line %-5d Passed\r\n", function, line); 166 | } 167 | } 168 | sea_tests_passed++; 169 | } 170 | } 171 | 172 | void seatest_assert_true(int test, const char* function, unsigned int line) 173 | { 174 | seatest_simple_test_result(test, "Should have been true", function, line); 175 | 176 | } 177 | 178 | void seatest_assert_false(int test, const char* function, unsigned int line) 179 | { 180 | seatest_simple_test_result(!test, "Should have been false", function, line); 181 | } 182 | 183 | 184 | void seatest_assert_int_equal(int expected, int actual, const char* function, unsigned int line) 185 | { 186 | char s[SEATEST_PRINT_BUFFER_SIZE]; 187 | sprintf(s, "Expected %d but was %d", expected, actual); 188 | seatest_simple_test_result(expected==actual, s, function, line); 189 | } 190 | 191 | void seatest_assert_ulong_equal(unsigned long expected, unsigned long actual, const char* function, unsigned int line) 192 | { 193 | char s[SEATEST_PRINT_BUFFER_SIZE]; 194 | sprintf(s, "Expected %lu but was %lu", expected, actual); 195 | seatest_simple_test_result(expected==actual, s, function, line); 196 | } 197 | 198 | void seatest_assert_float_equal( float expected, float actual, float delta, const char* function, unsigned int line ) 199 | { 200 | char s[SEATEST_PRINT_BUFFER_SIZE]; 201 | float result = expected-actual; 202 | sprintf(s, "Expected %f but was %f", expected, actual); 203 | if(result < 0.0) result = 0.0f - result; 204 | seatest_simple_test_result( result <= delta, s, function, line); 205 | } 206 | 207 | void seatest_assert_double_equal( double expected, double actual, double delta, const char* function, unsigned int line ) 208 | { 209 | char s[SEATEST_PRINT_BUFFER_SIZE]; 210 | double result = expected-actual; 211 | sprintf(s, "Expected %f but was %f", expected, actual); 212 | if(result < 0.0) result = 0.0 - result; 213 | seatest_simple_test_result( result <= delta, s, function, line); 214 | } 215 | 216 | void seatest_assert_string_equal(char* expected, char* actual, const char* function, unsigned int line) 217 | { 218 | int comparison; 219 | char s[SEATEST_PRINT_BUFFER_SIZE]; 220 | 221 | if ((expected == (char *)0) && (actual == (char *)0)) 222 | { 223 | sprintf(s, "Expected but was "); 224 | comparison = 1; 225 | } 226 | else if ((expected == (char *)0)) 227 | { 228 | sprintf(s, "Expected but was %s", actual); 229 | comparison = 0; 230 | } 231 | else if ((actual == (char *)0)) 232 | { 233 | sprintf(s, "Expected %s but was ", expected); 234 | comparison = 0; 235 | } 236 | else 237 | { 238 | comparison = strcmp(expected, actual) == 0; 239 | sprintf(s, "Expected %s but was %s", expected, actual); 240 | } 241 | 242 | seatest_simple_test_result(comparison, s, function, line); 243 | } 244 | 245 | void seatest_assert_string_ends_with(char* expected, char* actual, const char* function, unsigned int line) 246 | { 247 | char s[SEATEST_PRINT_BUFFER_SIZE]; 248 | sprintf(s, "Expected %s to end with %s", actual, expected); 249 | seatest_simple_test_result(strcmp(expected, actual+(strlen(actual)-strlen(expected)))==0, s, function, line); 250 | } 251 | 252 | void seatest_assert_string_starts_with(char* expected, char* actual, const char* function, unsigned int line) 253 | { 254 | char s[SEATEST_PRINT_BUFFER_SIZE]; 255 | sprintf(s, "Expected %s to start with %s", actual, expected); 256 | seatest_simple_test_result(strncmp(expected, actual, strlen(expected))==0, s, function, line); 257 | } 258 | 259 | void seatest_assert_string_contains(char* expected, char* actual, const char* function, unsigned int line) 260 | { 261 | char s[SEATEST_PRINT_BUFFER_SIZE]; 262 | sprintf(s, "Expected %s to be in %s", expected, actual); 263 | seatest_simple_test_result(strstr(actual, expected)!=0, s, function, line); 264 | } 265 | 266 | void seatest_assert_string_doesnt_contain(char* expected, char* actual, const char* function, unsigned int line) 267 | { 268 | char s[SEATEST_PRINT_BUFFER_SIZE]; 269 | sprintf(s, "Expected %s not to have %s in it", actual, expected); 270 | seatest_simple_test_result(strstr(actual, expected)==0, s, function, line); 271 | } 272 | 273 | void seatest_run_test(char* fixture, char* test) 274 | { 275 | sea_tests_run++; 276 | } 277 | 278 | void seatest_header_printer(char* s, int length, char f) 279 | { 280 | int l = strlen(s); 281 | int d = (length- (l + 2)) / 2; 282 | int i; 283 | if(seatest_is_display_only() || seatest_machine_readable) return; 284 | for(i = 0; iargc)) return 0; 408 | if(runner->argv[arg+1][0]=='-') return 0; 409 | return 1; 410 | } 411 | 412 | int seatest_parse_commandline_option_with_value(seatest_testrunner_t* runner, int arg, char* option, seatest_void_string setter) 413 | { 414 | if(seatest_is_string_equal_i(runner->argv[arg], option)) 415 | { 416 | if(!seatest_commandline_has_value_after(runner, arg)) 417 | { 418 | printf("Error: The %s option expects to be followed by a value\r\n", option); 419 | runner->action = SEATEST_DO_ABORT; 420 | return 0; 421 | } 422 | setter(runner->argv[arg+1]); 423 | return 1; 424 | } 425 | return 0; 426 | } 427 | 428 | void seatest_interpret_commandline(seatest_testrunner_t* runner) 429 | { 430 | int arg; 431 | for(arg=0; (arg < runner->argc) && (runner->action != SEATEST_DO_ABORT); arg++) 432 | { 433 | if(seatest_is_string_equal_i(runner->argv[arg], "help")) 434 | { 435 | seatest_show_help(); 436 | runner->action = SEATEST_DO_NOTHING; 437 | return; 438 | } 439 | if(seatest_is_string_equal_i(runner->argv[arg], "-d")) runner->action = SEATEST_DISPLAY_TESTS; 440 | if(seatest_is_string_equal_i(runner->argv[arg], "-v")) seatest_verbose = 1; 441 | if(seatest_is_string_equal_i(runner->argv[arg], "-m")) seatest_machine_readable = 1; 442 | if(seatest_parse_commandline_option_with_value(runner,arg,"-t", test_filter)) arg++; 443 | if(seatest_parse_commandline_option_with_value(runner,arg,"-f", fixture_filter)) arg++; 444 | if(seatest_parse_commandline_option_with_value(runner,arg,"-k", set_magic_marker)) arg++; 445 | } 446 | } 447 | 448 | void seatest_testrunner_create(seatest_testrunner_t* runner, int argc, char** argv ) 449 | { 450 | runner->action = SEATEST_RUN_TESTS; 451 | runner->argc = argc; 452 | runner->argv = argv; 453 | seatest_interpret_commandline(runner); 454 | } 455 | 456 | int seatest_testrunner(int argc, char** argv, seatest_void_void tests, seatest_void_void setup, seatest_void_void teardown) 457 | { 458 | seatest_testrunner_t runner; 459 | seatest_testrunner_create(&runner, argc, argv); 460 | switch(runner.action) 461 | { 462 | case SEATEST_DISPLAY_TESTS: 463 | { 464 | seatest_display_only = 1; 465 | run_tests(tests); 466 | break; 467 | } 468 | case SEATEST_RUN_TESTS: 469 | { 470 | suite_setup(setup); 471 | suite_teardown(teardown); 472 | return run_tests(tests); 473 | } 474 | case SEATEST_DO_NOTHING: 475 | case SEATEST_DO_ABORT: 476 | default: 477 | { 478 | /* nothing to do, probably because there was an error which should of been already printed out. */ 479 | } 480 | } 481 | return 1; 482 | } 483 | 484 | #ifdef SEATEST_INTERNAL_TESTS 485 | void seatest_simple_test_result_nolog(int passed, char* reason, const char* function, unsigned int line) 486 | { 487 | sea_test_last_passed = passed; 488 | } 489 | 490 | void seatest_assert_last_passed() 491 | { 492 | assert_int_equal(1, sea_test_last_passed); 493 | } 494 | 495 | void seatest_assert_last_failed() 496 | { 497 | assert_int_equal(0, sea_test_last_passed); 498 | } 499 | 500 | void seatest_disable_logging() 501 | { 502 | seatest_simple_test_result = seatest_simple_test_result_nolog; 503 | } 504 | 505 | void seatest_enable_logging() 506 | { 507 | seatest_simple_test_result = seatest_simple_test_result_log; 508 | } 509 | #endif 510 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | 294 | Copyright (C) 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | , 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. -------------------------------------------------------------------------------- /NOTES: -------------------------------------------------------------------------------- 1 | # -*-org-*- 2 | 3 | * Opening a Database 4 | 5 | All database operations are performed on an open database, which is represented 6 | by a database handle. 7 | 8 | Here's an example: 9 | 10 | #+begin_src c 11 | rc = tsdb_open(path, 12 | &handler, 13 | &num_values_per_entry, 14 | rrd_slot_time_duration, 15 | read_only) 16 | #+end_src 17 | 18 | path is the path to the database. 19 | 20 | handler is the reference to the database. This can be use in subsequent calls. 21 | 22 | num_values_per_entry specifies the number of values per entry when the database 23 | is created. If the database already exists, it's set to the number used when 24 | the database was created. Once set, it cannot be changed. 25 | 26 | rrd_slot_time_duration specifies the number of seconds per time slot. E.g. a 27 | value of 60 means that each slot in the database covers 60 seconds. 28 | 29 | * Setting the Current Epoch 30 | 31 | Use tsdb_goto_epoch to set the current epoch. This is used to read time series 32 | per key using tsdb_get. 33 | 34 | If load_on_demand is set to 1, the operation will always succeed. If set to 0, 35 | the operation will fail if the epoch doesn't exist. 36 | 37 | This operation has the substantial side effect of loading all of the time 38 | series for that specified epoch into memory. As time series are stored in one 39 | or more compressed fragments, this operation is not cheap. 40 | 41 | The advantage is that all of the time series for the epoch are available via a 42 | quick mapping of key to index offset. 43 | 44 | * Setting Values 45 | 46 | There are three operations involved in setting time series values: 47 | 48 | - Open a database (tsdb_open) 49 | - Go to an epoch (tsdb_goto_epoch) 50 | - Set the value (tsdb_set) 51 | 52 | * Indexes 53 | 54 | Keys are associated with indexes. 55 | 56 | The index for a key depends on the epoch. tsdb supports multiple indexes per 57 | key -- I *suspect* because at some point, the chunk, which is a fixed size, 58 | will fill up (all available indexes will be used) and in order to get an index, 59 | tsdb will create a new map of indexes. The new map will start at the current 60 | epoch and will remain in effect until all of the available indexes are used. 61 | 62 | What is the point of rsv-INDEX? 63 | 64 | An index mapping is represented by a "map-INDEX" entry in the db. 65 | 66 | The mapping consists of 67 | 68 | - epoch_start, which is set to the current chunk epoch 69 | - epoch_end, which is set to 0, indicating that the mapping is not bounded 70 | - the index value 71 | 72 | When a new index is created, it's also reserved by way of a "rsv-INDEX" entry 73 | in the db. 74 | 75 | Indexes aren't reused -- they keep growing as needed for each key. So they must 76 | be "rem'd" to the range of the value array. What then of chunks? 77 | 78 | As more keys are needed for an epoch, chunks are added. 79 | 80 | From what I can tell (suspect) as time goes by and keys change, we'll have 81 | chunks whose sole purppose is to maintain a consistent index offset. 82 | 83 | While there's support for mapping different indexes to different epoch ranges, 84 | it doesn't appear that it's even possible to reset/change an index for a key. 85 | 86 | The more I look, the more it looks like the whole mapping scheme is misguided: 87 | 88 | - At no point is the map expanded to a new epoch range 89 | 90 | - If a key comes along that has an index that's beyond the current epoch's 91 | range, a new chunk is added -- values are always accommodated by making room 92 | rather than creating a new mapping 93 | 94 | The scheme looks like it could deteriorate over time -- indexes that land past 95 | the previous chunks will cause the chunk size for new epochs to grow. There 96 | ought to be a way to reuse indexes for new epochs to avoid this. So e.g. the 97 | first 10K indexes could be phased out by a another 10K indexes and have new 98 | epochs still use only 1 chunk. 99 | 100 | How would this work? 101 | 102 | - You'd assign indexes up to num_values (10K) 103 | - At num_values + 1, the index would reset to 1 104 | - You'd mark the epoch end for *all* indexes that the current epoch 105 | 106 | This would reset the index counter. Old indexes would be preserved because 107 | they're tied to the old epoch range. New indexes would be assigned based on the 108 | current index counter and be associated with a new mapping entry. 109 | 110 | But rather than specify explicit start and stop epochsin each map, it'd be 111 | better to associate the map with a range generation. E.g. an index map would 112 | look like this: 113 | 114 | #+begin_src c 115 | typedef struct { 116 | u_int8_t generation; 117 | u_int32_t hash_idx; 118 | } tsdb_hash_mapping; 119 | #+end_src 120 | 121 | The generation would be associated with a range. 122 | 123 | There'd be a current generation, which would be used for new maps. 124 | 125 | Map entries would be added in LIFO order, so that the most recent maps occurred 126 | first in the array. 127 | 128 | The generation ranges could be stored in an array, and referenced by the 129 | generation value. This would make bounds checking very fast. 130 | 131 | There'd be no reason to use idx-INDEX entries -- you'd rely strictly on the 132 | index counter for the current map generation. 133 | 134 | * Purging Old Data 135 | 136 | I'm slightly inclined to *not* support data deletions (it's not supported now 137 | anyway). 138 | 139 | I think it'd be easier to push the bruden on multi DB queries. 140 | 141 | What's the cost of querying two or more databases to collect and consolidate 142 | values? Using the API as it is today, it would appear that there's not much 143 | cost -- values are retrieved one key at a time, each at a current epoch. 144 | 145 | Using multiple DBs could have a performance increase, as operations could be 146 | run in parallel. 147 | 148 | But what if we wanted to purge old values? 149 | 150 | One super easy method is to simply delete the epoch entries. This would clear 151 | out the majority of the values. 152 | 153 | That would leave cleaning indexes up -- the map-INDEX entries would continue to 154 | expand -- there'd be no easy way to reclaim old indexes (i.e. indexes that are 155 | defunct because there are no values associated with them), nor any way to reset 156 | the index counter. 157 | 158 | However, for use cases where indexes don't grow beyond the 10K limit (or 159 | whatever), dropping old epochs is a very easy way to reclaim space. 160 | 161 | * Questions 162 | 163 | ** load_page_on_demand 164 | 165 | When using load_page_on_demand=1 in tsdb_goto_epoch, map_raw_get always 166 | fails. For the time being, using load_page_on_demand=0. 167 | 168 | ** Missing Values 169 | 170 | The default_unknown_value for a handler is used as the resp in tsdb_get as 171 | expected, but tsdb_get returns -1, suggesting that the get failed. Is -1 172 | explicitly different from a general error, indicating that the default value 173 | was used? It seems to me that default value might not be needed, if we can 174 | infer that -1 mean "not set". 175 | 176 | ** Multiple Values 177 | 178 | What are some use cases for storing more than one value per key per slot? 179 | E.g. I could see this for double precision values. 180 | 181 | ** Open API Inconsistencies 182 | 183 | The open API is a bit odd I think: 184 | 185 | - On create num_values_per_entry and rrd_slot_time_duration are both sensibly 186 | used to initialize the database 187 | 188 | - On open, num_values_per_entry is ignored as input and is instread written to 189 | as an output variable, however rrd_slot_time_duration is neither read as 190 | input nor written to as output 191 | 192 | I'd expect something like this: 193 | 194 | - Both args are used as input/output 195 | 196 | - There's some indication as to whether or not the db was created or not 197 | 198 | Or 199 | 200 | - Both args are only input and are ignored on open 201 | 202 | - User needs to read from the handler to get the real values 203 | 204 | * Current Vision 205 | 206 | Same performance characteristics (storage, retrieval, space) of current tsdb, 207 | but with tag based queries. 208 | 209 | * Bugs 210 | 211 | ** test-simple fails sometimes, related to tracing 212 | 213 | Steps to reproduce: 214 | 215 | - Force a chunk growth with test-simple 216 | 217 | The test typically fails with: 218 | 219 | main Line 75 Expected 1000 but was 0 220 | 221 | Ways to make the test pass: 222 | 223 | - Set log level below warnings 224 | - Comment out the trace msg 225 | - trace something before the test is run (e.g. at the top of test_simple:main) 226 | - Comment out this line in tsdb_trace.c: 227 | 228 | strftime(theDate, 32, "%d/%b/%Y %H:%M:%S", localtime(&theTime)); 229 | 230 | This very odd behavior feels like a race condition, but on what I have no idea. 231 | 232 | * Next Steps 233 | 234 | ** DONE Clean up source 235 | 236 | Source code could use some reformatting: 237 | 238 | - 79 max line length 239 | - Improved white spacing 240 | - Misc cleanup 241 | 242 | ** DONE traceEvent to trace_info, etc 243 | ** DONE Misc refactor 244 | 245 | *** DONE db_set -> db_put 246 | *** DONE lowerCamelCase to standard 247 | *** DONE consistent names 248 | 249 | - hash/hash_index -> index 250 | - name/hash_name -> key 251 | 252 | *** DONE Shorter names 253 | 254 | - num_xxx -> xxx 255 | - xxx_mode -> xxx 256 | - alive_and_kicking -> open 257 | 258 | ** DONE Refactor map-XXX 259 | 260 | *** DONE Rename to key-XXX 261 | *** DONE Store only the index 262 | *** DONE Drop index map 263 | 264 | ** DONE Refactor rsv-XXX 265 | 266 | This is pointless. We can use the next counter authoritatively. 267 | 268 | ** DONE Refactor normalize_epoch 269 | 270 | Should be a void. 271 | 272 | ** DONE Possibly simplify goto_epoch options 273 | 274 | |---+--------+----------+-------+--------+------+-----+----------+-------| 275 | | # | Exists | Option | Value | Create | Load | Ret | Writable | Scrap | 276 | |---+--------+----------+-------+--------+------+-----+----------+-------| 277 | | 1 | no | create | 0 | no | n/a | -1 | no | | 278 | | | | ondemand | 0 | | | | | | 279 | |---+--------+----------+-------+--------+------+-----+----------+-------| 280 | | 3 | no | create | 1 | yes | yes | 0 | yes | | 281 | | | | ondemand | 0 | | | | | | 282 | |---+--------+----------+-------+--------+------+-----+----------+-------| 283 | | 5 | no | create | 0 | no | no | 0 | no | | 284 | | | | ondemand | 1 | | | | | | 285 | |---+--------+----------+-------+--------+------+-----+----------+-------| 286 | | 7 | no | create | 1 | yes | no | 0 | yes | | 287 | | | | ondemand | 1 | | | | | | 288 | |---+--------+----------+-------+--------+------+-----+----------+-------| 289 | | 2 | yes | create | 0 | n/a | yes | 0 | yes | y | 290 | | | | ondemand | 0 | | | | | | 291 | |---+--------+----------+-------+--------+------+-----+----------+-------| 292 | | 4 | yes | create | 1 | n/a | yes | 0 | yes | y | 293 | | | | ondemand | 0 | | | | | | 294 | |---+--------+----------+-------+--------+------+-----+----------+-------| 295 | | 6 | yes | create | 0 | n/a | no | 0 | yes | y | 296 | | | | ondemand | 1 | | | | | | 297 | |---+--------+----------+-------+--------+------+-----+----------+-------| 298 | | 8 | yes | create | 1 | n/a | no | 0 | yes | y | 299 | | | | ondemand | 1 | | | | | | 300 | |---+--------+----------+-------+--------+------+-----+----------+-------| 301 | 302 | 1. Fail if the epoch isn't there. This is useful for checking if an epoch 303 | exists. Could be handled with fail_if_missing=1 flag. 304 | 305 | 2. Actively load the epoch if it's there. 306 | 307 | if epoch found: 308 | load current epoch 309 | else: 310 | if error_on_missing flag: 311 | return error 312 | else: 313 | set current epoch as missing, create/load on demand on first write 314 | return success 315 | 316 | Proposed API: 317 | 318 | tsdb_goto_epoch(db, epoch, fail_on_missing, growable) 319 | 320 | ** DONE get_key_offset -> prepare_read_write 321 | ** DONE Lookup by index 322 | 323 | Provide an API for looking up epoch values using the epoch array offset 324 | directly, rather than dereference it each time: 325 | 326 | tsdb_get_by_index(db, index) 327 | 328 | This would require a corresponding function for getting a key's array offset: 329 | 330 | tsdb_get_key_index(db, key) 331 | 332 | We'd use this for range queries of a particular key by looking up the index 333 | first, then using it for each epoch. 334 | ** TODO More direct index support 335 | 336 | *** Return index used with tsdb_set 337 | 338 | We might as well return this value, as it's convenient. It can be used for 339 | followup index related operations, e.g. adding tags via index. 340 | 341 | *** add_tag_for_index 342 | 343 | We'll have the index when we set a value (see previous bullet) and can followup 344 | by adding applicable tags for it. 345 | 346 | If we maintained an index of uncompressed tags, this update operation would be 347 | 0(1) as it's a bit set operation on the array index. 348 | 349 | If we're lazy and reload the tag array on each operation, it's still O(1) for 350 | the insert, but the disk/db load cost (probably) overwhelms that savings. 351 | 352 | ** TODO Consider: tag array cache 353 | 354 | There will be a number of common tags that should be cached, saving the 355 | compress/decompress cycle and taking advantage of the O(1) cost of setting an 356 | index for the tag. 357 | 358 | This looks like a simple starting point for a LRU cache we could use for 359 | uncompressed tag arrays: 360 | 361 | http://www.geeksforgeeks.org/implement-lru-cache/ 362 | 363 | ** TODO Sure up read only mode 364 | 365 | Need some tests for read only mode. 366 | 367 | ** TODO Test growable = false and new keys 368 | ** TODO Exit on impossible conditions 369 | 370 | malloc, e.g. should cause an immediate exit. 371 | 372 | ** TODO Fix tracing 373 | 374 | I'm not seeing value from the tracing facility so far. If it continues to be 375 | more distration than helpful, nuke it. 376 | 377 | ** TODO Tags 378 | 379 | Support a new index type: a tab. 380 | 381 | The function would look like this: 382 | 383 | tsdb_add_key_tag(db, key, tag) 384 | 385 | E.g. 386 | 387 | tsdb_add_key_tag(db, "test-1", "metric=test") 388 | tsdb_add_key_tag(db, "test-2", "server=2") 389 | tsdb_add_key_tag(db, "test-3", "size=small") 390 | 391 | For each tag, ensure a tag-XXX entry in the database. 392 | 393 | Each tax-xxx db value would be an array of unsorted, unique key indexes. 394 | 395 | Each tag-XXX element would be an array. Each element in the array would be a 0 396 | or 1. 1 indicates that the tag applies to the epoch values at the correponding 397 | index. E.g. if position 0 of the tag array contained a 1, it would indicate 398 | that the tag applied to epoch values at position 1. 399 | 400 | For example, we'll start by adding a value to an empty database: 401 | 402 | tsdb_goto_epoch(db, 60, 0, 0) 403 | tsdb_set(db, "test-1", 100) 404 | 405 | We have in our db the following: 406 | 407 | key-test-1: 0 408 | 60-0: [ 56 ] 409 | 410 | And another key: 411 | 412 | tsdb_set(db, "test-2", 200) 413 | 414 | Our db: 415 | 416 | key-test-1: 0 417 | key-test-2: 1 418 | 1-0: [ 100, 200 ] 419 | 420 | Let's tag a key: 421 | 422 | tsdb_add_key_tag(db, "test-1", "metric=test") 423 | 424 | Our db has a new value: 425 | 426 | tag-metric=test: [ 1 ] 427 | 428 | This indicates that the epoch values stored at index 0 (i.e. key test-1) have 429 | the tag "metric=test". 430 | 431 | We'll do the same for test-2: 432 | 433 | tsdb_add_key_tag(db, "test-2", "metric=test") 434 | 435 | Now our tag element looks like this: 436 | 437 | tag-metric=test: [ 1, 1 ] 438 | 439 | Some more tags: 440 | 441 | tsdb_add_key_tag(db, "test-1", "server=1") 442 | tsdb_add_key_tag(db, "test-1", "size=small") 443 | tsdb_add_key_tag(db, "test-2", "server=2") 444 | tsdb_add_key_tag(db, "test-2", "size=large") 445 | 446 | Here's the full listing of all tag elements: 447 | 448 | tag-metric=test: [ 1, 1 ] 449 | tag-size=small: [ 1, 0 ] 450 | tag-size=large: [ 0, 1 ] 451 | tag-server=1: [ 1, 0 ] 452 | tag-server=2: [ 0, 1 ] 453 | 454 | Here's a query for metric=test & size=small: 455 | 456 | "intersection of metric=test and size=small" 457 | 458 | This is how we'd answer the query: 459 | 460 | - Load the arrays for the tags metric=test and size=small 461 | - Apply a logic AND operation for each array cell, up to the shorted array 462 | - Return values for epoch positions for each resulting 1 463 | 464 | E.g. 465 | 466 | tag-metric=test: [ 1, 1 ] 467 | tag-size=small: [ 1, 0 ] 468 | AND: [ 1, 0 ] 469 | 470 | We can implement a similar function for the union using OR, e.g. 471 | 472 | "union of metric=test and size=small" 473 | 474 | tag-metric=test: [ 1, 1 ] 475 | tag-size=small: [ 1, 0 ] 476 | OR: [ 1, 1 ] 477 | 478 | *** More 479 | 480 | We can implement NOT operations, e.g. 481 | 482 | tsdb_get_by_tag_not_intersection(db, ["metric=cpu", "server_size=small"]) 483 | tsdb_get_by_tag_not_union(db, ["metric=cpu", "server_size=small"]) 484 | 485 | By applying a NOT to the result. 486 | 487 | The most flexible scheme would be a simple query language. 488 | 489 | Here's a union of 2 tags. Each tag is preceded by its byte length. 490 | 491 | tsdb_query(db, [ "u" 2 10 "metric=cpu" 17 "server_size=small"]) 492 | 493 | Let's say I want all of the cpu metrics for servers that aren't small: 494 | 495 | tsdb_query(db, [ "i" 2 "t" 10 "metric=cpu" "n" "t" 17 "server_size=small" ]) 496 | 497 | Here's the key we're using so far: 498 | 499 | i = intersection 500 | u = union 501 | n = not 502 | t = tag 503 | 504 | So we'd implement this query in these steps: 505 | 506 | 1. Get the epoch indexes for the tag metric=cpu 507 | 508 | tag-metric=cpu: [ 1, 1 ] 509 | 510 | 2. Get the epoch indexes for the tag server_size=small 511 | 512 | tag-server_size=small: [ 1, 0 ] 513 | 514 | 3. Apply NOT to the result from step 2 515 | 516 | NOT tag-server_size=small: [ 0, 1 ] 517 | 518 | 4. Apply AND to the results from steps 1 and 3 519 | 520 | tag-metric=cpu: [ 1, 1 ] 521 | NOT tag-server_size=small: [ 0, 1 ] 522 | AND: [ 0, 1 ] 523 | 524 | We could obviously use bit arrays, which would reduce our space requirements 525 | for tagging by 32x. This at some minor code complexity. 526 | 527 | We could minimize the storage and IO requirements for these tag indexes further 528 | using compression, as is done currently with epoch values. 529 | 530 | A possible performance problem with tags is that they have to be read fully 531 | from the database, updated, and then stored again. If a tag array didn't have 532 | to be read from the database, it could be updated in constant time. 533 | 534 | However, before we worry too much about performance, we should consider the 535 | caching that occurs in the database. It may be pointless to cache tag arrays if 536 | they're already being cached by the database. As long as we're not compressing 537 | the array, it *might* be the case that we have direct access to the database's 538 | value in memory. 539 | 540 | -------------------------------------------------------------------------------- /quicklz.c: -------------------------------------------------------------------------------- 1 | // Fast data compression library 2 | // Copyright (C) 2006-2011 Lasse Mikkel Reinhold 3 | // lar@quicklz.com 4 | // 5 | // QuickLZ can be used for free under the GPL 1, 2 or 3 license (where anything 6 | // released into public must be open source) or under a commercial license if such 7 | // has been acquired (see http://www.quicklz.com/order.html). The commercial license 8 | // does not cover derived or ported versions created by third parties under GPL. 9 | 10 | // 1.5.0 final 11 | 12 | #include "quicklz.h" 13 | 14 | #if QLZ_VERSION_MAJOR != 1 || QLZ_VERSION_MINOR != 5 || QLZ_VERSION_REVISION != 0 15 | #error quicklz.c and quicklz.h have different versions 16 | #endif 17 | 18 | #if (defined(__X86__) || defined(__i386__) || defined(i386) || defined(_M_IX86) || defined(__386__) || defined(__x86_64__) || defined(_M_X64)) 19 | #define X86X64 20 | #endif 21 | 22 | #define MINOFFSET 2 23 | #define UNCONDITIONAL_MATCHLEN 6 24 | #define UNCOMPRESSED_END 4 25 | #define CWORD_LEN 4 26 | 27 | #if QLZ_COMPRESSION_LEVEL == 1 && defined QLZ_PTR_64 && QLZ_STREAMING_BUFFER == 0 28 | #define OFFSET_BASE source 29 | #define CAST (ui32)(size_t) 30 | #else 31 | #define OFFSET_BASE 0 32 | #define CAST 33 | #endif 34 | 35 | int qlz_get_setting(int setting) 36 | { 37 | switch (setting) 38 | { 39 | case 0: return QLZ_COMPRESSION_LEVEL; 40 | case 1: return sizeof(qlz_state_compress); 41 | case 2: return sizeof(qlz_state_decompress); 42 | case 3: return QLZ_STREAMING_BUFFER; 43 | #ifdef QLZ_MEMORY_SAFE 44 | case 6: return 1; 45 | #else 46 | case 6: return 0; 47 | #endif 48 | case 7: return QLZ_VERSION_MAJOR; 49 | case 8: return QLZ_VERSION_MINOR; 50 | case 9: return QLZ_VERSION_REVISION; 51 | } 52 | return -1; 53 | } 54 | 55 | #if QLZ_COMPRESSION_LEVEL == 1 56 | static int same(const unsigned char *src, size_t n) 57 | { 58 | while(n > 0 && *(src + n) == *src) 59 | n--; 60 | return n == 0 ? 1 : 0; 61 | } 62 | #endif 63 | 64 | static void reset_table_compress(qlz_state_compress *state) 65 | { 66 | int i; 67 | for(i = 0; i < QLZ_HASH_VALUES; i++) 68 | { 69 | #if QLZ_COMPRESSION_LEVEL == 1 70 | state->hash[i].offset = 0; 71 | #else 72 | state->hash_counter[i] = 0; 73 | #endif 74 | } 75 | } 76 | 77 | static void reset_table_decompress(qlz_state_decompress *state) 78 | { 79 | int i; 80 | (void)state; 81 | (void)i; 82 | #if QLZ_COMPRESSION_LEVEL == 2 83 | for(i = 0; i < QLZ_HASH_VALUES; i++) 84 | { 85 | state->hash_counter[i] = 0; 86 | } 87 | #endif 88 | } 89 | 90 | static __inline ui32 hash_func(ui32 i) 91 | { 92 | #if QLZ_COMPRESSION_LEVEL == 2 93 | return ((i >> 9) ^ (i >> 13) ^ i) & (QLZ_HASH_VALUES - 1); 94 | #else 95 | return ((i >> 12) ^ i) & (QLZ_HASH_VALUES - 1); 96 | #endif 97 | } 98 | 99 | static __inline ui32 fast_read(void const *src, ui32 bytes) 100 | { 101 | #ifndef X86X64 102 | unsigned char *p = (unsigned char*)src; 103 | switch (bytes) 104 | { 105 | case 4: 106 | return(*p | *(p + 1) << 8 | *(p + 2) << 16 | *(p + 3) << 24); 107 | case 3: 108 | return(*p | *(p + 1) << 8 | *(p + 2) << 16); 109 | case 2: 110 | return(*p | *(p + 1) << 8); 111 | case 1: 112 | return(*p); 113 | } 114 | return 0; 115 | #else 116 | if (bytes >= 1 && bytes <= 4) 117 | return *((ui32*)src); 118 | else 119 | return 0; 120 | #endif 121 | } 122 | 123 | static __inline ui32 hashat(const unsigned char *src) 124 | { 125 | ui32 fetch, hash; 126 | fetch = fast_read(src, 3); 127 | hash = hash_func(fetch); 128 | return hash; 129 | } 130 | 131 | static __inline void fast_write(ui32 f, void *dst, size_t bytes) 132 | { 133 | #ifndef X86X64 134 | unsigned char *p = (unsigned char*)dst; 135 | 136 | switch (bytes) 137 | { 138 | case 4: 139 | *p = (unsigned char)f; 140 | *(p + 1) = (unsigned char)(f >> 8); 141 | *(p + 2) = (unsigned char)(f >> 16); 142 | *(p + 3) = (unsigned char)(f >> 24); 143 | return; 144 | case 3: 145 | *p = (unsigned char)f; 146 | *(p + 1) = (unsigned char)(f >> 8); 147 | *(p + 2) = (unsigned char)(f >> 16); 148 | return; 149 | case 2: 150 | *p = (unsigned char)f; 151 | *(p + 1) = (unsigned char)(f >> 8); 152 | return; 153 | case 1: 154 | *p = (unsigned char)f; 155 | return; 156 | } 157 | #else 158 | switch (bytes) 159 | { 160 | case 4: 161 | *((ui32*)dst) = f; 162 | return; 163 | case 3: 164 | *((ui32*)dst) = f; 165 | return; 166 | case 2: 167 | *((ui16 *)dst) = (ui16)f; 168 | return; 169 | case 1: 170 | *((unsigned char*)dst) = (unsigned char)f; 171 | return; 172 | } 173 | #endif 174 | } 175 | 176 | 177 | size_t qlz_size_decompressed(const char *source) 178 | { 179 | ui32 n, r; 180 | n = (((*source) & 2) == 2) ? 4 : 1; 181 | r = fast_read(source + 1 + n, n); 182 | r = r & (0xffffffff >> ((4 - n)*8)); 183 | return r; 184 | } 185 | 186 | size_t qlz_size_compressed(const char *source) 187 | { 188 | ui32 n, r; 189 | n = (((*source) & 2) == 2) ? 4 : 1; 190 | r = fast_read(source + 1, n); 191 | r = r & (0xffffffff >> ((4 - n)*8)); 192 | return r; 193 | } 194 | 195 | static size_t qlz_size_header(const char *source) 196 | { 197 | size_t n = 2*((((*source) & 2) == 2) ? 4 : 1) + 1; 198 | return n; 199 | } 200 | 201 | 202 | static __inline void memcpy_up(unsigned char *dst, const unsigned char *src, ui32 n) 203 | { 204 | // Caution if modifying memcpy_up! Overlap of dst and src must be special handled. 205 | #ifndef X86X64 206 | unsigned char *end = dst + n; 207 | while(dst < end) 208 | { 209 | *dst = *src; 210 | dst++; 211 | src++; 212 | } 213 | #else 214 | ui32 f = 0; 215 | do 216 | { 217 | *(ui32 *)(dst + f) = *(ui32 *)(src + f); 218 | f += MINOFFSET + 1; 219 | } 220 | while (f < n); 221 | #endif 222 | } 223 | 224 | static __inline void update_hash(qlz_state_decompress *state, const unsigned char *s) 225 | { 226 | #if QLZ_COMPRESSION_LEVEL == 1 227 | ui32 hash; 228 | hash = hashat(s); 229 | state->hash[hash].offset = s; 230 | state->hash_counter[hash] = 1; 231 | #elif QLZ_COMPRESSION_LEVEL == 2 232 | ui32 hash; 233 | unsigned char c; 234 | hash = hashat(s); 235 | c = state->hash_counter[hash]; 236 | state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = s; 237 | c++; 238 | state->hash_counter[hash] = c; 239 | #endif 240 | (void)state; 241 | (void)s; 242 | } 243 | 244 | #if QLZ_COMPRESSION_LEVEL <= 2 245 | static void update_hash_upto(qlz_state_decompress *state, unsigned char **lh, const unsigned char *max) 246 | { 247 | while(*lh < max) 248 | { 249 | (*lh)++; 250 | update_hash(state, *lh); 251 | } 252 | } 253 | #endif 254 | 255 | static size_t qlz_compress_core(const unsigned char *source, unsigned char *destination, size_t size, qlz_state_compress *state) 256 | { 257 | const unsigned char *last_byte = source + size - 1; 258 | const unsigned char *src = source; 259 | unsigned char *cword_ptr = destination; 260 | unsigned char *dst = destination + CWORD_LEN; 261 | ui32 cword_val = 1U << 31; 262 | const unsigned char *last_matchstart = last_byte - UNCONDITIONAL_MATCHLEN - UNCOMPRESSED_END; 263 | ui32 fetch = 0; 264 | unsigned int lits = 0; 265 | 266 | (void) lits; 267 | 268 | if(src <= last_matchstart) 269 | fetch = fast_read(src, 3); 270 | 271 | while(src <= last_matchstart) 272 | { 273 | if ((cword_val & 1) == 1) 274 | { 275 | // store uncompressed if compression ratio is too low 276 | if (src > source + (size >> 1) && dst - destination > src - source - ((src - source) >> 5)) 277 | return 0; 278 | 279 | fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN); 280 | 281 | cword_ptr = dst; 282 | dst += CWORD_LEN; 283 | cword_val = 1U << 31; 284 | fetch = fast_read(src, 3); 285 | } 286 | #if QLZ_COMPRESSION_LEVEL == 1 287 | { 288 | const unsigned char *o; 289 | ui32 hash, cached; 290 | 291 | hash = hash_func(fetch); 292 | cached = fetch ^ state->hash[hash].cache; 293 | state->hash[hash].cache = fetch; 294 | 295 | o = state->hash[hash].offset + OFFSET_BASE; 296 | state->hash[hash].offset = CAST(src - OFFSET_BASE); 297 | 298 | #ifdef X86X64 299 | if ((cached & 0xffffff) == 0 && o != OFFSET_BASE && (src - o > MINOFFSET || (src == o + 1 && lits >= 3 && src > source + 3 && same(src - 3, 6)))) 300 | { 301 | if(cached != 0) 302 | { 303 | #else 304 | if (cached == 0 && o != OFFSET_BASE && (src - o > MINOFFSET || (src == o + 1 && lits >= 3 && src > source + 3 && same(src - 3, 6)))) 305 | { 306 | if (*(o + 3) != *(src + 3)) 307 | { 308 | #endif 309 | hash <<= 4; 310 | cword_val = (cword_val >> 1) | (1U << 31); 311 | fast_write((3 - 2) | hash, dst, 2); 312 | src += 3; 313 | dst += 2; 314 | } 315 | else 316 | { 317 | const unsigned char *old_src = src; 318 | size_t matchlen; 319 | hash <<= 4; 320 | 321 | cword_val = (cword_val >> 1) | (1U << 31); 322 | src += 4; 323 | 324 | if(*(o + (src - old_src)) == *src) 325 | { 326 | src++; 327 | if(*(o + (src - old_src)) == *src) 328 | { 329 | size_t q = last_byte - UNCOMPRESSED_END - (src - 5) + 1; 330 | size_t remaining = q > 255 ? 255 : q; 331 | src++; 332 | while(*(o + (src - old_src)) == *src && (size_t)(src - old_src) < remaining) 333 | src++; 334 | } 335 | } 336 | 337 | matchlen = src - old_src; 338 | if (matchlen < 18) 339 | { 340 | fast_write((ui32)(matchlen - 2) | hash, dst, 2); 341 | dst += 2; 342 | } 343 | else 344 | { 345 | fast_write((ui32)(matchlen << 16) | hash, dst, 3); 346 | dst += 3; 347 | } 348 | } 349 | fetch = fast_read(src, 3); 350 | lits = 0; 351 | } 352 | else 353 | { 354 | lits++; 355 | *dst = *src; 356 | src++; 357 | dst++; 358 | cword_val = (cword_val >> 1); 359 | #ifdef X86X64 360 | fetch = fast_read(src, 3); 361 | #else 362 | fetch = (fetch >> 8 & 0xffff) | (*(src + 2) << 16); 363 | #endif 364 | } 365 | } 366 | #elif QLZ_COMPRESSION_LEVEL >= 2 367 | { 368 | const unsigned char *o, *offset2; 369 | ui32 hash, matchlen, k, m, best_k = 0; 370 | unsigned char c; 371 | size_t remaining = (last_byte - UNCOMPRESSED_END - src + 1) > 255 ? 255 : (last_byte - UNCOMPRESSED_END - src + 1); 372 | (void)best_k; 373 | 374 | 375 | //hash = hashat(src); 376 | fetch = fast_read(src, 3); 377 | hash = hash_func(fetch); 378 | 379 | c = state->hash_counter[hash]; 380 | 381 | offset2 = state->hash[hash].offset[0]; 382 | if(offset2 < src - MINOFFSET && c > 0 && ((fast_read(offset2, 3) ^ fetch) & 0xffffff) == 0) 383 | { 384 | matchlen = 3; 385 | if(*(offset2 + matchlen) == *(src + matchlen)) 386 | { 387 | matchlen = 4; 388 | while(*(offset2 + matchlen) == *(src + matchlen) && matchlen < remaining) 389 | matchlen++; 390 | } 391 | } 392 | else 393 | matchlen = 0; 394 | for(k = 1; k < QLZ_POINTERS && c > k; k++) 395 | { 396 | o = state->hash[hash].offset[k]; 397 | #if QLZ_COMPRESSION_LEVEL == 3 398 | if(((fast_read(o, 3) ^ fetch) & 0xffffff) == 0 && o < src - MINOFFSET) 399 | #elif QLZ_COMPRESSION_LEVEL == 2 400 | if(*(src + matchlen) == *(o + matchlen) && ((fast_read(o, 3) ^ fetch) & 0xffffff) == 0 && o < src - MINOFFSET) 401 | #endif 402 | { 403 | m = 3; 404 | while(*(o + m) == *(src + m) && m < remaining) 405 | m++; 406 | #if QLZ_COMPRESSION_LEVEL == 3 407 | if ((m > matchlen) || (m == matchlen && o > offset2)) 408 | #elif QLZ_COMPRESSION_LEVEL == 2 409 | if (m > matchlen) 410 | #endif 411 | { 412 | offset2 = o; 413 | matchlen = m; 414 | best_k = k; 415 | } 416 | } 417 | } 418 | o = offset2; 419 | state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src; 420 | c++; 421 | state->hash_counter[hash] = c; 422 | 423 | #if QLZ_COMPRESSION_LEVEL == 3 424 | if(matchlen > 2 && src - o < 131071) 425 | { 426 | ui32 u; 427 | size_t offset = src - o; 428 | 429 | for(u = 1; u < matchlen; u++) 430 | { 431 | hash = hashat(src + u); 432 | c = state->hash_counter[hash]++; 433 | state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src + u; 434 | } 435 | 436 | cword_val = (cword_val >> 1) | (1U << 31); 437 | src += matchlen; 438 | 439 | if(matchlen == 3 && offset <= 63) 440 | { 441 | *dst = (unsigned char)(offset << 2); 442 | dst++; 443 | } 444 | else if (matchlen == 3 && offset <= 16383) 445 | { 446 | ui32 f = (ui32)((offset << 2) | 1); 447 | fast_write(f, dst, 2); 448 | dst += 2; 449 | } 450 | else if (matchlen <= 18 && offset <= 1023) 451 | { 452 | ui32 f = ((matchlen - 3) << 2) | ((ui32)offset << 6) | 2; 453 | fast_write(f, dst, 2); 454 | dst += 2; 455 | } 456 | 457 | else if(matchlen <= 33) 458 | { 459 | ui32 f = ((matchlen - 2) << 2) | ((ui32)offset << 7) | 3; 460 | fast_write(f, dst, 3); 461 | dst += 3; 462 | } 463 | else 464 | { 465 | ui32 f = ((matchlen - 3) << 7) | ((ui32)offset << 15) | 3; 466 | fast_write(f, dst, 4); 467 | dst += 4; 468 | } 469 | } 470 | else 471 | { 472 | *dst = *src; 473 | src++; 474 | dst++; 475 | cword_val = (cword_val >> 1); 476 | } 477 | #elif QLZ_COMPRESSION_LEVEL == 2 478 | 479 | if(matchlen > 2) 480 | { 481 | cword_val = (cword_val >> 1) | (1U << 31); 482 | src += matchlen; 483 | 484 | if (matchlen < 10) 485 | { 486 | ui32 f = best_k | ((matchlen - 2) << 2) | (hash << 5); 487 | fast_write(f, dst, 2); 488 | dst += 2; 489 | } 490 | else 491 | { 492 | ui32 f = best_k | (matchlen << 16) | (hash << 5); 493 | fast_write(f, dst, 3); 494 | dst += 3; 495 | } 496 | } 497 | else 498 | { 499 | *dst = *src; 500 | src++; 501 | dst++; 502 | cword_val = (cword_val >> 1); 503 | } 504 | #endif 505 | } 506 | #endif 507 | } 508 | while (src <= last_byte) 509 | { 510 | if ((cword_val & 1) == 1) 511 | { 512 | fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN); 513 | cword_ptr = dst; 514 | dst += CWORD_LEN; 515 | cword_val = 1U << 31; 516 | } 517 | #if QLZ_COMPRESSION_LEVEL < 3 518 | if (src <= last_byte - 3) 519 | { 520 | #if QLZ_COMPRESSION_LEVEL == 1 521 | ui32 hash, fetch1; 522 | fetch1 = fast_read(src, 3); 523 | hash = hash_func(fetch1); 524 | state->hash[hash].offset = CAST(src - OFFSET_BASE); 525 | state->hash[hash].cache = fetch1; 526 | #elif QLZ_COMPRESSION_LEVEL == 2 527 | ui32 hash; 528 | unsigned char c; 529 | hash = hashat(src); 530 | c = state->hash_counter[hash]; 531 | state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src; 532 | c++; 533 | state->hash_counter[hash] = c; 534 | #endif 535 | } 536 | #endif 537 | *dst = *src; 538 | src++; 539 | dst++; 540 | cword_val = (cword_val >> 1); 541 | } 542 | 543 | while((cword_val & 1) != 1) 544 | cword_val = (cword_val >> 1); 545 | 546 | fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN); 547 | 548 | // min. size must be 9 bytes so that the qlz_size functions can take 9 bytes as argument 549 | return dst - destination < 9 ? 9 : dst - destination; 550 | } 551 | 552 | static size_t qlz_decompress_core(const unsigned char *source, unsigned char *destination, size_t size, qlz_state_decompress *state, const unsigned char *history) 553 | { 554 | const unsigned char *src = source + qlz_size_header((const char *)source); 555 | unsigned char *dst = destination; 556 | const unsigned char *last_destination_byte = destination + size - 1; 557 | ui32 cword_val = 1; 558 | const unsigned char *last_matchstart = last_destination_byte - UNCONDITIONAL_MATCHLEN - UNCOMPRESSED_END; 559 | unsigned char *last_hashed = destination - 1; 560 | const unsigned char *last_source_byte = source + qlz_size_compressed((const char *)source) - 1; 561 | static const ui32 bitlut[16] = {4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0}; 562 | 563 | (void) last_source_byte; 564 | (void) last_hashed; 565 | (void) state; 566 | (void) history; 567 | 568 | for(;;) 569 | { 570 | ui32 fetch; 571 | 572 | if (cword_val == 1) 573 | { 574 | #ifdef QLZ_MEMORY_SAFE 575 | if(src + CWORD_LEN - 1 > last_source_byte) 576 | return 0; 577 | #endif 578 | cword_val = fast_read(src, CWORD_LEN); 579 | src += CWORD_LEN; 580 | } 581 | 582 | #ifdef QLZ_MEMORY_SAFE 583 | if(src + 4 - 1 > last_source_byte) 584 | return 0; 585 | #endif 586 | 587 | fetch = fast_read(src, 4); 588 | 589 | if ((cword_val & 1) == 1) 590 | { 591 | ui32 matchlen; 592 | const unsigned char *offset2; 593 | 594 | #if QLZ_COMPRESSION_LEVEL == 1 595 | ui32 hash; 596 | cword_val = cword_val >> 1; 597 | hash = (fetch >> 4) & 0xfff; 598 | offset2 = (const unsigned char *)(size_t)state->hash[hash].offset; 599 | 600 | if((fetch & 0xf) != 0) 601 | { 602 | matchlen = (fetch & 0xf) + 2; 603 | src += 2; 604 | } 605 | else 606 | { 607 | matchlen = *(src + 2); 608 | src += 3; 609 | } 610 | 611 | #elif QLZ_COMPRESSION_LEVEL == 2 612 | ui32 hash; 613 | unsigned char c; 614 | cword_val = cword_val >> 1; 615 | hash = (fetch >> 5) & 0x7ff; 616 | c = (unsigned char)(fetch & 0x3); 617 | offset2 = state->hash[hash].offset[c]; 618 | 619 | if((fetch & (28)) != 0) 620 | { 621 | matchlen = ((fetch >> 2) & 0x7) + 2; 622 | src += 2; 623 | } 624 | else 625 | { 626 | matchlen = *(src + 2); 627 | src += 3; 628 | } 629 | 630 | #elif QLZ_COMPRESSION_LEVEL == 3 631 | ui32 offset; 632 | cword_val = cword_val >> 1; 633 | if ((fetch & 3) == 0) 634 | { 635 | offset = (fetch & 0xff) >> 2; 636 | matchlen = 3; 637 | src++; 638 | } 639 | else if ((fetch & 2) == 0) 640 | { 641 | offset = (fetch & 0xffff) >> 2; 642 | matchlen = 3; 643 | src += 2; 644 | } 645 | else if ((fetch & 1) == 0) 646 | { 647 | offset = (fetch & 0xffff) >> 6; 648 | matchlen = ((fetch >> 2) & 15) + 3; 649 | src += 2; 650 | } 651 | else if ((fetch & 127) != 3) 652 | { 653 | offset = (fetch >> 7) & 0x1ffff; 654 | matchlen = ((fetch >> 2) & 0x1f) + 2; 655 | src += 3; 656 | } 657 | else 658 | { 659 | offset = (fetch >> 15); 660 | matchlen = ((fetch >> 7) & 255) + 3; 661 | src += 4; 662 | } 663 | 664 | offset2 = dst - offset; 665 | #endif 666 | 667 | #ifdef QLZ_MEMORY_SAFE 668 | if(offset2 < history || offset2 > dst - MINOFFSET - 1) 669 | return 0; 670 | 671 | if(matchlen > (ui32)(last_destination_byte - dst - UNCOMPRESSED_END + 1)) 672 | return 0; 673 | #endif 674 | 675 | memcpy_up(dst, offset2, matchlen); 676 | dst += matchlen; 677 | 678 | #if QLZ_COMPRESSION_LEVEL <= 2 679 | update_hash_upto(state, &last_hashed, dst - matchlen); 680 | last_hashed = dst - 1; 681 | #endif 682 | } 683 | else 684 | { 685 | if (dst < last_matchstart) 686 | { 687 | unsigned int n = bitlut[cword_val & 0xf]; 688 | #ifdef X86X64 689 | *(ui32 *)dst = *(ui32 *)src; 690 | #else 691 | memcpy_up(dst, src, 4); 692 | #endif 693 | cword_val = cword_val >> n; 694 | dst += n; 695 | src += n; 696 | #if QLZ_COMPRESSION_LEVEL <= 2 697 | update_hash_upto(state, &last_hashed, dst - 3); 698 | #endif 699 | } 700 | else 701 | { 702 | while(dst <= last_destination_byte) 703 | { 704 | if (cword_val == 1) 705 | { 706 | src += CWORD_LEN; 707 | cword_val = 1U << 31; 708 | } 709 | #ifdef QLZ_MEMORY_SAFE 710 | if(src >= last_source_byte + 1) 711 | return 0; 712 | #endif 713 | *dst = *src; 714 | dst++; 715 | src++; 716 | cword_val = cword_val >> 1; 717 | } 718 | 719 | #if QLZ_COMPRESSION_LEVEL <= 2 720 | update_hash_upto(state, &last_hashed, last_destination_byte - 3); // todo, use constant 721 | #endif 722 | return size; 723 | } 724 | 725 | } 726 | } 727 | } 728 | 729 | size_t qlz_compress(const void *source, char *destination, size_t size, qlz_state_compress *state) 730 | { 731 | size_t r; 732 | ui32 compressed; 733 | size_t base; 734 | 735 | if(size == 0 || size > 0xffffffff - 400) 736 | return 0; 737 | 738 | if(size < 216) 739 | base = 3; 740 | else 741 | base = 9; 742 | 743 | #if QLZ_STREAMING_BUFFER > 0 744 | if (state->stream_counter + size - 1 >= QLZ_STREAMING_BUFFER) 745 | #endif 746 | { 747 | reset_table_compress(state); 748 | r = base + qlz_compress_core((const unsigned char *)source, (unsigned char*)destination + base, size, state); 749 | #if QLZ_STREAMING_BUFFER > 0 750 | reset_table_compress(state); 751 | #endif 752 | if(r == base) 753 | { 754 | memcpy(destination + base, source, size); 755 | r = size + base; 756 | compressed = 0; 757 | } 758 | else 759 | { 760 | compressed = 1; 761 | } 762 | state->stream_counter = 0; 763 | } 764 | #if QLZ_STREAMING_BUFFER > 0 765 | else 766 | { 767 | unsigned char *src = state->stream_buffer + state->stream_counter; 768 | 769 | memcpy(src, source, size); 770 | r = base + qlz_compress_core(src, (unsigned char*)destination + base, size, state); 771 | 772 | if(r == base) 773 | { 774 | memcpy(destination + base, src, size); 775 | r = size + base; 776 | compressed = 0; 777 | reset_table_compress(state); 778 | } 779 | else 780 | { 781 | compressed = 1; 782 | } 783 | state->stream_counter += size; 784 | } 785 | #endif 786 | if(base == 3) 787 | { 788 | *destination = (unsigned char)(0 | compressed); 789 | *(destination + 1) = (unsigned char)r; 790 | *(destination + 2) = (unsigned char)size; 791 | } 792 | else 793 | { 794 | *destination = (unsigned char)(2 | compressed); 795 | fast_write((ui32)r, destination + 1, 4); 796 | fast_write((ui32)size, destination + 5, 4); 797 | } 798 | 799 | *destination |= (QLZ_COMPRESSION_LEVEL << 2); 800 | *destination |= (1 << 6); 801 | *destination |= ((QLZ_STREAMING_BUFFER == 0 ? 0 : (QLZ_STREAMING_BUFFER == 100000 ? 1 : (QLZ_STREAMING_BUFFER == 1000000 ? 2 : 3))) << 4); 802 | 803 | // 76543210 804 | // 01SSLLHC 805 | 806 | return r; 807 | } 808 | 809 | size_t qlz_decompress(const char *source, void *destination, qlz_state_decompress *state) 810 | { 811 | size_t dsiz = qlz_size_decompressed(source); 812 | 813 | #if QLZ_STREAMING_BUFFER > 0 814 | if (state->stream_counter + qlz_size_decompressed(source) - 1 >= QLZ_STREAMING_BUFFER) 815 | #endif 816 | { 817 | if((*source & 1) == 1) 818 | { 819 | reset_table_decompress(state); 820 | dsiz = qlz_decompress_core((const unsigned char *)source, (unsigned char *)destination, dsiz, state, (const unsigned char *)destination); 821 | } 822 | else 823 | { 824 | memcpy(destination, source + qlz_size_header(source), dsiz); 825 | } 826 | state->stream_counter = 0; 827 | reset_table_decompress(state); 828 | } 829 | #if QLZ_STREAMING_BUFFER > 0 830 | else 831 | { 832 | unsigned char *dst = state->stream_buffer + state->stream_counter; 833 | if((*source & 1) == 1) 834 | { 835 | dsiz = qlz_decompress_core((const unsigned char *)source, dst, dsiz, state, (const unsigned char *)state->stream_buffer); 836 | } 837 | else 838 | { 839 | memcpy(dst, source + qlz_size_header(source), dsiz); 840 | reset_table_decompress(state); 841 | } 842 | memcpy(destination, dst, dsiz); 843 | state->stream_counter += dsiz; 844 | } 845 | #endif 846 | return dsiz; 847 | } 848 | 849 | -------------------------------------------------------------------------------- /tsdb_api.c: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright (C) 2011 IIT/CNR (http://www.iit.cnr.it/en) 4 | * Luca Deri 5 | * 6 | * This program is free software; you can redistribute it and/or modify 7 | * it under the terms of the GNU General Public License as published by 8 | * the Free Software Foundation; either version 2 of the License, or 9 | * (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License along 17 | * with this program; if not, write to the Free Software Foundation, Inc., 18 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 19 | */ 20 | 21 | #include "tsdb_api.h" 22 | #include "tsdb_bitmap.h" 23 | 24 | static void db_put(tsdb_handler *handler, 25 | void *key, u_int32_t key_len, 26 | void *value, u_int32_t value_len) { 27 | DBT key_data, data; 28 | 29 | if (handler->read_only) { 30 | trace_warning("Unable to set value (read-only mode)"); 31 | return; 32 | } 33 | 34 | memset(&key_data, 0, sizeof(key_data)); 35 | memset(&data, 0, sizeof(data)); 36 | 37 | key_data.data = key; 38 | key_data.size = key_len; 39 | data.data = value; 40 | data.size = value_len; 41 | 42 | if (handler->db->put(handler->db, NULL, &key_data, &data, 0) != 0) { 43 | trace_error("Error while map_set(%u, %u)", key, value); 44 | } 45 | } 46 | 47 | static int db_get(tsdb_handler *handler, 48 | void *key, u_int32_t key_len, 49 | void **value, u_int32_t *value_len) { 50 | DBT key_data, data; 51 | 52 | memset(&key_data, 0, sizeof(key_data)); 53 | memset(&data, 0, sizeof(data)); 54 | 55 | key_data.data = key; 56 | key_data.size = key_len; 57 | 58 | if (handler->db->get(handler->db, NULL, &key_data, &data, 0) == 0) { 59 | *value = data.data, *value_len = data.size; 60 | return 0; 61 | } else { 62 | return -1; 63 | } 64 | } 65 | 66 | int tsdb_open(char *tsdb_path, tsdb_handler *handler, 67 | u_int16_t *values_per_entry, 68 | u_int32_t slot_duration, 69 | u_int8_t read_only) { 70 | void *value; 71 | u_int32_t value_len; 72 | int ret, mode; 73 | 74 | memset(handler, 0, sizeof(tsdb_handler)); 75 | 76 | handler->read_only = read_only; 77 | 78 | if ((ret = db_create(&handler->db, NULL, 0)) != 0) { 79 | trace_error("Error while creating DB handler [%s]", db_strerror(ret)); 80 | return -1; 81 | } 82 | 83 | mode = (read_only ? 00444 : 00664 ); 84 | 85 | if ((ret = handler->db->open(handler->db, 86 | NULL, 87 | (const char*)tsdb_path, 88 | NULL, 89 | DB_BTREE, 90 | (read_only ? 0 : DB_CREATE), 91 | mode)) != 0) { 92 | trace_error("Error while opening DB %s [%s][r/o=%u,mode=%o]", 93 | tsdb_path, db_strerror(ret), read_only, mode); 94 | return -1; 95 | } 96 | 97 | if (db_get(handler, "lowest_free_index", 98 | strlen("lowest_free_index"), 99 | &value, &value_len) == 0) { 100 | handler->lowest_free_index = *((u_int32_t*)value); 101 | } else { 102 | if (!handler->read_only) { 103 | handler->lowest_free_index = 0; 104 | db_put(handler, "lowest_free_index", 105 | strlen("lowest_free_index"), 106 | &handler->lowest_free_index, 107 | sizeof(handler->lowest_free_index)); 108 | } 109 | } 110 | 111 | if (db_get(handler, "slot_duration", 112 | strlen("slot_duration"), 113 | &value, &value_len) == 0) { 114 | handler->slot_duration = *((u_int32_t*)value); 115 | } else { 116 | if (!handler->read_only) { 117 | handler->slot_duration = slot_duration; 118 | db_put(handler, "slot_duration", 119 | strlen("slot_duration"), 120 | &handler->slot_duration, 121 | sizeof(handler->slot_duration)); 122 | } 123 | } 124 | 125 | if (db_get(handler, "values_per_entry", 126 | strlen("values_per_entry"), 127 | &value, &value_len) == 0) { 128 | *values_per_entry = handler->values_per_entry = 129 | *((u_int16_t*)value); 130 | } else { 131 | if (!handler->read_only) { 132 | handler->values_per_entry = *values_per_entry; 133 | db_put(handler, "values_per_entry", 134 | strlen("values_per_entry"), 135 | &handler->values_per_entry, 136 | sizeof(handler->values_per_entry)); 137 | } 138 | } 139 | 140 | handler->values_len = handler->values_per_entry * sizeof(tsdb_value); 141 | 142 | trace_info("lowest_free_index: %u", handler->lowest_free_index); 143 | trace_info("slot_duration: %u", handler->slot_duration); 144 | trace_info("values_per_entry: %u", handler->values_per_entry); 145 | 146 | memset(&handler->state_compress, 0, sizeof(handler->state_compress)); 147 | memset(&handler->state_decompress, 0, sizeof(handler->state_decompress)); 148 | 149 | handler->alive = 1; 150 | 151 | return 0; 152 | } 153 | 154 | static void tsdb_flush_chunk(tsdb_handler *handler) { 155 | char *compressed; 156 | u_int compressed_len, new_len, num_fragments, i; 157 | u_int fragment_size; 158 | char str[32]; 159 | 160 | if (!handler->chunk.data) return; 161 | 162 | fragment_size = handler->values_len * CHUNK_GROWTH; 163 | new_len = handler->chunk.data_len + CHUNK_LEN_PADDING; 164 | compressed = (char*)malloc(new_len); 165 | if (!compressed) { 166 | trace_error("Not enough memory (%u bytes)", new_len); 167 | return; 168 | } 169 | 170 | // Split chunks on the DB 171 | num_fragments = handler->chunk.data_len / fragment_size; 172 | 173 | for (i=0; i < num_fragments; i++) { 174 | u_int offset; 175 | 176 | if ((!handler->read_only) && handler->chunk.fragment_changed[i]) { 177 | offset = i * fragment_size; 178 | 179 | compressed_len = qlz_compress(&handler->chunk.data[offset], 180 | compressed, fragment_size, 181 | &handler->state_compress); 182 | 183 | trace_info("Compression %u -> %u [fragment %u] [%.1f %%]", 184 | fragment_size, compressed_len, i, 185 | ((float)(compressed_len*100))/((float)fragment_size)); 186 | 187 | snprintf(str, sizeof(str), "%u-%u", handler->chunk.epoch, i); 188 | 189 | db_put(handler, str, strlen(str), compressed, compressed_len); 190 | } else { 191 | trace_info("Skipping fragment %u (unchanged)", i); 192 | } 193 | } 194 | 195 | free(compressed); 196 | free(handler->chunk.data); 197 | memset(&handler->chunk, 0, sizeof(handler->chunk)); 198 | handler->chunk.epoch = 0; 199 | handler->chunk.data_len = 0; 200 | } 201 | 202 | void tsdb_close(tsdb_handler *handler) { 203 | 204 | if (!handler->alive) { 205 | return; 206 | } 207 | 208 | tsdb_flush_chunk(handler); 209 | 210 | if (!handler->read_only) { 211 | trace_info("Flushing database changes..."); 212 | } 213 | 214 | handler->db->close(handler->db, 0); 215 | 216 | handler->alive = 0; 217 | } 218 | 219 | void normalize_epoch(tsdb_handler *handler, u_int32_t *epoch) { 220 | *epoch -= *epoch % handler->slot_duration; 221 | *epoch += timezone - daylight * 3600; 222 | } 223 | 224 | int tsdb_get_key_index(tsdb_handler *handler, char *key, u_int32_t *index) { 225 | void *ptr; 226 | u_int32_t len; 227 | char str[32] = { 0 }; 228 | 229 | snprintf(str, sizeof(str), "key-%s", key); 230 | 231 | if (db_get(handler, str, strlen(str), &ptr, &len) == 0) { 232 | *index = *(u_int32_t*)ptr; 233 | return 0; 234 | } 235 | 236 | return -1; 237 | } 238 | 239 | static void set_key_index(tsdb_handler *handler, char *key, u_int32_t index) { 240 | char str[32]; 241 | 242 | snprintf(str, sizeof(str), "key-%s", key); 243 | 244 | db_put(handler, str, strlen(str), &index, sizeof(index)); 245 | 246 | trace_info("[SET] Mapping %s -> %u", key, index); 247 | } 248 | 249 | int tsdb_goto_epoch(tsdb_handler *handler, 250 | u_int32_t epoch, 251 | u_int8_t fail_if_missing, 252 | u_int8_t growable) { 253 | int rc; 254 | void *value; 255 | u_int32_t value_len, fragment = 0; 256 | char str[32]; 257 | 258 | if (handler->chunk.epoch == epoch) { 259 | return 0; 260 | } 261 | 262 | tsdb_flush_chunk(handler); 263 | 264 | normalize_epoch(handler, &epoch); 265 | snprintf(str, sizeof(str), "%u-%u", epoch, fragment); 266 | 267 | rc = db_get(handler, str, strlen(str), &value, &value_len); 268 | 269 | if (rc == -1 && fail_if_missing) { 270 | return -1; 271 | } 272 | 273 | handler->chunk.epoch = epoch; 274 | handler->chunk.growable = growable; 275 | 276 | if (rc == 0) { 277 | u_int32_t len, offset = 0; 278 | u_int8_t *ptr; 279 | 280 | trace_info("Loading epoch %u", epoch); 281 | 282 | while (1) { 283 | len = qlz_size_decompressed(value); 284 | ptr = (u_int8_t*)malloc(handler->chunk.data_len + len); 285 | if (ptr == NULL) { 286 | trace_error("Not enough memory (%u bytes)", 287 | handler->chunk.data_len+len); 288 | free(value); 289 | return -2; 290 | } 291 | if (handler->chunk.data_len > 0) { 292 | memcpy(ptr, handler->chunk.data, handler->chunk.data_len); 293 | } 294 | len = qlz_decompress(value, &ptr[offset], 295 | &handler->state_decompress); 296 | 297 | trace_info("Decompression %u -> %u [fragment %u] [%.1f %%]", 298 | value_len, len, fragment, 299 | ((float)(len*100))/((float)value_len)); 300 | 301 | handler->chunk.data_len += len; 302 | fragment++; 303 | offset = handler->chunk.data_len; 304 | 305 | free(handler->chunk.data); 306 | handler->chunk.data = ptr; 307 | 308 | snprintf(str, sizeof(str), "%u-%u", epoch, fragment); 309 | if (db_get(handler, str, strlen(str), &value, &value_len) == -1) { 310 | break; // No more fragments 311 | } 312 | } 313 | } 314 | 315 | return 0; 316 | } 317 | 318 | static int ensure_key_index(tsdb_handler *handler, char *key, 319 | u_int32_t *index, u_int8_t for_write) { 320 | if (tsdb_get_key_index(handler, key, index) == 0) { 321 | trace_info("Index %s mapped to hash %u", key, *index); 322 | return 0; 323 | } 324 | 325 | if (!for_write) { 326 | trace_info("Unable to find index %s", key); 327 | return -1; 328 | } 329 | 330 | *index = handler->lowest_free_index++; 331 | set_key_index(handler, key, *index); 332 | 333 | db_put(handler, 334 | "lowest_free_index", strlen("lowest_free_index"), 335 | &handler->lowest_free_index, 336 | sizeof(handler->lowest_free_index)); 337 | 338 | return 0; 339 | } 340 | 341 | static int prepare_offset_by_index(tsdb_handler *handler, u_int32_t *index, 342 | u_int64_t *offset, u_int8_t for_write) { 343 | 344 | if (!handler->chunk.data) { 345 | if (!for_write) { 346 | return -1; 347 | } 348 | 349 | u_int32_t fragment = *index / CHUNK_GROWTH, value_len; 350 | char str[32]; 351 | void *value; 352 | 353 | // Load the epoch handler->chunk.epoch/fragment 354 | 355 | snprintf(str, sizeof(str), "%u-%u", handler->chunk.epoch, fragment); 356 | if (db_get(handler, str, strlen(str), &value, &value_len) == -1) { 357 | u_int32_t mem_len = handler->values_len * CHUNK_GROWTH; 358 | handler->chunk.data_len = mem_len; 359 | handler->chunk.data = (u_int8_t*)malloc(mem_len); 360 | if (handler->chunk.data == NULL) { 361 | trace_error("Not enough memory (%u bytes)", mem_len); 362 | return -2; 363 | } 364 | memset(handler->chunk.data, 365 | handler->unknown_value, 366 | handler->chunk.data_len); 367 | } else { 368 | handler->chunk.data_len = qlz_size_decompressed(value); 369 | handler->chunk.data = (u_int8_t*)malloc(handler->chunk.data_len); 370 | if (handler->chunk.data == NULL) { 371 | trace_error("Not enough memory (%u bytes)", 372 | handler->chunk.data_len); 373 | return -2; 374 | } 375 | qlz_decompress(value, handler->chunk.data, 376 | &handler->state_decompress); 377 | } 378 | handler->chunk.base_index = fragment * CHUNK_GROWTH; 379 | index -= handler->chunk.base_index; 380 | } 381 | 382 | get_offset: 383 | 384 | if (*index >= (handler->chunk.data_len / handler->values_len)) { 385 | if (!for_write || !handler->chunk.growable) { 386 | return -1; 387 | } 388 | 389 | u_int32_t to_add = CHUNK_GROWTH * handler->values_len; 390 | u_int32_t new_len = handler->chunk.data_len + to_add; 391 | u_int8_t *ptr = malloc(new_len); 392 | 393 | if (!ptr) { 394 | trace_error("Not enough memory (%u bytes): unable to grow " 395 | "table", new_len); 396 | return -2; 397 | } 398 | 399 | memcpy(ptr, handler->chunk.data, handler->chunk.data_len); 400 | memset(&ptr[handler->chunk.data_len], 401 | handler->unknown_value, to_add); 402 | handler->chunk.data = ptr; 403 | handler->chunk.data_len = new_len; 404 | 405 | trace_warning("Epoch grown to %u", new_len); 406 | 407 | goto get_offset; 408 | } 409 | 410 | *offset = handler->values_len * *index; 411 | 412 | if (*offset >= handler->chunk.data_len) { 413 | trace_error("INTERNAL ERROR [Id: %u][Offset: %u/%u]", 414 | *index, *offset, handler->chunk.data_len); 415 | } 416 | 417 | return 0; 418 | } 419 | 420 | static int prepare_offset_by_key(tsdb_handler *handler, char *key, 421 | u_int64_t *offset, u_int8_t for_write) { 422 | u_int32_t index; 423 | 424 | if (!handler->chunk.epoch) { 425 | return -1; 426 | } 427 | 428 | if (ensure_key_index(handler, key, &index, for_write) == -1) { 429 | trace_info("Unable to find index %s", key); 430 | return -1; 431 | } 432 | 433 | trace_info("%s mapped to idx %u", key, index); 434 | 435 | return prepare_offset_by_index(handler, &index, offset, for_write); 436 | } 437 | 438 | int tsdb_set_with_index(tsdb_handler *handler, char *key, 439 | tsdb_value *value, u_int32_t *index) { 440 | u_int32_t *chunk_ptr; 441 | u_int64_t offset; 442 | int rc; 443 | 444 | if (!handler->alive) { 445 | return -1; 446 | } 447 | 448 | if (!handler->chunk.epoch) { 449 | trace_error("Missing epoch"); 450 | return -2; 451 | } 452 | 453 | rc = prepare_offset_by_key(handler, key, &offset, 1); 454 | if (rc == 0) { 455 | chunk_ptr = (tsdb_value*)(&handler->chunk.data[offset]); 456 | memcpy(chunk_ptr, value, handler->values_len); 457 | 458 | // Mark a fragment as changed 459 | int fragment = offset / CHUNK_GROWTH; 460 | if (fragment > MAX_NUM_FRAGMENTS) { 461 | trace_error("Internal error [%u > %u]", 462 | fragment, MAX_NUM_FRAGMENTS); 463 | } else { 464 | handler->chunk.fragment_changed[fragment] = 1; 465 | } 466 | 467 | *index = offset / handler->values_len; 468 | } 469 | 470 | return rc; 471 | } 472 | 473 | int tsdb_set(tsdb_handler *handler, char *key, tsdb_value *value) { 474 | u_int32_t index; 475 | return tsdb_set_with_index(handler, key, value, &index); 476 | } 477 | 478 | int tsdb_get_by_key(tsdb_handler *handler, char *key, tsdb_value **value) { 479 | u_int64_t offset; 480 | int rc; 481 | 482 | if (!handler->alive || !handler->chunk.data) { 483 | return -1; 484 | } 485 | 486 | rc = prepare_offset_by_key(handler, key, &offset, 0); 487 | if (rc == 0) { 488 | *value = (tsdb_value*)(handler->chunk.data + offset); 489 | } 490 | 491 | return rc ; 492 | } 493 | 494 | int tsdb_get_by_index(tsdb_handler *handler, u_int32_t *index, 495 | tsdb_value **value) { 496 | u_int64_t offset; 497 | int rc; 498 | 499 | if (!handler->alive || !handler->chunk.data) { 500 | return -1; 501 | } 502 | 503 | rc = prepare_offset_by_index(handler, index, &offset, 0); 504 | if (rc == 0) { 505 | *value = (tsdb_value*)(handler->chunk.data + offset); 506 | } 507 | 508 | return rc ; 509 | } 510 | 511 | void tsdb_flush(tsdb_handler *handler) { 512 | if (!handler->alive || handler->read_only) { 513 | return; 514 | } 515 | trace_info("Flushing database changes"); 516 | tsdb_flush_chunk(handler); 517 | handler->db->sync(handler->db, 0); 518 | } 519 | 520 | static int load_tag_array(tsdb_handler *handler, char *name, 521 | tsdb_tag *tag) { 522 | void *ptr; 523 | u_int32_t len; 524 | char str[255] = { 0 }; 525 | 526 | snprintf(str, sizeof(str), "tag-%s", name); 527 | 528 | if (db_get(handler, str, strlen(str), &ptr, &len) == 0) { 529 | u_int32_t *array; 530 | array = (u_int32_t*)malloc(len); 531 | if (array == NULL) { 532 | free(ptr); 533 | return -2; 534 | } 535 | memcpy(array, ptr, len); 536 | tag->array = array; 537 | tag->array_len = len; 538 | return 0; 539 | } 540 | 541 | return -1; 542 | } 543 | 544 | static int allocate_tag_array(tsdb_tag *tag) { 545 | u_int32_t array_len = CHUNK_GROWTH / sizeof(u_int32_t); 546 | 547 | u_int32_t* array = malloc(array_len); 548 | if (!array) { 549 | return -1; 550 | } 551 | 552 | memset(array, 0, array_len); 553 | 554 | tag->array = array; 555 | tag->array_len = array_len; 556 | 557 | return 0; 558 | } 559 | 560 | static void set_tag(tsdb_handler *handler, char *name, tsdb_tag *tag) { 561 | char str[255]; 562 | 563 | snprintf(str, sizeof(str), "tag-%s", name); 564 | 565 | db_put(handler, str, strlen(str), tag->array, tag->array_len); 566 | } 567 | 568 | static int ensure_tag_array(tsdb_handler *handler, char *name, tsdb_tag *tag) { 569 | if (load_tag_array(handler, name, tag) == 0) { 570 | return 0; 571 | } 572 | 573 | if (allocate_tag_array(tag) == 0) { 574 | return 0; 575 | } 576 | 577 | return -1; 578 | } 579 | 580 | int tsdb_tag_key(tsdb_handler *handler, char *key, char *tag_name) { 581 | u_int32_t index; 582 | 583 | if (tsdb_get_key_index(handler, key, &index) == -1) { 584 | return -1; 585 | } 586 | 587 | tsdb_tag tag; 588 | if (ensure_tag_array(handler, tag_name, &tag)) { 589 | return -1; 590 | } 591 | 592 | set_bit(tag.array, index); 593 | set_tag(handler, tag_name, &tag); 594 | 595 | free(tag.array); 596 | 597 | return 0; 598 | } 599 | 600 | void scan_tag_indexes(tsdb_tag *tag, u_int32_t *indexes, 601 | u_int32_t max_index, u_int32_t *count) { 602 | u_int32_t i, j, index; 603 | u_int32_t max_word = max_index / BITS_PER_WORD; 604 | 605 | *count = 0; 606 | 607 | for (i = 0; i <= max_word; i++) { 608 | if (tag->array[i] == 0) { 609 | continue; 610 | } 611 | for (j = 0; j < BITS_PER_WORD; j++) { 612 | index = i * BITS_PER_WORD + j; 613 | if (index > max_index) { 614 | break; 615 | } 616 | if (get_bit(tag->array, index)) { 617 | indexes[(*count)++] = index; 618 | } 619 | } 620 | } 621 | } 622 | 623 | static u_int32_t max_tag_index(tsdb_handler *handler, u_int32_t max_len) { 624 | if (handler->lowest_free_index < max_len) { 625 | return handler->lowest_free_index - 1; 626 | } else { 627 | return max_len - 1; 628 | } 629 | } 630 | 631 | int tsdb_get_tag_indexes(tsdb_handler *handler, char *tag_name, 632 | u_int32_t *indexes, u_int32_t indexes_len, 633 | u_int32_t *count) { 634 | tsdb_tag tag; 635 | if (load_tag_array(handler, tag_name, &tag) == 0) { 636 | u_int32_t max_index = max_tag_index(handler, indexes_len); 637 | scan_tag_indexes(&tag, indexes, max_index, count); 638 | free(tag.array); 639 | return 0; 640 | } 641 | 642 | return -1; 643 | } 644 | 645 | int tsdb_get_consolidated_tag_indexes(tsdb_handler *handler, 646 | char **tag_names, 647 | u_int16_t tag_names_len, 648 | int consolidator, 649 | u_int32_t *indexes, 650 | u_int32_t indexes_len, 651 | u_int32_t *count) { 652 | u_int32_t i, j, max_index; 653 | tsdb_tag consolidated, current; 654 | 655 | consolidated.array = NULL; 656 | consolidated.array_len = 0; 657 | max_index = max_tag_index(handler, indexes_len); 658 | 659 | *count = 0; 660 | 661 | for (i = 0; i < tag_names_len; i++) { 662 | if (load_tag_array(handler, tag_names[i], ¤t) == 0) { 663 | if (consolidated.array) { 664 | for (j = 0; j < max_index; j++) { 665 | switch (consolidator) { 666 | case TSDB_AND: 667 | consolidated.array[j] &= current.array[j]; 668 | break; 669 | case TSDB_OR: 670 | consolidated.array[j] |= current.array[j]; 671 | break; 672 | default: 673 | consolidated.array[j] = current.array[j]; 674 | } 675 | } 676 | } else { 677 | consolidated.array = current.array; 678 | consolidated.array_len = current.array_len; 679 | } 680 | } 681 | } 682 | 683 | if (consolidated.array) { 684 | scan_tag_indexes(&consolidated, indexes, max_index, count); 685 | free(consolidated.array); 686 | } 687 | 688 | return 0; 689 | } 690 | --------------------------------------------------------------------------------