├── Makefile ├── configuration.c ├── configuration.h ├── configuration_samples ├── check_for_persistence_n_dump_auditlog_id.conf └── dump_auditlog_id.conf ├── input_elements ├── filesystem-walker.c ├── filesystem-walker.h ├── input_batch.c ├── input_batch.h ├── read_from_filesystem.c └── read_from_filesystem.h ├── mlogc-ng-complete.conf.example ├── mlogc-ng.c ├── mlogc-ng.conf.example ├── mlogc-ng.h ├── pipe_elements ├── dump.c ├── dump.h ├── json-conversion.c ├── json-conversion.h ├── persistence.c ├── persistence.h ├── send_to_server.c └── send_to_server.h ├── pipeline.c └── pipeline.h /Makefile: -------------------------------------------------------------------------------- 1 | CC?=gcc 2 | CFLAGS+=-I. -Ipipe_elements/. -Iinput_elements/. -Wno-pointer-sign -Wno-variadic-macros -O0 -g 3 | LIBS+=-lyajl -lcurl -lssl -lcrypto 4 | 5 | all: mlogc-ng 6 | 7 | %.o: %.c 8 | $(CC) -c -o $@ $< $(CFLAGS) 9 | 10 | OBJS = \ 11 | configuration.o \ 12 | input_elements/read_from_filesystem.o \ 13 | input_elements/input_batch.o \ 14 | input_elements/filesystem-walker.o \ 15 | mlogc-ng.o \ 16 | pipeline.o \ 17 | pipe_elements/dump.o \ 18 | pipe_elements/persistence.o \ 19 | pipe_elements/send_to_server.o 20 | 21 | mlogc-ng: $(OBJS) 22 | $(CC) -o $@ $^ $(CFLAGS) $(LIBS) 23 | 24 | clean: 25 | rm -f *.o pipe_elements/*.o input_elements/*.o mlogc-ng 26 | -------------------------------------------------------------------------------- /configuration.c: -------------------------------------------------------------------------------- 1 | 2 | 3 | #include 4 | #include 5 | 6 | #include "mlogc-ng.h" 7 | #include "configuration.h" 8 | 9 | #include "yajl/yajl_tree.h" 10 | 11 | #include "pipeline.h" 12 | 13 | int read_configuration_file(const char *config_file, struct pipeline_t *p) 14 | { 15 | int res = 0; 16 | static unsigned char fileData[65536]; 17 | size_t rd; 18 | yajl_val node; 19 | char errbuf[1024]; 20 | FILE *fp; 21 | const char * path[] = { "mlogc_pipeline", (const char *) 0 }; 22 | size_t len; 23 | int i; 24 | 25 | fileData[0] = errbuf[0] = 0; 26 | 27 | /* read the entire config file */ 28 | fp = fopen(config_file, "r"); 29 | if (!fp) 30 | { 31 | res = -1; 32 | goto failed_to_open_config_file; 33 | } 34 | 35 | rd = fread((void *) fileData, 1, sizeof(fileData) - 1, fp); 36 | fclose(fp); 37 | 38 | /* we have the whole config file in memory. let's parse it ... */ 39 | node = yajl_tree_parse((const char *) fileData, errbuf, sizeof(errbuf)); 40 | 41 | if (node == NULL) { 42 | e("Config parser error: "); 43 | if (strlen(errbuf)) 44 | { 45 | e("%s", errbuf); 46 | } 47 | else 48 | { 49 | e("unknown error"); 50 | } 51 | 52 | e("\n"); 53 | res = -1; 54 | goto parser_error; 55 | } 56 | 57 | 58 | yajl_val v = yajl_tree_get( node, path, yajl_t_array ); 59 | if ( !v || !YAJL_IS_ARRAY( v ) ) { 60 | e("Bad configuration format\n"); 61 | res = -1; 62 | goto parser_error; 63 | } 64 | 65 | 66 | len = v->u.array.len; 67 | for ( i = 0; i < len; ++i ) { 68 | yajl_val obj = v->u.array.values[ i ]; 69 | size_t nelem = obj->u.object.len; 70 | int ii; 71 | for ( ii = 0; ii < nelem; ++ii ) { 72 | const char *n_path[] = {"directory",(const char *)0}; 73 | 74 | const char *key = obj->u.object.keys[ ii ]; 75 | yajl_val val = obj->u.object.values[ ii ]; 76 | d("Creating pipeline element: %s\n", key); 77 | struct pipeline_element_t *e = create_pipe_element(key, &val, p); 78 | if (e == NULL) 79 | { 80 | e("Problem creating pipe element: %s\n", key); 81 | res = -1; 82 | goto pipe_problem; 83 | } 84 | add_pipe_element(p, e); 85 | } 86 | } 87 | 88 | pipe_problem: 89 | parser_error: 90 | failed_to_open_config_file: 91 | failed_allocate_memory: 92 | return res; 93 | } 94 | 95 | -------------------------------------------------------------------------------- /configuration.h: -------------------------------------------------------------------------------- 1 | 2 | #include "mlogc-ng.h" 3 | #include "pipeline.h" 4 | 5 | 6 | #ifndef __CONFIGURATION_H__ 7 | #define __CONFIGURATION_H__ 8 | 9 | int read_configuration_file(const char *config_file, struct pipeline_t *p); 10 | 11 | #endif 12 | -------------------------------------------------------------------------------- /configuration_samples/check_for_persistence_n_dump_auditlog_id.conf: -------------------------------------------------------------------------------- 1 | { 2 | "mlogc_pipeline":[ 3 | { 4 | "read_from_filesystem":{ 5 | "directory":"/var/my/auditlogs/path" 6 | } 7 | }, 8 | { 9 | "check_persistence":{ 10 | "store_database":"/tmp/persistence-database/" 11 | } 12 | }, 13 | { 14 | "dump":{ 15 | "id": "true", 16 | "date": "false" 17 | } 18 | }, 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /configuration_samples/dump_auditlog_id.conf: -------------------------------------------------------------------------------- 1 | { 2 | "mlogc_pipeline":[ 3 | { 4 | "read_from_filesystem":{ 5 | "directory":"/var/my/auditlogs/path" 6 | } 7 | }, 8 | { 9 | "dump":{ 10 | "id": "true", 11 | "date": "false" 12 | } 13 | }, 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /input_elements/filesystem-walker.c: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "mlogc-ng.h" 9 | #include "input_batch.h" 10 | #include "read_from_filesystem.h" 11 | 12 | int inspect_file(unsigned char *full_path, void *audit_log_entry_cb, 13 | struct read_from_filesystem_config_t *conf) 14 | { 15 | int res = 0; 16 | FILE *fp; 17 | struct stat s; 18 | char *file_content_buf = NULL; 19 | struct audit_log_entry_t audit_log; 20 | void (*ptrEntry)(struct audit_log_entry_t *, 21 | struct read_from_filesystem_config_t *); 22 | ptrEntry = audit_log_entry_cb; 23 | 24 | memset(&audit_log, '\0', sizeof(audit_log)); 25 | 26 | d("inspecting: %s\n", full_path); 27 | 28 | res = stat(full_path, &s); 29 | if (res < 0) 30 | { 31 | e("Not able to get file size: %s\n", full_path); 32 | goto failed; 33 | } 34 | 35 | if (s.st_size > MAX_FILE_SIZE) 36 | { 37 | e("This file is too big for us. Limit is: %d bytes\n", MAX_FILE_SIZE); 38 | res = -1; 39 | goto failed; 40 | } 41 | 42 | file_content_buf = malloc(s.st_size + 1); 43 | if (file_content_buf == NULL) 44 | { 45 | e("Failed to allocate memory to read: %s\n", full_path); 46 | res = -1; 47 | goto failed; 48 | } 49 | 50 | /* Everything is ok, lets have the data inside a buffer. */ 51 | fp = fopen(full_path, "r"); 52 | if (fp == NULL) 53 | { 54 | e("Failed to open the file: %s\n", full_path); 55 | res = -1; 56 | goto failed_fp; 57 | } 58 | 59 | fread(file_content_buf, sizeof(char), s.st_size, fp); 60 | fclose(fp); 61 | 62 | audit_log.hostname = "-"; 63 | audit_log.remote_addr = "-"; 64 | audit_log.remote_user = "-"; 65 | audit_log.local_user = "-"; 66 | audit_log.logtime = "-"; 67 | audit_log.request = "-"; 68 | audit_log.response_status = "-"; 69 | audit_log.bytes_sent = "-"; 70 | audit_log.refer = "-"; 71 | audit_log.user_agent = "-"; 72 | audit_log.uniqueid = "-"; 73 | audit_log.sessionid = "-"; 74 | audit_log.audit_file = full_path; 75 | audit_log.extra = "0"; 76 | audit_log.md5 = "-"; 77 | audit_log.audit_size = "0"; 78 | 79 | if (load_buffer(file_content_buf, &audit_log) == 0) 80 | { 81 | (*ptrEntry)(&audit_log, conf); 82 | } 83 | 84 | failed_fp: 85 | free(file_content_buf); 86 | failed: 87 | return res; 88 | } 89 | 90 | int open_directory_recursive(const unsigned char *path, void *audit_log_entry_cb, 91 | struct read_from_filesystem_config_t *conf) 92 | { 93 | DIR *d; 94 | struct dirent *entry; 95 | int res = 0; 96 | 97 | d = opendir(path); 98 | if (d == NULL) 99 | { 100 | e("Failed to open directory: %s\n", path); 101 | res = -1; 102 | goto failed; 103 | } 104 | 105 | entry = readdir(d); 106 | while (entry != NULL) 107 | { 108 | const char *d_name = NULL; 109 | int full_path_len = 0; 110 | char *full_path = NULL; 111 | 112 | d_name = entry->d_name; 113 | 114 | full_path_len = snprintf(NULL, 0, "%s/%s", path, d_name); 115 | full_path = malloc(sizeof(char) * (full_path_len + 1)); 116 | if (full_path == NULL) 117 | { 118 | e("Failed: missing memory to continue.\n"); 119 | return -1; 120 | } 121 | 122 | full_path_len = snprintf(full_path, full_path_len + 1, "%s/%s", path, 123 | d_name); 124 | 125 | /* if it is a dir (different from "." or ".." we want to jump in. */ 126 | if (entry->d_type & DT_DIR) 127 | { 128 | if (strcmp (d_name, "..") != 0 && strcmp (d_name, ".") != 0) 129 | { 130 | /* FIXME: we should not ignore this return value. */ 131 | open_directory_recursive(full_path, audit_log_entry_cb, conf); 132 | } 133 | } 134 | 135 | /* if it is a file, let see inside. */ 136 | if (!(entry->d_type & DT_DIR)) 137 | { 138 | inspect_file(full_path, audit_log_entry_cb, conf); 139 | } 140 | 141 | free(full_path); 142 | 143 | entry = readdir(d); 144 | } 145 | 146 | failed_read_dir: 147 | closedir(d); 148 | failed: 149 | return res; 150 | } 151 | 152 | -------------------------------------------------------------------------------- /input_elements/filesystem-walker.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef __FILESYSTEM_WALKER_H__ 3 | #define __FILESYSTEM_WALKER_H__ 4 | 5 | #include "read_from_filesystem.h" 6 | 7 | int inspect_file(unsigned char *, void *); 8 | int open_directory_recursive(const unsigned char *, void *, 9 | struct read_from_filesystem_config_t *); 10 | 11 | #endif 12 | -------------------------------------------------------------------------------- /input_elements/input_batch.c: -------------------------------------------------------------------------------- 1 | 2 | #include "mlogc-ng.h" 3 | #include "input_batch.h" 4 | #include "filesystem-walker.h" 5 | #include "pipeline.h" 6 | 7 | #if defined(__APPLE__) 8 | 9 | #define COMMON_DIGEST_FOR_OPENSSL 10 | #include 11 | #define SHA1 CC_SHA1 12 | 13 | #else 14 | 15 | #include 16 | 17 | #endif 18 | 19 | 20 | char *md5sum(const char *str, int length) { 21 | int n; 22 | MD5_CTX c; 23 | unsigned char digest[16]; 24 | char *out = (char*)malloc(33); 25 | 26 | MD5_Init(&c); 27 | 28 | while (length > 0) { 29 | if (length > 512) { 30 | MD5_Update(&c, str, 512); 31 | } else { 32 | MD5_Update(&c, str, length); 33 | } 34 | length -= 512; 35 | str += 512; 36 | } 37 | 38 | MD5_Final(digest, &c); 39 | 40 | for (n = 0; n < 16; ++n) { 41 | snprintf(&(out[n*2]), 16*2, "%02x", (unsigned int)digest[n]); 42 | } 43 | 44 | return out; 45 | } 46 | 47 | int parse_section_a(struct audit_log_entry_t *a) 48 | { 49 | int start = 0; 50 | int count = 0; 51 | char *date = NULL; 52 | char *id = NULL; 53 | char *origin = NULL; 54 | char *dest = NULL; 55 | char *origin_port = NULL; 56 | char *dest_port = NULL; 57 | char *buf = a->section_a; 58 | 59 | /* Date */ 60 | while (strlen(buf) > count && buf[count] != ' ') 61 | { 62 | count++; 63 | } 64 | date = strndup(buf + 1, count - 1); 65 | a->logtime = date; 66 | 67 | 68 | while (strlen(buf) > count && buf[count] != ']') 69 | count++; 70 | 71 | /* ID */ 72 | count++; 73 | count++; 74 | start = count; 75 | while (strlen(buf) > count && buf[count] != ' ') 76 | { 77 | count++; 78 | } 79 | id = strndup(buf + start, count - start); 80 | a->uniqueid = id; 81 | 82 | /* origin */ 83 | count++; 84 | start = count; 85 | while (strlen(buf) > count && buf[count] != ' ') 86 | { 87 | count++; 88 | } 89 | 90 | origin = malloc(sizeof(char) * (count - start + 1)); 91 | memset(origin, '\0', count - start + 1); 92 | memcpy(origin, buf + start, count - start); 93 | a->remote_addr = origin; 94 | 95 | /* origin port */ 96 | count++; 97 | start = count; 98 | while (strlen(buf) > count && buf[count] != ' ') 99 | { 100 | count++; 101 | } 102 | 103 | origin_port = malloc(sizeof(char) * (count - start + 1)); 104 | memset(origin_port, '\0', count - start + 1); 105 | memcpy(origin_port, buf + start, count - start); 106 | 107 | /* dest */ 108 | count++; 109 | start = count; 110 | while (strlen(buf) > count && buf[count] != ' ') 111 | { 112 | count++; 113 | } 114 | 115 | dest = malloc(sizeof(char) * (count - start + 1)); 116 | memset(dest, '\0', count - start + 1); 117 | memcpy(dest, buf + start, count - start); 118 | 119 | /* dest port */ 120 | count++; 121 | start = count; 122 | while (strlen(buf) > count && buf[count] != ' ') 123 | { 124 | count++; 125 | } 126 | 127 | dest_port = malloc(sizeof(char) * (count - start + 1)); 128 | memset(dest_port, '\0', count - start + 1); 129 | memcpy(dest_port, buf + start, count - start); 130 | 131 | 132 | a->id = id; 133 | a->date = date; 134 | a->origin = origin; 135 | /* a->origin_port = atoi(origin_port); */ 136 | a->origin_port = origin_port; 137 | a->dest = dest; 138 | /* a->dest_port = atoi(dest_port); */ 139 | a->dest_port = dest_port; 140 | 141 | return 0; 142 | } 143 | 144 | int parse_section_f(struct audit_log_entry_t *audit_log) 145 | { 146 | char *buf = audit_log->section_f; 147 | char *line_start = buf; 148 | char *line_end = NULL; 149 | int z = 0; 150 | 151 | if (buf == NULL) 152 | { 153 | return 0; 154 | } 155 | 156 | line_end = strchr(buf, 10); 157 | 158 | while (line_end != NULL) 159 | { 160 | char *line = NULL; 161 | size_t len = line_end - line_start; 162 | 163 | line = strndup(line_start, len); 164 | 165 | // FIXME: check the sizes. 166 | z = z + strlen(line) + 1; 167 | line_start = line_end + 1; 168 | line_end = strchr(buf + z, 10); 169 | 170 | if (strncmp("HTTP", line, 4) == 0) 171 | { 172 | audit_log->response_status = strndup(strchr(line, 32) + 1, 3); 173 | } 174 | else if (strncasecmp(line, "Content-Length: ", 16) == 0) 175 | { 176 | audit_log->bytes_sent = strdup(line + 16); 177 | } 178 | } 179 | return 0; 180 | } 181 | 182 | int parse_section_b(struct audit_log_entry_t *audit_log) 183 | { 184 | char *buf = audit_log->section_b; 185 | char *line_start = buf; 186 | char *line_end = NULL; 187 | int z = 0; 188 | 189 | line_end = strchr(buf, 10); 190 | 191 | while (line_end != NULL) 192 | { 193 | char *line = NULL; 194 | size_t len = line_end - line_start; 195 | 196 | line = strndup(line_start, len); 197 | 198 | // FIXME: check the sizes. 199 | z = z + strlen(line) + 1; 200 | line_start = line_end + 1; 201 | line_end = strchr(buf + z, 10); 202 | 203 | if (strncmp("GET ", line, 4) == 0) 204 | { 205 | audit_log->request = strndup(line + 4, strchr(line, 32) - line); 206 | } 207 | else if (strncmp("POST ", line, 5) == 0) 208 | { 209 | audit_log->request = strndup(line + 5, strchr(line, 32) - line); 210 | } 211 | else if (strncmp("HEAD ", line, 5) == 0) 212 | { 213 | audit_log->request = strndup(line + 5, strchr(line, 32) - line); 214 | } 215 | else if (strncasecmp(line, "refer: ", 7) == 0) 216 | { 217 | audit_log->refer = strdup(line + 7); 218 | } 219 | else if (strncasecmp(line, "host: ", 6) == 0) 220 | { 221 | audit_log->hostname = strdup(line + 6); 222 | } 223 | else if (strncasecmp(line, "user-agent: ", 12) == 0) 224 | { 225 | audit_log->user_agent = strdup(line + 12); 226 | } 227 | } 228 | return 0; 229 | } 230 | 231 | char *load_section(const char section, struct audit_log_entry_t *a) 232 | { 233 | int count = 0; 234 | int start = 0; 235 | int end = 0; 236 | char *section_buf = NULL; 237 | 238 | char *marking = NULL; 239 | 240 | marking = malloc(sizeof(char) * (strlen(a->mark) + 1 + 4)); 241 | memset(marking, '\0', strlen(a->mark) + 1 + 4); 242 | snprintf(marking, strlen(a->mark) + 4, "%s%c--", a->mark, section); 243 | 244 | // Find the start point. 245 | while (a->buf_len > count + strlen(marking) && 246 | strncmp(a->buf + count, marking, strlen(marking)) != 0) 247 | { 248 | count++; 249 | } 250 | 251 | if (!(strlen(a->buf + count) > strlen(marking))) 252 | { 253 | return NULL; 254 | } 255 | 256 | start = count + strlen(marking) + 1; 257 | 258 | // Find the end of the section 259 | count = start; 260 | while (a->buf_len > count + strlen(a->mark) && 261 | strncmp(a->buf + count, a->mark, strlen(a->mark)) != 0) 262 | { 263 | count++; 264 | } 265 | 266 | if (!(a->buf_len > count + strlen(a->mark))) 267 | { 268 | return NULL; 269 | } 270 | 271 | end = count; 272 | 273 | section_buf = malloc(sizeof(char) * (end - start + 1)); 274 | memset(section_buf, '\0', end - start + 1); 275 | strncpy(section_buf, a->buf + start, end - start); 276 | 277 | return section_buf; 278 | } 279 | 280 | int load_buffer(char *buf, struct audit_log_entry_t *audit_log) 281 | { 282 | char *mark = NULL; 283 | int res = 0; 284 | int count = 0; 285 | int buf_len = strlen(buf); 286 | 287 | /* finding the separator */ 288 | while (buf_len > count && buf[count] != '\n') 289 | { 290 | count++; 291 | } 292 | 293 | if (buf[count] != '\n') 294 | { 295 | e("Not an valid audit log entry"); 296 | res = -1; 297 | goto not_valid_audit_log; 298 | } 299 | 300 | mark = malloc(sizeof(char) * (count - 3 + 1)); 301 | memset(mark, '\0', count - 3 + 1); 302 | mark = strncpy(mark, buf, count - 3); 303 | 304 | char number[20]; 305 | memset(number, '\0', 20); 306 | snprintf(number, 20, "%lu", strlen(buf)); 307 | 308 | audit_log->mark = mark; 309 | audit_log->buf = buf; 310 | audit_log->buf_len = strlen(buf); 311 | audit_log->audit_size = strdup(number); 312 | audit_log->md5 = md5sum(buf, audit_log->buf_len); 313 | audit_log->uploaded = 0; 314 | 315 | audit_log->section_a = load_section('A', audit_log); 316 | audit_log->section_b = load_section('B', audit_log); 317 | audit_log->section_f = load_section('F', audit_log); 318 | audit_log->section_e = load_section('E', audit_log); 319 | audit_log->section_h = load_section('H', audit_log); 320 | 321 | res = parse_section_a(audit_log); 322 | res = parse_section_b(audit_log); 323 | res = parse_section_f(audit_log); 324 | 325 | size_t a = strlen(audit_log->hostname) + 1 + 326 | strlen(audit_log->remote_addr) + 1 + 327 | strlen(audit_log->remote_user) + 1 + 328 | strlen(audit_log->local_user) + 1 + 329 | strlen(audit_log->logtime) + 1 + 330 | strlen(audit_log->request) + 1 + 2 + 331 | strlen(audit_log->response_status) + 1 + 332 | strlen(audit_log->bytes_sent) + 1 + 333 | strlen(audit_log->refer) + 1 + 2 + 334 | strlen(audit_log->user_agent) + 1 + 2 + 335 | strlen(audit_log->uniqueid) + 1 + 336 | strlen(audit_log->sessionid) + 1 + 2 + 337 | strlen(audit_log->audit_file) + 1 + 338 | strlen(audit_log->extra) + 1 + 339 | strlen(audit_log->md5) + 1 + 4 + 340 | strlen(audit_log->audit_size) + 1; 341 | 342 | audit_log->summary_line = malloc(a + 1); 343 | memset(audit_log->summary_line, '\0', a + 1); 344 | 345 | snprintf(audit_log->summary_line, a, "%s %s %s %s %s \"%s\" %s %s \"%s\" \"%s\" %s \"%s\" %s %s %s md5:%s\n", 346 | audit_log->hostname, 347 | audit_log->remote_addr, 348 | audit_log->remote_user, 349 | audit_log->local_user, 350 | audit_log->logtime, 351 | audit_log->request, 352 | audit_log->response_status, 353 | audit_log->bytes_sent, 354 | audit_log->refer, 355 | audit_log->user_agent, 356 | audit_log->uniqueid, 357 | audit_log->sessionid, 358 | audit_log->audit_file, 359 | audit_log->extra, 360 | audit_log->audit_size, 361 | audit_log->md5); 362 | 363 | not_valid_audit_log: 364 | return res; 365 | } 366 | 367 | -------------------------------------------------------------------------------- /input_elements/input_batch.h: -------------------------------------------------------------------------------- 1 | 2 | #include "mlogc-ng.h" 3 | 4 | #ifndef __INPUT_BATCH_H__ 5 | #define __INPUT_BATCH_H__ 6 | 7 | int load_buffer(char *, struct audit_log_entry_t *); 8 | 9 | #endif 10 | -------------------------------------------------------------------------------- /input_elements/read_from_filesystem.c: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | #include 5 | #include "yajl/yajl_tree.h" 6 | 7 | #include "mlogc-ng.h" 8 | #include "configuration.h" 9 | 10 | #include "read_from_filesystem.h" 11 | #include "filesystem-walker.h" 12 | 13 | 14 | int read_from_filesystem_call_entry_cb(struct audit_log_entry_t *audit_log, 15 | struct read_from_filesystem_config_t *conf) 16 | { 17 | pipeline_process(conf->pipeline, audit_log); 18 | return 0; 19 | } 20 | 21 | 22 | struct read_from_filesystem_config_t *create_read_from_filesystem_instance( 23 | const yajl_val *val, struct pipeline_t *p) 24 | { 25 | const char *n_path[] = {"directory",(const char *)0}; 26 | struct read_from_filesystem_config_t *inst = malloc(sizeof( 27 | struct read_from_filesystem_config_t)); 28 | 29 | yajl_val n = yajl_tree_get(*val, n_path, yajl_t_string); 30 | 31 | inst->directory = YAJL_GET_STRING(n); 32 | inst->pipeline = p; 33 | 34 | d("Read file system element was created!\n"); 35 | d("Reading data from: %s\n", inst->directory); 36 | 37 | return inst; 38 | } 39 | 40 | 41 | int read_from_filesystem_process(struct pipeline_element_t *e, 42 | struct audit_log_entry_t *audit_log) 43 | { 44 | int res = 0; 45 | struct read_from_filesystem_config_t *conf = e->payload; 46 | 47 | d("Starting batch operation at the directory: %s\n", conf->directory); 48 | 49 | res = open_directory_recursive(conf->directory, 50 | read_from_filesystem_call_entry_cb, conf); 51 | 52 | if (res) 53 | { 54 | res = -1; 55 | e("Failed to open the directory: '%s' or one of its sub-directories\n", 56 | conf->directory); 57 | goto failed_opening_directory; 58 | } 59 | 60 | failed_opening_directory: 61 | return 0; 62 | } 63 | 64 | 65 | -------------------------------------------------------------------------------- /input_elements/read_from_filesystem.h: -------------------------------------------------------------------------------- 1 | 2 | #include "yajl/yajl_tree.h" 3 | 4 | #include "mlogc-ng.h" 5 | #include "configuration.h" 6 | 7 | #ifndef __READ_FROM_FILESYSTEM_H__ 8 | #define __READ_FROM_FILESYSTEM_H__ 9 | 10 | struct read_from_filesystem_config_t 11 | { 12 | const char *directory; 13 | struct pipeline_t *pipeline; 14 | }; 15 | 16 | struct read_from_filesystem_config_t *create_read_from_filesystem_instance( 17 | const yajl_val *, struct pipeline_t *); 18 | 19 | int read_from_filesystem_process(struct pipeline_element_t *e, 20 | struct audit_log_entry_t *); 21 | 22 | #endif 23 | -------------------------------------------------------------------------------- /mlogc-ng-complete.conf.example: -------------------------------------------------------------------------------- 1 | { 2 | "mlogc_pipeline":[ 3 | { 4 | "read_from_filesystem":{ 5 | "directory":"/tmp/logs" 6 | } 7 | }, 8 | { 9 | "check_persistence":{ 10 | "store_database":"/tmp/logs-database/" 11 | } 12 | }, 13 | { 14 | "transform_to_json":{ 15 | 16 | } 17 | }, 18 | { 19 | "send_to_server":{ 20 | "host":"https://server1", 21 | "username":"test", 22 | "password":"test", 23 | "validate_certificate_chain":"true", 24 | "number_of_clients": 10 25 | } 26 | }, 27 | { 28 | "send_to_server":{ 29 | "host":"https://server2", 30 | "username":"test", 31 | "password":"test", 32 | "validate_certificate_chain":"true" 33 | } 34 | } 35 | ] 36 | } 37 | -------------------------------------------------------------------------------- /mlogc-ng.c: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | #include 5 | 6 | #include "mlogc-ng.h" 7 | #include "configuration.h" 8 | #include "input_batch.h" 9 | #include "pipeline.h" 10 | 11 | /* 12 | * mlogc-ng 13 | * 14 | * TODO: write something about it. 15 | * 16 | */ 17 | 18 | #ifdef VERBOSE 19 | int verbose = 1; 20 | #else 21 | int verbose = 0; 22 | #endif 23 | 24 | #ifdef DEBUG 25 | int debug = 1; 26 | #else 27 | int debug = 0; 28 | #endif 29 | 30 | void help (void) 31 | { 32 | p("Use: ./mgloc-ng /path/to/config/file\n\n"); 33 | } 34 | 35 | int main (int argc, char **argv) 36 | { 37 | int res = 0; 38 | char *config_file = NULL; 39 | struct pipeline_t pipeline; 40 | struct pipeline_element_t *input; 41 | 42 | if (argc < 2) 43 | { 44 | help(); 45 | goto missing_arguments; 46 | } 47 | 48 | config_file = argv[1]; 49 | res = read_configuration_file(config_file, &pipeline); 50 | if (res) 51 | { 52 | e("Problems reading the configuration " \ 53 | "file: %s\n", config_file); 54 | goto bad_configuration; 55 | } 56 | 57 | input = pipeline.elements; 58 | if (input == NULL) 59 | { 60 | e("Missing pipeline.\n"); 61 | goto bad_configuration; 62 | } 63 | 64 | p("Starting pipeline, input element: '%s'\n", input->name); 65 | 66 | input->process(input, NULL); 67 | 68 | bad_configuration: 69 | missing_arguments: 70 | return res; 71 | } 72 | 73 | -------------------------------------------------------------------------------- /mlogc-ng.conf.example: -------------------------------------------------------------------------------- 1 | { 2 | "mlogc_pipeline":[ 3 | { 4 | "read_from_filesystem":{ 5 | "directory":"/tmp/audit-logs" 6 | } 7 | }, 8 | { 9 | "send_to_server":{ 10 | "host":"https://localhost:8443/rpc/auditLogReceiver", 11 | "username":"test", 12 | "password":"test", 13 | "validate_certificate_chain":"false", 14 | "number_of_clients": 10 15 | } 16 | }, 17 | { 18 | "dump": { 19 | "id": "true", 20 | "audit_file": "true" 21 | } 22 | } 23 | ] 24 | } 25 | -------------------------------------------------------------------------------- /mlogc-ng.h: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | #include 5 | 6 | 7 | #ifndef __MLOGC_NG_H__ 8 | #define __MLOGC_NG_H__ 9 | 10 | struct audit_log_entry_t 11 | { 12 | /* buf len */ 13 | char *buf; 14 | size_t buf_len; 15 | 16 | char *id; 17 | char *date; 18 | char *origin; 19 | char *origin_port; 20 | char *dest; 21 | char *dest_port; 22 | 23 | /* Sections */ 24 | char *mark; 25 | char *section_a; 26 | char *section_b; 27 | char *section_f; 28 | char *section_e; 29 | char *section_h; 30 | 31 | /* summary line */ 32 | char *summary_line; 33 | char *hostname; 34 | char *remote_addr; 35 | char *remote_user; 36 | char *local_user; 37 | char *logtime; 38 | char *request; 39 | char *response_status; 40 | char *bytes_sent; 41 | char *refer; 42 | char *user_agent; 43 | char *uniqueid; 44 | char *sessionid; 45 | char *audit_file; 46 | char *extra; 47 | char *audit_size; 48 | char *md5; 49 | 50 | /* internal */ 51 | size_t uploaded; 52 | }; 53 | 54 | /* Verbose, Debug and Print definitions */ 55 | extern int verbose; 56 | extern int debug; 57 | 58 | /* FIXME: Check if that value makes sense. Use bytes here. */ 59 | #define MAX_FILE_SIZE 1024*1024 60 | 61 | 62 | #ifndef WIN32 63 | 64 | #define d(fmt, ARGS...) do { if (debug) printf("%s:%d:%s(): " fmt, __FILE__, \ 65 | __LINE__, __func__, ## ARGS); } while (0) 66 | 67 | #define v(fmt, ARGS...) do { if (verbose) printf("%s:%d:%s(): " fmt, __FILE__, \ 68 | __LINE__, __func__, ## ARGS); } while (0) 69 | 70 | #define e(fmt, ARGS...) do { fprintf(stderr, "%s:%d:%s(): " fmt, __FILE__, \ 71 | __LINE__, __func__, ## ARGS); } while (0) 72 | 73 | 74 | #define p(fmt, ARGS...) do { printf(fmt, ## ARGS); } while (0) 75 | 76 | #else 77 | 78 | #define p printf 79 | #define e printf 80 | #define v if (verbose) printf 81 | #define d if (debug) printf 82 | 83 | #endif 84 | 85 | #endif 86 | -------------------------------------------------------------------------------- /pipe_elements/dump.c: -------------------------------------------------------------------------------- 1 | 2 | #include "mlogc-ng.h" 3 | #include "configuration.h" 4 | #include "dump.h" 5 | 6 | int dump_process (struct pipeline_element_t *e, 7 | struct audit_log_entry_t *audit_log) 8 | { 9 | struct dump_config_t *conf = e->payload; 10 | 11 | if (conf->id == 1) 12 | { 13 | p("ID: %s\n", audit_log->uniqueid); 14 | } 15 | 16 | if (conf->date == 1) 17 | { 18 | p("DATE: %s\n", audit_log->date); 19 | } 20 | 21 | if (conf->dest == 1) 22 | { 23 | p("DEST: %s\n", audit_log->dest); 24 | } 25 | 26 | if (conf->audit_file == 1) 27 | { 28 | p("FILE: %s\n", audit_log->audit_file); 29 | } 30 | 31 | return 0; 32 | } 33 | 34 | struct dump_config_t *create_dump_instance(const yajl_val *val, 35 | struct pipeline_t *p) 36 | { 37 | const char *n_path_id[] = {"id",(const char *)0}; 38 | const char *n_path_date[] = {"date",(const char *)0}; 39 | const char *n_path_dest[] = {"dest",(const char *)0}; 40 | const char *n_path_audit_file[] = {"audit_file",(const char *)0}; 41 | char *id, *date, *dest, *audit_file; 42 | 43 | struct dump_config_t *inst = malloc(sizeof(struct dump_config_t)); 44 | 45 | inst->id = 0; 46 | inst->date = 0; 47 | inst->dest = 0; 48 | inst->audit_file = 0; 49 | 50 | yajl_val id_ = yajl_tree_get(*val, n_path_id, yajl_t_string); 51 | id = YAJL_GET_STRING(id_); 52 | yajl_val date_ = yajl_tree_get(*val, n_path_date, yajl_t_string); 53 | date = YAJL_GET_STRING(date_); 54 | yajl_val dest_ = yajl_tree_get(*val, n_path_dest, yajl_t_string); 55 | dest = YAJL_GET_STRING(dest_); 56 | yajl_val audit_file_ = yajl_tree_get(*val, n_path_audit_file, yajl_t_string); 57 | audit_file = YAJL_GET_STRING(audit_file_); 58 | 59 | if (id && strcmp("true", id) == 0) 60 | { 61 | inst->id = 1; 62 | } 63 | if (date && strcmp("true", date) == 0) 64 | { 65 | inst->date = 1; 66 | } 67 | if (dest && strcmp("true", dest) == 0) 68 | { 69 | inst->dest = 1; 70 | } 71 | if (audit_file && strcmp("true", audit_file) == 0) 72 | { 73 | inst->audit_file = 1; 74 | } 75 | 76 | 77 | return inst; 78 | } 79 | 80 | -------------------------------------------------------------------------------- /pipe_elements/dump.h: -------------------------------------------------------------------------------- 1 | 2 | 3 | #include "mlogc-ng.h" 4 | #include "configuration.h" 5 | 6 | #ifndef __DUMP_H__ 7 | #define __DUMP_H__ 8 | 9 | struct dump_config_t { 10 | int id; 11 | int date; 12 | int dest; 13 | int audit_file; 14 | }; 15 | 16 | int dump_process (struct pipeline_element_t *, struct audit_log_entry_t *); 17 | struct dump_config_t *create_dump_instance(const yajl_val *, 18 | struct pipeline_t *); 19 | 20 | #endif 21 | -------------------------------------------------------------------------------- /pipe_elements/json-conversion.c: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | int dump_audit_log(struct audit_log_entry_t *a) 5 | { 6 | yajl_gen g; 7 | const unsigned char * buf; 8 | size_t len; 9 | 10 | g = yajl_gen_alloc(NULL); 11 | yajl_gen_config(g, yajl_gen_beautify, 1); 12 | 13 | yajl_gen_array_open(g); 14 | 15 | yajl_gen_map_open(g); 16 | 17 | yajl_gen_string(g, "id", strlen("id")); 18 | yajl_gen_string(g, a->id, strlen(a->id)); 19 | 20 | yajl_gen_string(g, "date", strlen("date")); 21 | yajl_gen_string(g, a->date, strlen(a->date)); 22 | 23 | yajl_gen_string(g, "origin", strlen("origin")); 24 | yajl_gen_map_open(g); 25 | yajl_gen_string(g, "ip", strlen("ip")); 26 | yajl_gen_string(g, a->origin, strlen(a->origin)); 27 | yajl_gen_string(g, "port", strlen("port")); 28 | yajl_gen_number(g, a->origin_port, strlen(a->origin_port)); 29 | yajl_gen_map_close(g); 30 | 31 | yajl_gen_string(g, "dest", strlen("dest")); 32 | yajl_gen_map_open(g); 33 | yajl_gen_string(g, "ip", strlen("ip")); 34 | yajl_gen_string(g, a->dest, strlen(a->dest)); 35 | yajl_gen_string(g, "port", strlen("port")); 36 | yajl_gen_number(g, a->dest_port, strlen(a->dest_port)); 37 | yajl_gen_map_close(g); 38 | 39 | if (a->section_a != NULL) 40 | { 41 | yajl_gen_string(g, "Section A", strlen("Section A")); 42 | yajl_gen_string(g, a->section_a, strlen(a->section_a)); 43 | } 44 | 45 | if (a->section_b != NULL) 46 | { 47 | yajl_gen_string(g, "Section B", strlen("Section B")); 48 | yajl_gen_string(g, a->section_b, strlen(a->section_b)); 49 | } 50 | 51 | if (a->section_f != NULL) 52 | { 53 | yajl_gen_string(g, "Section F", strlen("Section F")); 54 | yajl_gen_string(g, a->section_f, strlen(a->section_f)); 55 | } 56 | 57 | if (a->section_e != NULL) 58 | { 59 | yajl_gen_string(g, "Section E", strlen("Section E")); 60 | yajl_gen_string(g, a->section_e, strlen(a->section_e)); 61 | } 62 | 63 | if (a->section_h != NULL) 64 | { 65 | yajl_gen_string(g, "Section H", strlen("Section H")); 66 | yajl_gen_string(g, a->section_h, strlen(a->section_h)); 67 | } 68 | 69 | yajl_gen_map_close(g); 70 | 71 | yajl_gen_array_close(g); 72 | 73 | yajl_gen_get_buf(g, &buf, &len); 74 | fwrite(buf, 1, len, stdout); 75 | 76 | yajl_gen_free(g); 77 | 78 | #if 0 79 | fprintf(stdout, "Section A: %s\n", a->section_a); 80 | fprintf(stdout, "Section B: %s\n", a->section_b); 81 | fprintf(stdout, "Section F: %s\n", a->section_f); 82 | fprintf(stdout, "Section E: %s\n", a->section_e); 83 | fprintf(stdout, "Section H: %s\n", a->section_h); 84 | #endif 85 | 86 | } 87 | 88 | 89 | char *load_section(const char section, struct audit_log_entry_t *a) 90 | { 91 | int count = 0; 92 | int start = 0; 93 | int end = 0; 94 | char *section_buf = NULL; 95 | 96 | char *marking = NULL; 97 | 98 | marking = malloc(sizeof(char) * (strlen(a->mark) + 1 + 4)); 99 | memset(marking, '\0', strlen(a->mark) + 1 + 4); 100 | snprintf(marking, strlen(a->mark) + 4, "%s%c--", a->mark, section); 101 | 102 | /* Find the start point. */ 103 | while (a->buf_len > count + strlen(marking) && 104 | strncmp(a->buf + count, marking, strlen(marking)) != 0) 105 | { 106 | count++; 107 | } 108 | 109 | if (!(strlen(a->buf + count) > strlen(marking))) 110 | { 111 | return NULL; 112 | } 113 | 114 | start = count + strlen(marking) + 1; 115 | 116 | /* Find the end of the section */ 117 | count = start; 118 | while (a->buf_len > count + strlen(a->mark) && 119 | strncmp(a->buf + count, a->mark, strlen(a->mark)) != 0) 120 | { 121 | count++; 122 | } 123 | 124 | if (!(a->buf_len > count + strlen(a->mark))) 125 | { 126 | return NULL; 127 | } 128 | 129 | end = count; 130 | 131 | section_buf = malloc(sizeof(char) * (end - start + 1)); 132 | memset(section_buf, '\0', end - start + 1); 133 | strncpy(section_buf, a->buf + start, end - start); 134 | 135 | return section_buf; 136 | } 137 | 138 | int parse_section_a(unsigned char *buf, struct audit_log_entry_t *a) 139 | { 140 | int start = 0; 141 | int count = 0; 142 | char *date = NULL; 143 | char *id = NULL; 144 | char *origin = NULL; 145 | char *dest = NULL; 146 | char *origin_port = NULL; 147 | char *dest_port = NULL; 148 | int buf_len = strlen(buf); 149 | 150 | /* Date */ 151 | while (buf_len > count && buf[count] != ']') 152 | { 153 | count++; 154 | } 155 | 156 | date = malloc(sizeof(char) * (count - 1 + 1)); 157 | memset(date, '\0', count - 1 + 1); 158 | memcpy(date, buf + 1, count - 1); 159 | 160 | /* ID */ 161 | count++; 162 | count++; 163 | start = count; 164 | while (buf_len > count && buf[count] != ' ') 165 | { 166 | count++; 167 | } 168 | 169 | id = malloc(sizeof(char) * (count - start + 1)); 170 | memset(id, '\0', count - start + 1); 171 | memcpy(id, buf + start, count - start); 172 | 173 | /* origin */ 174 | count++; 175 | start = count; 176 | while (buf_len > count && buf[count] != ' ') 177 | { 178 | count++; 179 | } 180 | 181 | origin = malloc(sizeof(char) * (count - start + 1)); 182 | memset(origin, '\0', count - start + 1); 183 | memcpy(origin, buf + start, count - start); 184 | 185 | /* origin port */ 186 | count++; 187 | start = count; 188 | while (buf_len > count && buf[count] != ' ') 189 | { 190 | count++; 191 | } 192 | 193 | origin_port = malloc(sizeof(char) * (count - start + 1)); 194 | memset(origin_port, '\0', count - start + 1); 195 | memcpy(origin_port, buf + start, count - start); 196 | 197 | /* dest */ 198 | count++; 199 | start = count; 200 | while (buf_len > count && buf[count] != ' ') 201 | { 202 | count++; 203 | } 204 | 205 | dest = malloc(sizeof(char) * (count - start + 1)); 206 | memset(dest, '\0', count - start + 1); 207 | memcpy(dest, buf + start, count - start); 208 | 209 | /* dest port */ 210 | count++; 211 | start = count; 212 | while (buf_len > count && buf[count] != ' ') 213 | { 214 | count++; 215 | } 216 | 217 | dest_port = malloc(sizeof(char) * (count - start + 1)); 218 | memset(dest_port, '\0', count - start + 1); 219 | memcpy(dest_port, buf + start, count - start); 220 | 221 | 222 | a->id = id; 223 | a->date = date; 224 | a->origin = origin; 225 | a->origin_port = origin_port; 226 | a->dest = dest; 227 | a->dest_port = dest_port; 228 | 229 | } 230 | 231 | int load_buffer(unsigned char *buf, struct audit_log_entry_t *audit_log) 232 | { 233 | int count = 0; 234 | int buf_len = strlen(buf); 235 | char *mark = NULL; 236 | 237 | audit_log->section_a = NULL; 238 | audit_log->section_b = NULL; 239 | audit_log->section_f = NULL; 240 | audit_log->section_e = NULL; 241 | audit_log->section_h = NULL; 242 | 243 | /* finding the separator */ 244 | while (buf_len > count && buf[count] != '\n') 245 | { 246 | count++; 247 | } 248 | 249 | if (buf[count] != '\n') 250 | { 251 | fprintf(stderr, "Not an valid audit log entry"); 252 | return -1; 253 | } 254 | 255 | mark = malloc(sizeof(char) * (count - 3 + 1)); 256 | memset(mark, '\0', count - 3 + 1); 257 | mark = strncpy(mark, buf, count - 3); 258 | 259 | audit_log->mark = mark; 260 | audit_log->buf = buf; 261 | audit_log->buf_len = buf_len; 262 | 263 | audit_log->section_a = load_section('A', audit_log); 264 | audit_log->section_b = load_section('B', audit_log); 265 | audit_log->section_f = load_section('F', audit_log); 266 | audit_log->section_e = load_section('E', audit_log); 267 | audit_log->section_h = load_section('H', audit_log); 268 | 269 | parse_section_a(audit_log->section_a, audit_log); 270 | 271 | return 0; 272 | } 273 | 274 | int clean_audit_log_entry(struct audit_log_entry_t *audit_log) 275 | { 276 | free(audit_log->mark); 277 | 278 | if (audit_log->section_a) 279 | free(audit_log->section_a); 280 | if (audit_log->section_a) 281 | free(audit_log->section_b); 282 | if (audit_log->section_a) 283 | free(audit_log->section_f); 284 | if (audit_log->section_a) 285 | free(audit_log->section_e); 286 | if (audit_log->section_a) 287 | free(audit_log->section_h); 288 | } 289 | 290 | 291 | -------------------------------------------------------------------------------- /pipe_elements/json-conversion.h: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | 5 | #ifndef __JSON_CONVERSION_H__ 6 | #define __JSON_CONVERSION_H__ 7 | 8 | struct audit_log_entry_t 9 | { 10 | char *id; 11 | char *date; 12 | char *origin; 13 | char *origin_port; 14 | char *dest; 15 | char *dest_port; 16 | 17 | char *section_a; 18 | char *section_b; 19 | char *section_f; 20 | char *section_e; 21 | char *section_h; 22 | 23 | char *mark; 24 | char *buf; 25 | int buf_len; 26 | 27 | }; 28 | 29 | #endif 30 | -------------------------------------------------------------------------------- /pipe_elements/persistence.c: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | #include 5 | 6 | #include "mlogc-ng.h" 7 | #include "configuration.h" 8 | #include "persistence.h" 9 | #include "pipeline.h" 10 | 11 | int persistence_file_exists(const char *file) 12 | { 13 | struct stat buffer; 14 | return (stat (file, &buffer) == 0); 15 | } 16 | 17 | int persistence_process (struct pipeline_element_t *e, 18 | struct audit_log_entry_t *audit_log) 19 | { 20 | int res = 0; 21 | char *id = audit_log->id; 22 | char paths[6]; 23 | struct persistence_config_t *conf = e->payload; 24 | 25 | size_t file_size = strlen(id) + strlen(conf->directory) + 6 + 1; 26 | char *file = malloc(file_size); 27 | snprintf(file, file_size, 28 | "%s/%.4s/%.4s/%.4s/%.4s/%.4s/%.4s", conf->directory, 29 | id, id + 4, id + 8, id + 12, id + 16, id + 20); 30 | 31 | res = persistence_file_exists(file); 32 | 33 | if (!res || 1 == 1) 34 | { 35 | int i = 0; 36 | snprintf(file, file_size, "%s", conf->directory); 37 | 38 | for (i = 0; i < 5; i++) 39 | { 40 | snprintf(file + strlen(file), file_size - strlen(file), "/%.4s", 41 | id + (i*4)); 42 | mkdir(file, 0777); 43 | } 44 | snprintf(file + strlen(file), file_size - strlen(file), "/%.4s", 45 | id + 20); 46 | FILE *fd = fopen(file, "w"); 47 | fprintf(fd, "."); 48 | fclose(fd); 49 | 50 | goto end; 51 | } 52 | 53 | end: 54 | free (file); 55 | return res; 56 | } 57 | 58 | struct persistence_config_t *create_persistence_instance(const yajl_val *val, 59 | struct pipeline_t *p) 60 | { 61 | const char *n_path[] = {"directory",(const char *)0}; 62 | struct persistence_config_t *inst = malloc(sizeof( 63 | struct persistence_config_t)); 64 | 65 | yajl_val n = yajl_tree_get(*val, n_path, yajl_t_string); 66 | 67 | inst->directory = YAJL_GET_STRING(n); 68 | 69 | return inst; 70 | } 71 | 72 | -------------------------------------------------------------------------------- /pipe_elements/persistence.h: -------------------------------------------------------------------------------- 1 | 2 | 3 | #include "mlogc-ng.h" 4 | #include "configuration.h" 5 | 6 | #ifndef __PERSISTENCE_H__ 7 | #define __PERSISTENCE_H__ 8 | 9 | struct persistence_config_t { 10 | char *directory; 11 | }; 12 | 13 | struct persistence_config_t *create_persistence_instance(const yajl_val *, 14 | struct pipeline_t *); 15 | 16 | int persistence_init (struct configuration_t *config); 17 | int persistence_process (struct pipeline_element_t *, 18 | struct audit_log_entry_t *); 19 | 20 | #endif 21 | -------------------------------------------------------------------------------- /pipe_elements/send_to_server.c: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | 4 | #include "mlogc-ng.h" 5 | #include "configuration.h" 6 | #include "send_to_server.h" 7 | 8 | 9 | 10 | static size_t send_to_server_read_cb (void *ptr, size_t size, size_t nmemb, void *data) 11 | { 12 | struct audit_log_entry_t *audit = (struct audit_log_entry_t *)data; 13 | size_t left = strlen(audit->buf) - audit->uploaded; 14 | size_t max_chunk = size * nmemb; 15 | size_t retcode = left < max_chunk ? left : max_chunk; 16 | 17 | memcpy(ptr, audit->buf + audit->uploaded, retcode); 18 | 19 | audit->uploaded += retcode; 20 | 21 | return retcode; 22 | } 23 | 24 | size_t sent_to_server_write_cb(char *ptr, size_t size, size_t nmemb, void *userdata) 25 | { 26 | return size*nmemb; 27 | } 28 | 29 | 30 | int send_to_server_process (struct pipeline_element_t *e, 31 | struct audit_log_entry_t *audit_log) 32 | { 33 | struct send_to_server_config_t *conf = e->payload; 34 | CURL *curl; 35 | CURLcode res; 36 | 37 | curl_global_init(CURL_GLOBAL_DEFAULT); 38 | 39 | curl = curl_easy_init(); 40 | 41 | if (curl) 42 | { 43 | char *credentials = malloc((strlen(conf->username) + 44 | strlen(conf->password) + 1 + 1) * sizeof(char)); 45 | char *summary = malloc(strlen("X-ForensicLog-Summary: ") + strlen(audit_log->summary_line) + 1); 46 | struct curl_slist *headerlist = NULL; 47 | 48 | 49 | snprintf(summary, strlen("X-ForensicLog-Summary: ") + strlen(audit_log->summary_line) + 1, 50 | "X-ForensicLog-Summary: %s", audit_log->summary_line); 51 | 52 | snprintf(credentials, strlen(conf->username) + 53 | strlen(conf->password) + 1 + 1, "%s:%s", conf->username, 54 | conf->password); 55 | 56 | curl_easy_setopt(curl, CURLOPT_URL, conf->host); 57 | curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); 58 | curl_easy_setopt(curl, CURLOPT_USERPWD, credentials); 59 | 60 | curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L); 61 | 62 | curl_easy_setopt(curl, CURLOPT_READFUNCTION, send_to_server_read_cb); 63 | curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, sent_to_server_write_cb); 64 | curl_easy_setopt(curl, CURLOPT_READDATA, audit_log); 65 | curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, strlen(audit_log->buf)); 66 | 67 | //headerlist = curl_slist_append(headerlist, "Expect:"); 68 | //headerlist = curl_slist_append(headerlist, hash); 69 | headerlist = curl_slist_append(headerlist, summary); 70 | curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headerlist); 71 | 72 | 73 | if (conf->validate_certificate_chain == 0) 74 | { 75 | curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L); 76 | curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L); 77 | } 78 | res = curl_easy_perform(curl); 79 | 80 | if(res != CURLE_OK) 81 | { 82 | e("failed to send: %s\n%s\n", audit_log->audit_file, 83 | curl_easy_strerror(res)); 84 | } 85 | 86 | curl_easy_cleanup(curl); 87 | } 88 | 89 | curl_global_cleanup(); 90 | 91 | return 0; 92 | } 93 | 94 | struct send_to_server_config_t *create_send_to_server_instance( 95 | const yajl_val *val, struct pipeline_t *p) 96 | { 97 | const char *n_path_host[] = {"host",(const char *)0}; 98 | const char *n_path_username[] = {"username",(const char *)0}; 99 | const char *n_path_password[] = {"password",(const char *)0}; 100 | const char *n_path_sensor_id[] = {"sensor_id",(const char *)0}; 101 | const char *n_path_validate_cert_chain[] = {"validate_certificate_chain", 102 | (const char *)0}; 103 | const char *n_path_number_of_clients[] = {"number_of_clients", 104 | (const char *)0}; 105 | char *validate_certificate_chain; 106 | int number_of_clients; 107 | 108 | struct send_to_server_config_t *inst = malloc(sizeof( 109 | struct send_to_server_config_t)); 110 | 111 | inst->host = NULL; 112 | inst->username = NULL; 113 | inst->password = NULL; 114 | inst->sensor_id = NULL; 115 | inst->validate_certificate_chain = 0; 116 | inst->number_of_clients = 2; 117 | 118 | yajl_val host_ = yajl_tree_get(*val, n_path_host, yajl_t_string); 119 | inst->host = YAJL_GET_STRING(host_); 120 | 121 | yajl_val username_ = yajl_tree_get(*val, n_path_username, yajl_t_string); 122 | inst->username = YAJL_GET_STRING(username_); 123 | 124 | yajl_val password_ = yajl_tree_get(*val, n_path_password, yajl_t_string); 125 | inst->password = YAJL_GET_STRING(password_); 126 | 127 | yajl_val sensor_id_ = yajl_tree_get(*val, n_path_sensor_id, yajl_t_string); 128 | inst->sensor_id = YAJL_GET_STRING(sensor_id_); 129 | 130 | yajl_val chain_ = yajl_tree_get(*val, n_path_validate_cert_chain, 131 | yajl_t_string); 132 | validate_certificate_chain = YAJL_GET_STRING(chain_); 133 | 134 | yajl_val number_ = yajl_tree_get(*val, n_path_number_of_clients, 135 | yajl_t_number); 136 | number_of_clients = YAJL_GET_INTEGER(number_); 137 | 138 | if (validate_certificate_chain && 139 | strcmp("true", validate_certificate_chain) != 0) 140 | { 141 | inst->validate_certificate_chain = 0; 142 | } 143 | 144 | if (number_of_clients > 0) 145 | { 146 | inst->number_of_clients = number_of_clients; 147 | } 148 | 149 | return inst; 150 | } 151 | 152 | -------------------------------------------------------------------------------- /pipe_elements/send_to_server.h: -------------------------------------------------------------------------------- 1 | 2 | 3 | #include "mlogc-ng.h" 4 | #include "configuration.h" 5 | 6 | #ifndef __SEND_TO_SERVER_H__ 7 | #define __SEND_TO_SERVER_H__ 8 | 9 | struct send_to_server_config_t { 10 | char *host; 11 | char *username; 12 | char *password; 13 | char *sensor_id; 14 | int validate_certificate_chain; 15 | int number_of_clients; 16 | }; 17 | 18 | int send_to_server_process (struct pipeline_element_t *, 19 | struct audit_log_entry_t *); 20 | 21 | struct send_to_server_config_t *create_send_to_server_instance(const yajl_val *, 22 | struct pipeline_t *); 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /pipeline.c: -------------------------------------------------------------------------------- 1 | 2 | #include "yajl/yajl_tree.h" 3 | 4 | #include "pipeline.h" 5 | #include "input_batch.h" 6 | #include "dump.h" 7 | #include "persistence.h" 8 | #include "read_from_filesystem.h" 9 | #include "send_to_server.h" 10 | 11 | struct pipeline_element_t *create_pipe_element(const char *name, const yajl_val *val, struct pipeline_t *p) 12 | { 13 | struct pipeline_element_t *element; 14 | /* 15 | * We should have a dynamic list here, doing it hard coded 16 | * to have it running ASAP. 17 | * 18 | */ 19 | 20 | if (strcmp(name, "read_from_filesystem") == 0) 21 | { 22 | v("Reading form file system element is now being created\n"); 23 | element = malloc(sizeof(struct pipeline_element_t)); 24 | element->name = "read_from_filesystem"; 25 | element->payload = (void *)create_read_from_filesystem_instance(val, p); 26 | element->process = read_from_filesystem_process; 27 | } 28 | if (strcmp(name, "dump") == 0) 29 | { 30 | v("dump element is now being created\n"); 31 | element = malloc(sizeof(struct pipeline_element_t)); 32 | element->name = "dump"; 33 | element->payload = (void *)create_dump_instance(val, p); 34 | element->process = dump_process; 35 | } 36 | if (strcmp(name, "persistence") == 0) 37 | { 38 | v("persistence element is now being created\n"); 39 | element = malloc(sizeof(struct pipeline_element_t)); 40 | element->name = "persistence"; 41 | element->payload = (void *)create_persistence_instance(val, p); 42 | element->process = persistence_process; 43 | } 44 | if (strcmp(name, "send_to_server") == 0) 45 | { 46 | v("send_to_server element is now being created\n"); 47 | element = malloc(sizeof(struct pipeline_element_t)); 48 | element->name = "send_to_server"; 49 | element->payload = (void *)create_send_to_server_instance(val, p); 50 | element->process = send_to_server_process; 51 | } 52 | 53 | return element; 54 | } 55 | 56 | int add_pipe_element(struct pipeline_t *p, struct pipeline_element_t *e) 57 | { 58 | int res = 0; 59 | struct pipeline_element_t *last = NULL; 60 | 61 | e->prev = NULL; 62 | e->next = NULL; 63 | 64 | if (p->elements == NULL) 65 | { 66 | p->elements = (struct pipeline_element_t *)e; 67 | goto done; 68 | } 69 | 70 | last = p->elements; 71 | while (last->next) 72 | { 73 | last = last->next; 74 | } 75 | 76 | e->prev = last; 77 | last->next = e; 78 | done: 79 | return res; 80 | } 81 | 82 | int pipeline_process(struct pipeline_t *p, struct audit_log_entry_t *t) 83 | { 84 | struct pipeline_element_t *e = NULL; 85 | 86 | v("Sending %s over our pipeline\n", t->id); 87 | 88 | e = p->elements->next; 89 | 90 | while (e) 91 | { 92 | int res = 0; 93 | v("Pipe: %s\n", e->name); 94 | res = (int)(*(e->process))(e, t); 95 | if (res) 96 | { 97 | break; 98 | } 99 | e = e->next; 100 | } 101 | 102 | return 0; 103 | } 104 | 105 | -------------------------------------------------------------------------------- /pipeline.h: -------------------------------------------------------------------------------- 1 | 2 | #include "yajl/yajl_tree.h" 3 | 4 | #include "mlogc-ng.h" 5 | 6 | #ifndef __PIPELINE_H__ 7 | #define __PIPELINE_H__ 8 | 9 | struct pipeline_element_t { 10 | char *name; 11 | void *payload; 12 | 13 | void *init; 14 | int (*process)(struct pipeline_element_t *, struct audit_log_entry_t *); 15 | 16 | struct pipeline_element_t *next; 17 | struct pipeline_element_t *prev; 18 | }; 19 | 20 | struct pipeline_t { 21 | struct configuration_t *config; 22 | struct pipeline_element_t *elements; 23 | }; 24 | 25 | struct pipeline_element_t *create_pipe_element(const char *, const yajl_val *, 26 | struct pipeline_t *); 27 | int add_pipe_element(struct pipeline_t *, struct pipeline_element_t *); 28 | int pipeline_process(struct pipeline_t *, struct audit_log_entry_t *); 29 | 30 | #endif 31 | --------------------------------------------------------------------------------