├── .clang_complete ├── .gitignore ├── .gitlab-ci.yml ├── .gitmodules ├── COPYING ├── INSTALL ├── Makefile ├── Makefile.dir ├── README ├── src ├── Makefile ├── Makefile.dir ├── core │ ├── Makefile │ ├── Makefile.dir │ ├── configure.c │ ├── configure.h │ ├── context.c │ ├── context.h │ ├── core.txt │ ├── link_list.h │ ├── loader.c │ ├── loader.h │ ├── loop.c │ ├── loop.h │ ├── mem_pool.c │ ├── mem_pool.h │ ├── packet.c │ ├── packet.h │ ├── plugin.h │ ├── pluglib.c │ ├── pluglib.h │ ├── pluglib_list.h │ ├── pluglib_macros.h │ ├── recycler.h │ ├── startup.c │ ├── startup.h │ ├── trie.c │ ├── trie.h │ ├── tunable.h │ ├── uplink.c │ ├── uplink.h │ ├── uplink.txt │ ├── util.c │ └── util.h ├── lcollect │ ├── Makefile │ ├── Makefile.dir │ ├── lcollect-turris.cfg │ ├── lcollect.txt │ └── main.c ├── libs │ ├── Makefile │ ├── Makefile.dir │ ├── diffstore │ │ ├── Makefile │ │ ├── Makefile.dir │ │ ├── diff_store.c │ │ └── diff_store.h │ └── test │ │ ├── Makefile │ │ ├── Makefile.dir │ │ ├── main.c │ │ └── test.h ├── master │ ├── Makefile │ ├── Makefile.dir │ ├── activity.py │ ├── amihacked │ │ ├── README │ │ ├── bzchoose │ │ ├── compact.pl │ │ ├── export-repu │ │ ├── jsonize.pl │ │ ├── process │ │ │ ├── Cargo.toml │ │ │ └── src │ │ │ │ └── main.rs │ │ ├── repu_add │ │ ├── repu_init │ │ ├── split.pl │ │ └── to_db.pl │ ├── analyze │ │ ├── fake │ │ ├── fake.pl │ │ └── sniff.pl │ ├── archivist │ │ ├── archivist.pl │ │ └── initdb │ ├── auth.py │ ├── authenticator │ │ ├── Makefile │ │ ├── Makefile.dir │ │ ├── atsha204.c │ │ ├── authenticator.conf │ │ ├── authenticator.py │ │ └── gen-email-passwd.py │ ├── bandwidth_plugin.py │ ├── ca.pem │ ├── client.py │ ├── collect-master.conf │ ├── collect-master.py │ ├── collectmaster.txt │ ├── count_plugin.py │ ├── crash_plugin.py │ ├── database.py │ ├── dbscripts │ │ ├── AddrStoreBuild.pm │ │ ├── Tagger │ │ │ ├── FlowFilter.pm │ │ │ ├── Flows.pm │ │ │ └── FwUp.pm │ │ ├── activities │ │ ├── blacklist │ │ ├── capture_stats │ │ ├── counts │ │ ├── cvut-export.sql │ │ ├── dbconfig │ │ ├── dupaddr.pl │ │ ├── export_fake │ │ ├── export_fake.pl │ │ ├── groups.pl │ │ ├── initdb │ │ ├── log_uptime.pl │ │ ├── online.pl │ │ ├── parse-atsha.pl │ │ ├── pkg-unpack │ │ ├── plugin_hashes │ │ ├── plugin_hashes.pl │ │ ├── plugins.pl │ │ ├── purge │ │ ├── tagger │ │ ├── tagger.ini.example │ │ ├── tagger.pl │ │ └── testdb │ ├── diff_addr_store.py │ ├── fake_plugin.py │ ├── flow_plugin.py │ ├── fwup_plugin.py │ ├── log_extra.py │ ├── master_config.py │ ├── plugin.py │ ├── plugin_versions.py │ ├── protocol.py │ ├── rate_limit.py │ ├── refused_plugin.py │ ├── sniff │ │ ├── __init__.py │ │ ├── cert.py │ │ ├── main.py │ │ ├── ping.py │ │ └── task.py │ ├── soxy │ │ ├── conn.h │ │ ├── handler.h │ │ ├── main.cpp │ │ └── soxy.pro │ ├── spoof_plugin.py │ └── timers.py ├── plugins │ ├── Makefile │ ├── Makefile.dir │ ├── badconf │ │ ├── Makefile │ │ ├── Makefile.dir │ │ └── badconf.c │ ├── bandwidth │ │ ├── Makefile │ │ ├── Makefile.dir │ │ ├── bandwidth.c │ │ └── bandwidth.txt │ ├── cfgtest │ │ ├── Makefile │ │ ├── Makefile.dir │ │ └── cfgtest.c │ ├── count │ │ ├── Makefile │ │ ├── Makefile.dir │ │ ├── count.c │ │ └── count.txt │ ├── crash │ │ ├── Makefile │ │ ├── Makefile.dir │ │ └── crash.c │ ├── fake │ │ ├── Makefile │ │ ├── Makefile.dir │ │ ├── base64.c │ │ ├── base64.h │ │ ├── fake.txt │ │ ├── fake_firewall.txt │ │ ├── firewall-add │ │ ├── firewall-init │ │ ├── firewall-syslog │ │ ├── log.c │ │ ├── log.h │ │ ├── main.c │ │ ├── main.h │ │ ├── server.c │ │ ├── server.h │ │ ├── telnet.c │ │ ├── telnet.h │ │ ├── websrv.c │ │ └── websrv.h │ ├── flow │ │ ├── Makefile │ │ ├── Makefile.dir │ │ ├── filter.c │ │ ├── filter.h │ │ ├── flow.c │ │ ├── flow.h │ │ ├── flow.txt │ │ └── main.c │ ├── fwup │ │ ├── Makefile │ │ ├── Makefile.dir │ │ ├── fwup.txt │ │ ├── main.c │ │ ├── queue.c │ │ ├── queue.h │ │ ├── type.c │ │ └── type.h │ ├── majordomo │ │ ├── Makefile │ │ ├── Makefile.dir │ │ ├── majordomo.c │ │ ├── majordomo.txt │ │ └── scripts │ │ │ ├── majordomo-cron │ │ │ ├── majordomo.conf │ │ │ ├── majordomo_cache.lua │ │ │ ├── majordomo_db.sh │ │ │ ├── majordomo_lib.lua │ │ │ ├── majordomo_locked_precache.sh │ │ │ ├── majordomo_merge.lua │ │ │ └── majordomo_show.lua │ ├── plugtest │ │ ├── Makefile │ │ ├── Makefile.dir │ │ └── plugtest.c │ ├── refused │ │ ├── Makefile │ │ ├── Makefile.dir │ │ ├── icmp.c │ │ ├── icmp.h │ │ ├── refused.c │ │ └── refused.txt │ ├── sniff │ │ ├── Makefile │ │ ├── Makefile.dir │ │ ├── cert.c │ │ ├── cert.h │ │ ├── fork.c │ │ ├── fork.h │ │ ├── getip.cgi │ │ ├── nop.c │ │ ├── nop.h │ │ ├── parse.c │ │ ├── parse.h │ │ ├── ping.c │ │ ├── ping.h │ │ ├── sniff-cert │ │ ├── sniff-nat │ │ ├── sniff-ping │ │ ├── sniff.c │ │ ├── sniff.txt │ │ ├── task.c │ │ └── task.h │ └── spoof │ │ ├── Makefile │ │ ├── Makefile.dir │ │ ├── spoof.c │ │ └── spoof.txt └── ucollect │ ├── Makefile │ ├── Makefile.dir │ ├── main.c │ ├── main.cfg │ ├── ucollect-turris.cfg │ ├── ucollect.cfg │ ├── ucollect.pem │ └── ucollect.txt └── tools ├── hammer.pl └── update_hash.pl /.clang_complete: -------------------------------------------------------------------------------- 1 | -DPAGE_SIZE=4096 2 | -DPASSWD_HALF={0} 3 | -x c 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .objs/ 2 | .deps/ 3 | bin/ 4 | build/buildsystem.html 5 | docs/ 6 | src/ucollect/ucollect 7 | src/lcollect/lcollect 8 | /lib/ 9 | src/core/libucollect_core.* 10 | *.so 11 | *.html 12 | *.pyc 13 | src/plugins/buckets/hash_test 14 | src/master/collectmaster 15 | src/master/authenticator/authenticator 16 | tags 17 | src/master/soxy/*.o 18 | src/master/soxy/moc_* 19 | src/master/soxy/Makefile 20 | src/master/soxy/soxy 21 | src/plugins/majordomo/subnetfilter/majordomo_subnetfilter 22 | dumper.lua 23 | cov-int/ 24 | core 25 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | image: gl-ci-turris-testing 2 | 3 | stages: 4 | - test 5 | 6 | before_script: 7 | - git submodule init 8 | - git submodule update 9 | 10 | test: 11 | stage: test 12 | 13 | script: 14 | - nice make -j8 cppcheck 15 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "build"] 2 | path = build 3 | url = https://gitlab.labs.nic.cz/turris/buildsystem.git 4 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # This file contains configuration of the build. It may be generated by some 2 | # script in the future. 3 | 4 | # The RELATIVE variable is for the wrapper makefiles. 5 | O := $(RELATIVE). 6 | S := $(RELATIVE). 7 | ASCIIDOC := asciidoc 8 | AR := ar 9 | MAX_LOG_LEVEL := LLOG_DEBUG_VERBOSE 10 | PAGE_SIZE := $(shell getconf PAGE_SIZE) 11 | ifneq ($(NO_MASTER),1) 12 | ENABLE_PYTHON := 1 13 | endif 14 | 15 | include $(S)/Makefile.dir 16 | -------------------------------------------------------------------------------- /Makefile.dir: -------------------------------------------------------------------------------- 1 | CFLAGS_ALL += -I"$(O)/.objs" 2 | ifdef LOGIN_PASSWD_HALF 3 | CFLAGS_ALL += -DPASSWD_HALF="$$(cat $(LOGIN_PASSWD_HALF))" 4 | else 5 | CFLAGS_ALL += -DPASSWD_HALF='{0}' 6 | endif 7 | ifdef SOFT_LOGIN 8 | CFLAGS_ALL += -DSOFT_LOGIN=1 9 | endif 10 | ifndef NO_SIGNAL_REINIT 11 | CFLAGS += -DSIGNAL_REINIT 12 | endif 13 | 14 | include $(S)/build/Makefile.top 15 | 16 | # List of all makefiles in direct subdirectories. If a new subdirectory is 17 | # made, it goes here. 18 | include $(S)/src/Makefile.dir 19 | # This one for documentation for the build system 20 | include $(S)/build/Makefile.dir 21 | 22 | include $(S)/build/Makefile.bottom 23 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | Ucollect 2 | ======== 3 | 4 | Ucollect is small daemon for collecting and analyzing network data. It is 5 | written in C (with some unusual conventions about memory allocation, see the 6 | relevant documentation) and provides plugins for doing the actuall analyzes. 7 | 8 | The repository also contains the server part (in srv/master) and various 9 | support utilities. 10 | 11 | To build, fist initiialize git submodules (`git submodule init && git submodule 12 | update`) and then type `make`. You may need to supply some dependencies (guess 13 | from possible compiler errors). 14 | 15 | For further documentation, look into `docs` subdirectory (after you built it) 16 | or into .txt files scattered in the source tree. 17 | -------------------------------------------------------------------------------- /src/Makefile: -------------------------------------------------------------------------------- 1 | RESTRICT := src 2 | RELATIVE := ../ 3 | 4 | include $(RELATIVE)/Makefile 5 | -------------------------------------------------------------------------------- /src/Makefile.dir: -------------------------------------------------------------------------------- 1 | include $(S)/src/ucollect/Makefile.dir 2 | include $(S)/src/lcollect/Makefile.dir 3 | include $(S)/src/core/Makefile.dir 4 | include $(S)/src/libs/Makefile.dir 5 | include $(S)/src/plugins/Makefile.dir 6 | include $(S)/src/master/Makefile.dir 7 | -------------------------------------------------------------------------------- /src/core/Makefile: -------------------------------------------------------------------------------- 1 | RESTRICT := src/core 2 | RELATIVE := ../../ 3 | 4 | include $(RELATIVE)/Makefile 5 | -------------------------------------------------------------------------------- /src/core/Makefile.dir: -------------------------------------------------------------------------------- 1 | LIBRARIES += src/core/libucollect_core 2 | DOCS += $(addprefix src/core/,core uplink) 3 | 4 | libucollect_core_MODULES := mem_pool util loop context packet uplink loader configure trie startup pluglib 5 | libucollect_core_PKG_CONFIGS := zlib 6 | -------------------------------------------------------------------------------- /src/core/configure.h: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2013-2015 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #ifndef UCOLLECT_CONFIGURE_H 21 | #define UCOLLECT_CONFIGURE_H 22 | 23 | #include 24 | 25 | struct loop; 26 | 27 | /* 28 | * Set the configuration directory. Not copied, should be preserved for the 29 | * whole lifetime of the program. 30 | */ 31 | void config_set_dir(const char *dir) __attribute__((nonnull)); 32 | void config_set_package(const char *package_name) __attribute__((nonnull)); 33 | const char *config_get_package(); 34 | void config_allow_null_uplink(void); 35 | bool load_config(struct loop *loop) __attribute__((nonnull)); 36 | 37 | #endif 38 | -------------------------------------------------------------------------------- /src/core/context.c: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2013 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #include "context.h" 21 | 22 | struct context *current_context; 23 | -------------------------------------------------------------------------------- /src/core/context.h: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2013 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #ifndef UCOLLECT_CONTEXT_H 21 | #define UCOLLECT_CONTEXT_H 22 | 23 | // Forward declarations 24 | struct mem_pool; 25 | struct loop; 26 | /* 27 | * This is not a true forward declaration. This is not defined in this library at all. 28 | * We expect each plugin defines its own version. This is slightly better than 29 | * plain type casting, as the compiler does some minimal checks about type safety 30 | * inside the same plugin. 31 | */ 32 | struct user_data; 33 | 34 | struct context { 35 | struct mem_pool *permanent_pool; 36 | struct mem_pool *temp_pool; 37 | struct loop *loop; 38 | struct uplink *uplink; 39 | struct user_data *user_data; 40 | }; 41 | 42 | #endif 43 | -------------------------------------------------------------------------------- /src/core/loader.h: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2013-2015 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #ifndef UCOLLECT_LOADER_H 21 | #define UCOLLECT_LOADER_H 22 | 23 | #include 24 | 25 | struct plugin; 26 | struct pluglib; 27 | 28 | /* 29 | * Does the low-level loading of plugin libraries. It only asks the plugin 30 | * to provide the information in struct plugin, does not initialize it. 31 | * 32 | * Call it with the library name to load. 33 | */ 34 | 35 | void *plugin_load(const char *libname, struct plugin *target, uint8_t *hash, unsigned *api_version) __attribute__((nonnull)) __attribute__((malloc)); 36 | void *pluglib_load(const char *libname, struct pluglib *target, uint8_t *hash) __attribute__((nonnull)) __attribute__((malloc)); 37 | // Used both for plugins and pluglibs 38 | void plugin_unload(void *plugin) __attribute__((nonnull)); 39 | 40 | #endif 41 | -------------------------------------------------------------------------------- /src/core/mem_pool.h: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2013-2015 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #ifndef UCOLLECT_MEM_POOL_H 21 | #define UCOLLECT_MEM_POOL_H 22 | 23 | #include 24 | #include 25 | 26 | // Opaque handle to a memory pool. 27 | struct mem_pool; 28 | 29 | // Create a memory pool. The name is debug and error message aid. 30 | struct mem_pool *mem_pool_create(const char *name) __attribute__((malloc)) __attribute__((returns_nonnull)); 31 | // Destroy a memory pool, freeing all its memory as well. 32 | void mem_pool_destroy(struct mem_pool *pool) __attribute__((nonnull)); 33 | 34 | // Allocate bit of memory from given memory pool. Never returns NULL (crashes if it can't allocate) 35 | void *mem_pool_alloc(struct mem_pool *pool, size_t size) __attribute__((malloc)) __attribute__((nonnull)) __attribute__((alloc_size(2))) __attribute__((returns_nonnull)); 36 | // Free all memory allocated from this memory pool. The pool can be used to get more allocations. 37 | void mem_pool_reset(struct mem_pool *pool) __attribute__((nonnull)); 38 | 39 | // Some convenience functions 40 | 41 | // Copy a string to memory from the pool 42 | char *mem_pool_strdup(struct mem_pool *pool, const char *string) __attribute__((malloc)) __attribute__((nonnull)) __attribute__((returns_nonnull)); 43 | // Format a string by printf formatting to memory from the pool and return 44 | char *mem_pool_printf(struct mem_pool *pool, const char *format, ...) __attribute__((malloc)) __attribute__((nonnull(1, 2))) __attribute__((format(printf, 2, 3))) __attribute__((returns_nonnull)); 45 | // Format binary data to hex 46 | char *mem_pool_hex(struct mem_pool *pool, const uint8_t *data, size_t size) __attribute__((malloc)) __attribute__((nonnull)) __attribute__((returns_nonnull)); 47 | 48 | // Provide a string with statistics about all the memory pools. The result is allocated from tmp_pool 49 | char *mem_pool_stats(struct mem_pool *tmp_pool) __attribute__((malloc)) __attribute__((nonnull)); 50 | 51 | #endif 52 | -------------------------------------------------------------------------------- /src/core/plugin.h: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2013 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #ifndef UCOLLECT_PLUGIN_H 21 | #define UCOLLECT_PLUGIN_H 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | #include "pluglib.h" 29 | 30 | struct context; 31 | struct packet_info; 32 | 33 | struct config_node { 34 | const char **values; 35 | size_t value_count; 36 | }; 37 | 38 | typedef void (*packet_callback_t)(struct context *context, const struct packet_info *info); 39 | typedef void (*fd_callback_t)(struct context *context, int fd, void *tag); 40 | 41 | struct plugin { 42 | const char *name; 43 | packet_callback_t packet_callback; 44 | void (*init_callback)(struct context *context); 45 | void (*finish_callback)(struct context *context); 46 | void (*uplink_connected_callback)(struct context *context); 47 | void (*uplink_disconnected_callback)(struct context *context); 48 | void (*uplink_data_callback)(struct context *context, const uint8_t *data, size_t length); 49 | fd_callback_t fd_callback; 50 | uint16_t version; // Protocol version (not necessarily increased after each code change in the plugin, only if it may need change on the server) 51 | // Called when the plugin should check configuration. Use loop_plugin_option_get to read it and return if the config is OK. Do not use yet. 52 | bool (*config_check_callback)(struct context *context); 53 | // A decision has been made if this config is globaly OK. If activate is true, start using it (you can store it in config_check or read it any time later with loop_plugin_option_get). If it is false, then return to the previous config (and loop_plugin_option_get will return the old value). 54 | void (*config_finish_callback)(struct context *context, bool activate); 55 | /* ----- The below things are available only from API version 1 and above ----- */ 56 | // Functions imported from plugin libraries 57 | struct pluglib_import **imports; 58 | /* ----- The below things are available only from API version 2 and above ----- */ 59 | // Broadcasted when a child of ucollect dies. It may belong to other plugin, for example. The state is one from the wait() function. 60 | void (*child_died_callback)(struct context *context, int state, pid_t child); 61 | }; 62 | 63 | #define UCOLLECT_PLUGIN_API_VERSION 2 64 | 65 | #endif 66 | -------------------------------------------------------------------------------- /src/core/pluglib.c: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2015 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #include "pluglib.h" 21 | #define LIST_WANT_LFOR 22 | #include "pluglib_list.h" 23 | 24 | #include "util.h" 25 | 26 | #include 27 | 28 | static bool pluglib_resolve_functions_internal(const struct pluglib_list *libraries, struct pluglib_import **imports, bool link) { 29 | if (!imports) 30 | return true; // No imports, all satisfied 31 | for (; *imports; imports ++) { 32 | bool found = false; 33 | LFOR(pluglib_list, lib, libraries) { 34 | if (!lib->lib->exports) // Empty library ‒ useless, but valid 35 | continue; 36 | for (struct pluglib_export **export = lib->lib->exports; !found && *export; export ++) { 37 | if (strcmp((*export)->name, (*imports)->name) == 0) { 38 | // A candidate. Check the prototype. 39 | if (!(*export)->prototype || // There's no prototype available 40 | !(*imports)->prototype || // Any prototype goes 41 | strcmp((*export)->prototype, (*imports)->prototype) == 0) { // Prototype matches 42 | found = true; 43 | if (link) { 44 | *(*imports)->function = (*export)->function; 45 | ulog(LLOG_DEBUG, "Linking function %s\n", (*export)->name); 46 | } 47 | break; 48 | } else { 49 | if (link) 50 | ulog(LLOG_WARN, "Prototype for function %s does not match (%s vs %s)\n", (*export)->name, (*export)->prototype, (*imports)->prototype); 51 | } 52 | } 53 | } 54 | } 55 | if (!found) { 56 | ulog(LLOG_ERROR, "Couldn't find function %s\n", (*imports)->name); 57 | return false; 58 | } 59 | } 60 | return true; 61 | } 62 | 63 | bool pluglib_resolve_functions(const struct pluglib_list *libraries, struct pluglib_import **imports) { 64 | return pluglib_resolve_functions_internal(libraries, imports, true); 65 | } 66 | 67 | bool pluglib_check_functions(const struct pluglib_list *libraries, struct pluglib_import **imports) { 68 | return pluglib_resolve_functions_internal(libraries, imports, false); 69 | } 70 | -------------------------------------------------------------------------------- /src/core/pluglib.h: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2015 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #ifndef UCOLLECT_PLUGLIB_H 21 | #define UCOLLECT_PLUGLIB_H 22 | 23 | #include 24 | #include 25 | #include 26 | 27 | #include "tunable.h" 28 | 29 | typedef void (*pluglib_function)(void); 30 | 31 | struct pluglib_export; 32 | 33 | // This is the structure the pluglib exports. 34 | struct pluglib { 35 | const char *name; 36 | size_t ref_count; // For use by the core app, do not fill in. 37 | size_t compat; 38 | size_t version; 39 | struct pluglib *recycler_next; // For use by the core app, do not fill in. 40 | struct pluglib_export **exports; 41 | }; 42 | 43 | // Other structures are not handled explicitly by pluglibs or plugins. 44 | struct pluglib_export { 45 | const char *name; 46 | pluglib_function function; 47 | const char *prototype; 48 | }; 49 | 50 | struct pluglib_import { 51 | const char *name; 52 | pluglib_function *function; 53 | const char *prototype; 54 | }; 55 | 56 | struct pluglib_node { 57 | struct pluglib_node *next, *prev; 58 | struct pluglib *lib; 59 | void *handle; 60 | bool ready; 61 | uint8_t hash[CHALLENGE_LEN / 2]; 62 | }; 63 | 64 | struct pluglib_list { 65 | struct pluglib_node *head, *tail; 66 | }; 67 | 68 | // Link the functions from pluglib into a plugin 69 | bool pluglib_resolve_functions(const struct pluglib_list *libraries, struct pluglib_import **imports) __attribute__((nonnull(1))); 70 | // Check if all the imports could be satisfied. 71 | bool pluglib_check_functions(const struct pluglib_list *libraries, struct pluglib_import **imports) __attribute__((nonnull(1))); 72 | 73 | // What should be imported by the import macros 74 | #define PLUGLIB_LOCAL 1 // Both the import structures and the function pointer, all defined as static. For use in the same file as the plugins entry point. 75 | #define PLUGLIB_FUNCTIONS 2 // Declarations for extern functions. To be used in other files than the one with plugin entry point. 76 | #define PLUGLIB_PUBLIC 3 // The structures and function pointers, but the function pointers being accessible from outside of the compilation unit. 77 | 78 | #endif 79 | -------------------------------------------------------------------------------- /src/core/pluglib_list.h: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2015 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #include "pluglib.h" 21 | #define LIST_NODE struct pluglib_node 22 | #define LIST_BASE struct pluglib_list 23 | #define LIST_PREV prev 24 | #define LIST_NAME(X) pluglib_list_##X 25 | #include "link_list.h" 26 | -------------------------------------------------------------------------------- /src/core/recycler.h: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2014 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | /* 21 | * This header is little bit special in that it generates new code. 22 | * You define bunch of defines and macros and include the header. 23 | * The header introduces bunch of functions and undefines the macros. 24 | * 25 | * The header doesn't have the usual #ifndef guard, since it is expected 26 | * to include multiple times, with different defines. 27 | * 28 | * This is somewhat similar to C++ templates (but a way more powerful and 29 | * lightweight, and should produce more readable errors, though the code 30 | * is less convenient to read and it needs to be instantiated explicitly). 31 | * 32 | * This one contains the recycler - a place where objects can be put when 33 | * unused and recycled when needed. 34 | * 35 | * The definitions are: 36 | * - RECYCLER_NODE: The type of the object. 37 | * - RECYCLER_BASE: The object holding the recycler. 38 | * - RECYCLER_HEAD: Variable inside RECYCLER_BASE holding pointer to 39 | * RECYCLER_NODE. Defaults to 'head'. 40 | * - RECYCLER_NEXT: Variable inside RECYCLER_NODE holding pointer to 41 | * another RECYCLER_NODE. Defaults to next. 42 | * - RECYCLER_NAME(X): Macro returning name of functions provided part 43 | * of the name. Could be something like prefix_##X. 44 | */ 45 | 46 | #include "mem_pool.h" 47 | 48 | // Check all needed defines are there 49 | #ifndef RECYCLER_NODE 50 | #error "RECYCLER_NODE not defined" 51 | #endif 52 | #ifndef RECYCLER_BASE 53 | #error "RECYCLER_BASE not defined" 54 | #endif 55 | #ifndef RECYCLER_NAME 56 | #error "RECYCLER_NAME not defined" 57 | #endif 58 | 59 | // Define defaults, if not provided 60 | #ifndef RECYCLER_HEAD 61 | #define RECYCLER_HEAD head 62 | #endif 63 | #ifndef RECYCLER_NEXT 64 | #define RECYCLER_NEXT next 65 | #endif 66 | 67 | /* 68 | * Get an object. Either an unused one, or allocate a new one from memory 69 | * pool. 70 | */ 71 | static RECYCLER_NODE *RECYCLER_NAME(get)(RECYCLER_BASE *base, struct mem_pool *pool) { 72 | if (base->RECYCLER_HEAD) { 73 | RECYCLER_NODE *result = base->RECYCLER_HEAD; 74 | base->RECYCLER_HEAD = result->RECYCLER_NEXT; 75 | return result; 76 | } else { 77 | return mem_pool_alloc(pool, sizeof(RECYCLER_NODE)); 78 | } 79 | } 80 | 81 | /* 82 | * Release an object and put it to the recycler. It may be returned by get in future. 83 | */ 84 | static void RECYCLER_NAME(release)(RECYCLER_BASE *base, RECYCLER_NODE *object) { 85 | object->RECYCLER_NEXT = base->RECYCLER_HEAD; 86 | base->RECYCLER_HEAD = object; 87 | } 88 | 89 | #undef RECYCLER_NODE 90 | #undef RECYCLER_BASE 91 | #undef RECYCLER_HEAD 92 | #undef RECYCLER_NEXT 93 | #undef RECYCLER_NAME 94 | -------------------------------------------------------------------------------- /src/core/startup.c: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2014 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #include "startup.h" 21 | #include "loop.h" 22 | #include "uplink.h" 23 | #include "util.h" 24 | 25 | #include 26 | #include 27 | #include 28 | 29 | struct loop *loop; 30 | struct uplink *uplink; 31 | 32 | static void stop_signal_handler(int unused) { 33 | (void) unused; 34 | loop_break(loop); 35 | } 36 | 37 | static const int stop_signals[] = { 38 | SIGINT, 39 | SIGQUIT, 40 | SIGTERM 41 | }; 42 | 43 | void set_stop_signals(void) { 44 | // Register all stop signals. 45 | for (size_t i = 0; i < sizeof stop_signals / sizeof *stop_signals; i ++) { 46 | struct sigaction action = { 47 | .sa_handler = stop_signal_handler, 48 | /* 49 | * We want to disturb as little as possible (SA_RESTART). 50 | * If the cleanup gets stuck and the user gets impatient and presses CTRL+C again, 51 | * we want to terminate the hard wait instead of doing clean shutdown. So use 52 | * the default handler for the second attempt (SA_RESETHAND). 53 | */ 54 | .sa_flags = SA_RESTART | SA_RESETHAND 55 | }; 56 | if (sigaction(stop_signals[i], &action, NULL) != 0) 57 | die("Could not set signal handler for signal %d (%s)\n", stop_signals[i], strerror(errno)); 58 | } 59 | } 60 | 61 | void system_cleanup(void) { 62 | if (uplink) { 63 | uplink_destroy(uplink); 64 | uplink = NULL; 65 | } 66 | if (loop) { 67 | loop_destroy(loop); 68 | loop = NULL; 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/core/startup.h: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2014 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | /* 21 | * Some helper functions to start up the main process. They 22 | * don't do much, but repeating them is boring. 23 | */ 24 | 25 | #ifndef UCOLLECT_STARTUP_H 26 | #define UCOLLECT_STARTUP_H 27 | 28 | struct loop; 29 | struct uplink; 30 | 31 | // The main loop used by the process and the uplink (if any). 32 | extern struct loop *loop; 33 | extern struct uplink *uplink; 34 | 35 | // Set up shutdown signals 36 | void set_stop_signals(void); 37 | 38 | // Free the loop and uplink, if they exist 39 | void system_cleanup(void); 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /src/core/tunable.h: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2013-2015 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #ifndef UCOLLECT_TUNABLES_H 21 | #define UCOLLECT_TUNABLES_H 22 | 23 | // For the event loop 24 | #define MAX_EVENTS 10 25 | #define MAX_PACKETS 100 26 | #define PCAP_TIMEOUT 100 27 | #define PCAP_BUFFER 3276800 28 | 29 | // How many times a plugin may fail before we give up and disable it 30 | #define FAIL_COUNT 5 31 | // After how many milliseconds do we reset the count to zero? 32 | #define FAIL_COUNT_RESET (3600 * 1000) 33 | 34 | /* 35 | * How long to wait before interface reconfiguration attempt happens (10s). 36 | * This is done if the interface breaks down (like when it is turned off). 37 | */ 38 | #define IFACE_RECONFIGURE_TIME 10000 39 | 40 | // We expect there should be a packet at least once in 10 minutes. 41 | #define PCAP_WATCHDOG_TIME (10 * 1000 * 60) 42 | // If nothing comes in 50 + a bit minutes, panic and do a full reconfigure 43 | #define WATCHDOG_MISSED_COUNT 5 44 | 45 | // For the memory pool 46 | #define PAGE_CACHE_SIZE 20 47 | 48 | // Uplink compression level 49 | #define COMPRESSION_LEVEL 9 50 | 51 | // Buffer size for compression/decompression 52 | #define COMPRESSION_BUFFSIZE 1024 53 | // WARNING: Code was tested with small buffers and limiting is size for output 54 | // buffer of deflate(). There must be free space to store complete block header 55 | // (from 4 to 6 bytes) and some output. Do not use smaller buffers than 10 bytes. 56 | 57 | // Uplink reconnect times 58 | // First attempt after 2 seconds 59 | #define RECONNECT_BASE 2000 60 | // Maximum reconnect time of 5 minutes 61 | #define RECONNECT_MAX (1000 * 5 * 60) 62 | // Double the time for reconnect attempt on failure 63 | #define RECONNECT_MULTIPLY 2 64 | // The time to wait before reconnecting because of failed login 65 | #define RECONNECT_AUTH (1000 * 60 * 10) 66 | 67 | // How much time to wait between pings? 60s could be enough but not too much to timeout NAT 68 | #define PING_TIMEOUT (60 * 1000) 69 | // If so many pings are not answered, consider the link dead 70 | #define PING_COUNT 2 71 | 72 | // The challenge length in bytes to send to server 73 | #define CHALLENGE_LEN 32 74 | // Minimum time between connection attempts 75 | #define RECONN_TIME 1000 76 | 77 | // Time to sleep when we receive the stray read (milliseconds) 78 | #define STRAY_READ_SLEEP 500 79 | 80 | // How many attempts to log in before giving up and exiting? 81 | #define LOGIN_FAILURE_LIMIT 10 82 | 83 | // Dump stats every hour 84 | #define STAT_DUMP_TIMEOUT (3600 * 1000) 85 | 86 | // Base protocol version 87 | #define PROTOCOL_VERSION 1 88 | 89 | #endif 90 | -------------------------------------------------------------------------------- /src/core/util.c: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2013-2015 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #include "util.h" 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | void die(const char *format, ...) { 31 | va_list args; 32 | va_start(args, format); 33 | va_list copy; 34 | va_copy(copy, args); 35 | fputs("\x1b[31;1mDIE\x1b[0m: ", stderr); 36 | vfprintf(stderr, format, args); 37 | vsyslog(LOG_MAKEPRI(LOG_DAEMON, LOG_CRIT), format, copy); 38 | va_end(copy); 39 | va_end(args); 40 | // Make sure die means really die, no signal handler for this now. 41 | // No checking the sigaction here. We would have no way to handle that 42 | // anyway. 43 | sigaction(SIGABRT, &(struct sigaction) { 44 | .sa_handler = SIG_DFL 45 | }, NULL); 46 | abort(); 47 | // Last resort 48 | kill(getpid(), SIGKILL); 49 | } 50 | 51 | static const char *names[] = { 52 | [LLOG_DIE] = "\x1b[31;1mDIE\x1b[0m: ", 53 | [LLOG_ERROR] = "\x1b[31mERROR\x1b[0m: ", 54 | [LLOG_WARN] = "\x1b[35mWARN\x1b[0m: ", 55 | [LLOG_INFO] = "\x1b[34mINFO\x1b[0m: ", 56 | [LLOG_DEBUG] = "DEBUG: ", 57 | [LLOG_DEBUG_VERBOSE] = "DEBVE: " 58 | }; 59 | 60 | static const int prios[] = { 61 | [LLOG_DIE] = LOG_CRIT, 62 | [LLOG_ERROR] = LOG_ERR, 63 | [LLOG_WARN] = LOG_WARNING, 64 | [LLOG_INFO] = LOG_INFO, 65 | [LLOG_DEBUG] = LOG_DEBUG 66 | }; 67 | 68 | void ulog_internal(enum log_level log_level, const char *format, va_list *args) { 69 | (void) log_level; // Currently ignored 70 | if (log_level < LLOG_DEBUG_VERBOSE) { 71 | va_list copy; 72 | va_copy(copy, *args); 73 | vsyslog(prios[log_level], format, copy); 74 | va_end(copy); 75 | } 76 | fputs(names[log_level], stderr); 77 | vfprintf(stderr, format, *args); 78 | } 79 | 80 | void sanity_internal(const char *file, unsigned line, const char *check, const char *format, ...) { 81 | va_list args; 82 | va_start(args, format); 83 | va_list copy; 84 | va_copy(copy, args); 85 | size_t needed = vsnprintf(NULL, 0, format, args); 86 | char *output = alloca(needed + 1); 87 | vsnprintf(output, needed + 1, format, copy); 88 | va_end(args); 89 | va_end(copy); 90 | ulog(LLOG_ERROR, "%s:%u: Failed check '%s': %s", file, line, check, output); 91 | abort(); 92 | } 93 | 94 | void abort_safe(void) { 95 | // Disable catching the signal first. 96 | struct sigaction sa = { 97 | .sa_handler = SIG_DFL 98 | }; 99 | sigaction(SIGABRT, &sa, NULL); 100 | abort(); 101 | // Couldn't commit suicide yet? Try exit. 102 | exit(1); 103 | // Still nothing? 104 | kill(getpid(), SIGKILL); 105 | } 106 | -------------------------------------------------------------------------------- /src/core/util.h: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2013-2015 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #ifndef UCOLLECT_UTIL_H 21 | #define UCOLLECT_UTIL_H 22 | 23 | #include 24 | 25 | void die(const char *format, ...) __attribute__((format(printf, 1, 2))) __attribute__((noreturn)); 26 | 27 | #ifndef MAX_LOG_LEVEL 28 | #define MAX_LOG_LEVEL LLOG_DEBUG 29 | #endif 30 | 31 | enum log_level { 32 | LLOG_DIE, 33 | LLOG_ERROR, 34 | LLOG_WARN, 35 | LLOG_INFO, 36 | LLOG_DEBUG, 37 | LLOG_DEBUG_VERBOSE 38 | }; 39 | 40 | void ulog_internal(enum log_level log_level, const char *format, va_list *args); 41 | 42 | static inline void ulog(enum log_level log_level, const char *format, ...) __attribute__((format(printf, 2, 3))); 43 | static inline void ulog(enum log_level log_level, const char *format, ...) { 44 | if (log_level > MAX_LOG_LEVEL) 45 | return; 46 | va_list args; 47 | va_start(args, format); 48 | ulog_internal(log_level, format, &args); 49 | va_end(args); 50 | } 51 | 52 | void sanity_internal(const char *file, unsigned line, const char *check, const char *format, ...) __attribute__((format(printf, 4, 5))) __attribute__((noreturn)); 53 | 54 | // An assert-like function, but with printf message that can be added. It is not omitted from compilation like assert may be. Use for checking input parameters, for example. It may be used as usual asserts, to ensure we get the logs on server. Logs on the ERROR level and aborts, effectively killing a plugin that failed the check. 55 | #define sanity(check, ...) do { if (!(check)) sanity_internal(__FILE__, __LINE__, #check, __VA_ARGS__); } while (0) 56 | // Just failed a sanity check implemented on the caller's side 57 | #define insane(...) sanity(false, __VA_ARGS__) 58 | 59 | // Try very hard to commit a suicide. 60 | void abort_safe(); 61 | 62 | #endif 63 | -------------------------------------------------------------------------------- /src/lcollect/Makefile: -------------------------------------------------------------------------------- 1 | RESTRICT := src/lcollect 2 | RELATIVE := ../../ 3 | 4 | include $(RELATIVE)/Makefile 5 | -------------------------------------------------------------------------------- /src/lcollect/Makefile.dir: -------------------------------------------------------------------------------- 1 | BINARIES += src/lcollect/lcollect 2 | 3 | lcollect_MODULES := \ 4 | main 5 | lcollect_LOCAL_LIBS := ucollect_core 6 | # TODO: Make the build system take this from the ucollect_core somehow 7 | lcollect_SYSTEM_LIBS := pcap rt dl uci crypto ssl unbound atsha204 8 | 9 | DOCS += src/lcollect/lcollect 10 | -------------------------------------------------------------------------------- /src/lcollect/lcollect-turris.cfg: -------------------------------------------------------------------------------- 1 | package 'lcollect' 2 | 3 | config interface 4 | option ifname 'br-lan' 5 | -------------------------------------------------------------------------------- /src/lcollect/lcollect.txt: -------------------------------------------------------------------------------- 1 | The `lcollect` binary 2 | ===================== 3 | 4 | The `lcollect` is alternative implementation of ucollect that is able to run 5 | locally - without uplink. This behavior is useful for some plugins that process 6 | data on client only. 7 | 8 | 9 | The configuration works in the same way as configuration of ucollect (see 10 | ucollect documentation) with 2 exception: 11 | 12 | - lcollect configuration is stored in package lcollect 13 | (/etc/config/lcollect), not in package ucollect(/etc/config/ucollect). 14 | 15 | - The configuration doesn't contains the uplink section 16 | 17 | 18 | -------------------------------------------------------------------------------- /src/lcollect/main.c: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2014 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #include "../core/loop.h" 21 | #include "../core/util.h" 22 | #include "../core/configure.h" 23 | #include "../core/startup.h" 24 | 25 | #include 26 | 27 | int main(int argc, const char* argv[]) { 28 | (void) argc; 29 | openlog("lcollect", LOG_CONS | LOG_NDELAY | LOG_PID, LOG_DAEMON); 30 | if (argv[1]) { 31 | ulog(LLOG_DEBUG, "Setting config dir to %s\n", argv[1]); 32 | config_set_dir(argv[1]); 33 | } 34 | 35 | config_set_package("lcollect"); 36 | config_allow_null_uplink(); 37 | 38 | // Create the loop. 39 | loop = loop_create(); 40 | 41 | set_stop_signals(); 42 | 43 | if (!load_config(loop)) 44 | die("No configuration available\n"); 45 | 46 | // Run until a stop signal comes. 47 | loop_run(loop); 48 | 49 | system_cleanup(); 50 | return 0; 51 | } 52 | -------------------------------------------------------------------------------- /src/libs/Makefile: -------------------------------------------------------------------------------- 1 | RESTRICT := src/libs 2 | RELATIVE := ../../ 3 | 4 | include $(RELATIVE)/Makefile 5 | -------------------------------------------------------------------------------- /src/libs/Makefile.dir: -------------------------------------------------------------------------------- 1 | UCOLLECT_PLUGLIBS:= \ 2 | test \ 3 | diffstore 4 | 5 | include $(patsubst %,$(S)/src/libs/%/Makefile.dir,$(UCOLLECT_PLUGLIBS)) 6 | -------------------------------------------------------------------------------- /src/libs/diffstore/Makefile: -------------------------------------------------------------------------------- 1 | RESTRICT := src/libs/diffstore 2 | RELATIVE := ../../../ 3 | 4 | include $(RELATIVE)/Makefile 5 | -------------------------------------------------------------------------------- /src/libs/diffstore/Makefile.dir: -------------------------------------------------------------------------------- 1 | LIBRARIES += src/libs/diffstore/libpluglib_diffstore 2 | libpluglib_diffstore_MODULES := diff_store 3 | -------------------------------------------------------------------------------- /src/libs/diffstore/diff_store.h: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2015 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #ifndef UCOLLECT_DIFF_STORE_H 21 | #define UCOLLECT_DIFF_STORE_H 22 | 23 | #include 24 | #include 25 | 26 | struct trie; 27 | struct mem_pool; 28 | 29 | enum diff_store_action { 30 | DIFF_STORE_INCREMENTAL, // Ask for a differential update, 31 | DIFF_STORE_FULL, 32 | DIFF_STORE_CONFIG_RELOAD, 33 | DIFF_STORE_NO_ACTION, 34 | DIFF_STORE_UNKNOWN 35 | }; 36 | 37 | struct diff_addr_store; 38 | 39 | typedef void (*addr_hook_t)(struct diff_addr_store *store, const uint8_t *addr, size_t addr_len); 40 | typedef void (*addr_replace_hook_t)(struct diff_addr_store *store); 41 | 42 | struct diff_addr_store { 43 | const char *name; 44 | struct trie *trie; 45 | struct mem_pool *pool; 46 | uint32_t epoch, version; 47 | size_t added, deleted; // Statistics, to know when to re-requested the whole filter config 48 | /* The following 5 members - 4 hooks and userdata - may be filled directly by the user. 49 | * The hooks would be called at appropriate moments. All of them happen before the update 50 | * in the data structures. It is legal not to fill the hooks in (or set them as NULL), 51 | * they are called only if set. */ 52 | addr_hook_t add_hook; // When a new item is added 53 | addr_hook_t remove_hook; // When an item is removed 54 | addr_replace_hook_t replace_start_hook; // When the whole store is going to be replaced ‒ all old items are dropped (without calling remove_hook for each of them) 55 | addr_replace_hook_t replace_end_hook; // When the replacement is done 56 | /* The user data can be extracted by the hook from the passed store. It is not passed 57 | * as a parameter to the hooks directly. */ 58 | void *userdata; 59 | }; 60 | 61 | #endif 62 | 63 | #include "../../core/pluglib_macros.h" 64 | 65 | // memory pool to allocate from, name. The name is not copied, it is expected to exist for the whole lifetime of the structure. 66 | PLUGLIB_FUNC(diff_addr_store_init, struct diff_addr_store *, struct mem_pool *, const char *) 67 | // Copy (target, source, tmp_pool). The name, and mempool are not copied, only the active addresses. Hooks are not called. 68 | PLUGLIB_FUNC(diff_addr_store_cp, void, struct diff_addr_store *, const struct diff_addr_store *, struct mem_pool *) 69 | // (store, epoch, version, *orig_version) -> action 70 | PLUGLIB_FUNC(diff_addr_store_action, enum diff_store_action, struct diff_addr_store *, uint32_t, uint32_t, uint32_t *) 71 | PLUGLIB_FUNC(diff_addr_store_apply, enum diff_store_action, struct mem_pool *, struct diff_addr_store *, bool, uint32_t, uint32_t, uint32_t, const uint8_t *, size_t, uint32_t *) 72 | 73 | -------------------------------------------------------------------------------- /src/libs/test/Makefile: -------------------------------------------------------------------------------- 1 | RESTRICT := src/libs/test 2 | RELATIVE := ../../../ 3 | 4 | include $(RELATIVE)/Makefile 5 | -------------------------------------------------------------------------------- /src/libs/test/Makefile.dir: -------------------------------------------------------------------------------- 1 | LIBRARIES += src/libs/test/libpluglib_test 2 | libpluglib_test_MODULES := main 3 | -------------------------------------------------------------------------------- /src/libs/test/main.c: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2015 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #define PLUGLIB_DO_EXPORT 21 | #include "test.h" 22 | 23 | #include "../../core/util.h" 24 | 25 | void hello_world(void) { 26 | ulog(LLOG_WARN, "Hello new world!\n"); 27 | } 28 | 29 | struct pluglib *pluglib_info(void) { 30 | static struct pluglib_export *exports[] = { 31 | &hello_world_export, 32 | NULL 33 | }; 34 | static struct pluglib pluglib = { 35 | .name = "TestLib", 36 | .compat = 1, 37 | .version = 2, 38 | .exports = exports 39 | }; 40 | return &pluglib; 41 | } 42 | -------------------------------------------------------------------------------- /src/libs/test/test.h: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2015 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #include "../../core/pluglib.h" 21 | #include "../../core/pluglib_macros.h" 22 | PLUGLIB_FUNC(hello_world, void, void) 23 | -------------------------------------------------------------------------------- /src/master/Makefile: -------------------------------------------------------------------------------- 1 | RESTRICT := src/master 2 | RELATIVE := ../../ 3 | 4 | include $(RELATIVE)/Makefile 5 | -------------------------------------------------------------------------------- /src/master/Makefile.dir: -------------------------------------------------------------------------------- 1 | ifndef NO_MASTER 2 | SCRIPTS += src/master/collectmaster 3 | DOCS += src/master/collectmaster 4 | 5 | collectmaster_SOURCE = collect-master.py 6 | 7 | ifneq (,$(findstring ^$(RESTRICT),^src/master)) 8 | clean: clean-master clean-soxy 9 | all: all-soxy 10 | endif 11 | 12 | .PHONY: clean-master all-soxy 13 | 14 | clean-master: 15 | rm -f $(wildcard $(S)/src/master/*.pyc $(S)/src/master/*/*.pyc) 16 | 17 | include $(S)/src/master/authenticator/Makefile.dir 18 | 19 | all-soxy: 20 | mkdir -p $(O)/src/master/soxy 21 | +cd $(O)/src/master/soxy && qmake $(abspath $(S)/src/master/soxy/soxy.pro) && make -j$(J) 22 | mkdir -p $(O)/bin 23 | ln -fs $(O)/src/master/soxy/soxy $(O)/bin/soxy 24 | 25 | clean-soxy: 26 | mkdir -p $(O)/src/master/soxy 27 | +cd $(O)/src/master/soxy && qmake $(abspath $(S)/src/master/soxy/soxy.pro) && make distclean 28 | 29 | endif 30 | 31 | -------------------------------------------------------------------------------- /src/master/activity.py: -------------------------------------------------------------------------------- 1 | # 2 | # Ucollect - small utility for real-time analysis of network data 3 | # Copyright (C) 2013 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | # 5 | # This program is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation; either version 2 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License along 16 | # with this program; if not, write to the Free Software Foundation, Inc., 17 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | # 19 | 20 | import logging 21 | import database 22 | import threading 23 | 24 | logger = logging.getLogger(name='activity') 25 | 26 | __queue = [] 27 | # To be initialized on the first use 28 | __condition = None 29 | __thread = None 30 | 31 | def __keep_storing(): 32 | """ 33 | Run in separate thread. It keeps getting stuff from the queue and pushing it to the database. 34 | This effectively makes waiting for the databes commit asynchronous. 35 | """ 36 | global __condition 37 | global __queue 38 | logger.info('Activity thread started') 39 | run = True 40 | while run: 41 | actions = None 42 | with __condition: 43 | while not __queue: 44 | __condition.wait() 45 | actions = __queue 46 | __queue = [] 47 | 48 | try: 49 | with database.transaction() as t: 50 | for action in actions: 51 | if not action(t): 52 | run = False 53 | except Exception as e: 54 | logger.error("Unexpected exception in activity thread, ignoring: %s", e) 55 | logger.info('Activity thread terminated') 56 | 57 | def push(action): 58 | """ 59 | Push a function action into the queue to log some activity 60 | or similar. They are executed in order. 61 | """ 62 | global __queue 63 | global __condition 64 | global __thread 65 | if not __condition: 66 | logger.info('Starting the activity thread') 67 | # Initialize the thread machinery 68 | __condition = threading.Condition(threading.Lock()) 69 | __thread = threading.Thread(target=__keep_storing, name='activity') 70 | __thread.start() 71 | # Postpone it to separate thread 72 | with __condition: 73 | __queue.append(action) 74 | __condition.notify() 75 | 76 | def log_activity(client, activity): 77 | """ 78 | Log activity of a client. Pass name of the client (.cid()) and name 79 | of the activity (eg. "login"). 80 | """ 81 | logger.debug("Logging %s activity of %s", activity, client) 82 | def log(transaction): 83 | logger.debug("Pushing %s of %s", activity, client) 84 | transaction.execute("INSERT INTO activities (client, timestamp, activity) SELECT clients.id, CURRENT_TIMESTAMP AT TIME ZONE 'UTC', activity_types.id FROM clients CROSS JOIN activity_types WHERE clients.name = %s AND activity_types.name = %s", (client, activity)) 85 | return True 86 | push(log) 87 | 88 | def shutdown(): 89 | push(lambda transaction: False) 90 | -------------------------------------------------------------------------------- /src/master/amihacked/README: -------------------------------------------------------------------------------- 1 | This directory holds the script to provide data to the amihacked website. It is 2 | just that ‒ providing the data. The frontend lives elsewhere. 3 | 4 | There are three script entry-points: 5 | * export_repu: This is to be run on the archive.turris.cz machine. It generates 6 | exports of the whole history, in the form of several .csv.bz2 files. 7 | * repu_init: This one fills the initial data into a database. It expects to be 8 | run in a directory with the .csv.bz2 files from the above script. The 9 | database must already exist and the home directory should contain a db.ini file. 10 | * repu_add: This one adds data incrementaly, from files created by the 11 | archivist.pl. It expects the information to connect to the db in 12 | archivist.ini in the home directory. 13 | 14 | The other scripts are helpers and are not to be called directly. 15 | -------------------------------------------------------------------------------- /src/master/amihacked/bzchoose: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Run the pbzip2 compression or decompression, with -1 4 | 5 | PROG=pbzip2 6 | 7 | exec "$PROG" -1 "$@" 8 | -------------------------------------------------------------------------------- /src/master/amihacked/compact.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | use common::sense; 3 | 4 | # Sum consecutive lines with the same ip, date and kind together. Used to make 5 | # the primary export smaller (the db doesn't guarantee the similar lines to be 6 | # together, but they still often are). 7 | 8 | my ($ip, $date, $cnt, $kind); 9 | 10 | sub flush() { 11 | if ($ip) { 12 | print "$ip,$date,$cnt,$kind\n"; 13 | undef $ip; 14 | } 15 | } 16 | 17 | while (<>) { 18 | chomp; 19 | my ($new_ip, $new_date, $new_cnt, $new_kind) = split /,/; 20 | if (($new_ip ne $ip) or ($new_date ne $date) or ($new_kind ne $kind)) { 21 | flush; 22 | ($ip, $date, $cnt, $kind) = ($new_ip, $new_date, $new_cnt, $new_kind); 23 | } else { 24 | $cnt += $new_cnt; 25 | } 26 | } 27 | flush; 28 | -------------------------------------------------------------------------------- /src/master/amihacked/export-repu: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | q() { 6 | CMD="$1" 7 | NAME="$2" 8 | psql -q -d turris -1 -c "$CMD" | ./compact.pl | pbzip2 -5 >"$NAME".csv.bz2 9 | echo "Done $NAME on $(date +%X)" >&2 10 | } 11 | 12 | q "COPY (select remote, date(start_time), count(1) as attempt_count, 'ssh' from ssh_sessions where remote is not null group by date(start_time), remote) to STDOUT with CSV;" "ssh" & 13 | q "COPY (select remote, date, attempt_count, server from fake_attackers where attempt_count > 0) to STDOUT with CSV;" "telnet" & 14 | wait 15 | -------------------------------------------------------------------------------- /src/master/amihacked/jsonize.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | # This script takes series of CSV lines about attacks and constructs 3 | # corresponding JSON records for them. It expects the lines of one 4 | # IP address to be consequtive. 5 | # 6 | # The columns are: 7 | # * IP address 8 | # * date 9 | # * kind of attack 10 | # 11 | # The produced lines contain the IP address and JSON description. The 12 | # json object is indexed by the kind of attack and the date. This 13 | # holds the count of attacks on the day. 14 | # 15 | # eg: 16 | # 192.0.2.1 {"telnet": {"2015-10-05": 5}} 17 | # 18 | # All IPv6 attackers are aggregated into their /64 ranges as well, in addition 19 | # to individual records. 20 | use common::sense; 21 | use JSON qw(encode_json); 22 | use NetAddr::IP; 23 | 24 | my $last_ip; 25 | 26 | my $object; 27 | my %nets; 28 | 29 | sub flush() { 30 | return unless defined $last_ip; 31 | print $last_ip->canon(), "\t", encode_json $object, "\n"; 32 | undef $object; 33 | } 34 | 35 | my $ip6_strange = NetAddr::IP->new("f000::/8"); 36 | my $ip4_strange = NetAddr::IP->new("224.0.0.0/4"); 37 | 38 | while (<>) { 39 | chomp; 40 | my ($ip, $date, $cnt, $kind) = split /,/; 41 | $ip = NetAddr::IP->new($ip) or die "Bad IP: $ip\n"; 42 | # Skip addresses that are not interesting: 43 | # • Private IPv4 ranges (RFC 1918) 44 | # • Multicast IPv4 ranges 45 | # • f* IPv6 addresses (there are several kinds of strange addresses, like fe* local ones, ff*multicast ones, etc. 46 | next if $ip->is_rfc1918() or $ip->within($ip4_strange) or $ip->within($ip6_strange); 47 | if ($last_ip ne $ip) { 48 | flush; 49 | $last_ip = $ip; 50 | } 51 | $object->{$kind}->{$date} += $cnt; 52 | if ($ip->version() == 6) { 53 | my $net = NetAddr::IP->new($ip->canon(), 64)->network(); 54 | # Make sure it is stringified 55 | $nets{$net->canon()}->{$kind}->{$date} += $cnt; 56 | } 57 | } 58 | 59 | flush; 60 | 61 | while (my ($net, $obj) = each %nets) { 62 | print "$net/64\t", encode_json $obj, "\n"; 63 | } 64 | -------------------------------------------------------------------------------- /src/master/amihacked/process/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "process" 3 | version = "0.1.0" 4 | authors = ["Michal 'vorner' Vaner "] 5 | 6 | [dependencies] 7 | csv = "0.15" 8 | regex = "0.2" 9 | scoped-pool = "1" 10 | serde_json = "0.9" 11 | rustc-serialize = "0.3" 12 | fnv = "1" 13 | -------------------------------------------------------------------------------- /src/master/amihacked/repu_add: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Add incidents incrementally, from all the present csv files. 4 | 5 | set -e 6 | 7 | DIR=$(dirname "$0") 8 | 9 | if [ '!' -x "$DIR/process/target/release/process" ] ; then 10 | echo "Please compile and provide the process executable" 11 | exit 1 12 | fi 13 | 14 | FILES=$(ls *.csv) 15 | "$DIR/process/target/release/process" $FILES | "$DIR/to_db.pl" -d "$HOME/archivist.ini" 16 | rm $FILES 17 | -------------------------------------------------------------------------------- /src/master/amihacked/repu_init: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Import the data into database. 4 | 5 | set -e 6 | 7 | DIR=$(dirname "$0") 8 | 9 | if [ '!' -x "$DIR/process/target/release/process" ] ; then 10 | echo "Please compile and provide the process executable" 11 | exit 1 12 | fi 13 | 14 | "$DIR/process/target/release/process" telnet.csv.bz2 ssh.csv.bz2 | "$DIR/to_db.pl" -i -d "$HOME/db.ini" 15 | -------------------------------------------------------------------------------- /src/master/amihacked/split.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | 3 | # Sort the lines of input into several files, according to the IP at the front. 4 | # This helps parallelize the jsonize step (we can't split the IP address into two 5 | # groups, but if all the ones with the same prefix are together, this won't happen). 6 | use common::sense; 7 | 8 | mkdir "split"; 9 | my $last; 10 | my $f; 11 | 12 | while (<>) { 13 | my ($prefix) = /^(..)/; 14 | $prefix =~ s/:/_/g; 15 | if ($last ne $prefix) { 16 | close $f if $f; 17 | open $f, '|-', "gzip -1 >split/$prefix.csv.gz" or die "Failed to open split/$prefix.csv: $!\n"; 18 | $last = $prefix; 19 | } 20 | print $f $_; 21 | } 22 | 23 | close $f; 24 | -------------------------------------------------------------------------------- /src/master/amihacked/to_db.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | 3 | # This script adds provided attacker data to the database. It expects 4 | # the data to come in a form: 5 | # 6 | # ip-address JSON data 7 | # 8 | # (The separator is a tab) 9 | # 10 | # If -i is provided, the DB is cleaned and new data is inserted. If not, 11 | # the JSONs are summed together. 12 | use common::sense; 13 | use DBI; 14 | use JSON qw(encode_json decode_json); 15 | use Getopt::Long; 16 | use Config::IniFiles; 17 | 18 | my $initial; 19 | my $dbini = "db.ini"; 20 | 21 | GetOptions 22 | initial => \$initial, 23 | 'dbini=s' => \$dbini 24 | or die "Error parsing parameters\n"; 25 | 26 | my $cfg = Config::IniFiles->new(-file => $dbini) or die "Failed to read config: @Config::IniFiles::errors\n"; 27 | my ($dbname, $dbuser, $dbpass, $dbhost) = map { $cfg->val('amihacked', $_) } qw(db user passwd host); 28 | 29 | # Connect to the database. Use the username based authentication (→ no password) 30 | my $dbh = DBI->connect("dbi:Pg:dbname=$dbname" . ($dbhost ? ";host=$dbhost" : ""), $dbuser, $dbpass, {RaiseError => 1, AutoCommit => 0}); 31 | 32 | my $lookup = $dbh->prepare("SELECT data FROM amihacked_statistics WHERE address = ?"); 33 | my $insert = $dbh->prepare("INSERT INTO amihacked_statistics (address, data) VALUES (?, ?)"); 34 | my $update = $dbh->prepare("UPDATE amihacked_statistics SET data = ? WHERE address = ?"); 35 | 36 | # If we want to provide initial data, wipe the original 37 | $dbh->do("TRUNCATE amihacked_statistics") if $initial; 38 | 39 | while (<>) { 40 | chomp; 41 | my ($addr, $data) = split /\t/; 42 | my $previous; 43 | unless ($initial) { 44 | # Unless we fill the DB with initial data, try to look up previous value 45 | $lookup->execute($addr); 46 | ($previous) = $lookup->fetchrow_array; 47 | } 48 | if ($previous) { 49 | # Decode both old and new 50 | my $json_previous = decode_json $previous; 51 | my $json_data = decode_json $data; 52 | # Sum them together, fieldwise 53 | while (my ($kind, $kind_data) = each %$json_data) { 54 | while (my ($date, $cnt) = each %$kind_data) { 55 | # If the field is not there yet, it gets created (including all the necessary levels above it) 56 | $json_previous->{$kind}->{$date} += $cnt; 57 | } 58 | } 59 | # Store the new value 60 | $update->execute(encode_json $json_previous, $addr); 61 | } else { 62 | # Insert a new value 63 | $insert->execute($addr, $data); 64 | } 65 | } 66 | $dbh->commit; 67 | -------------------------------------------------------------------------------- /src/master/analyze/fake: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -ex 4 | 5 | LOCK_DIR=/tmp/fake_export.lock 6 | 7 | if ! mkdir "$LOCK_DIR" ; then 8 | echo "Already running?" 2>&1 9 | exit 1 10 | fi 11 | 12 | trap 'rm -rf "$LOCK_DIR"' EXIT INT QUIT TERM ABRT 13 | 14 | cd $HOME/ucollect/src/master/analyze 15 | ./fake.pl analyze.ini >$HOME/fake.csv.tmp 16 | chmod a+r $HOME/fake.csv.tmp 17 | mv $HOME/fake.csv.tmp $HOME/fake.csv 18 | -------------------------------------------------------------------------------- /src/master/analyze/fake.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | use common::sense; 3 | use DBI; 4 | use Config::IniFiles; 5 | use Net::Whois::IP qw(whoisip_query); 6 | use Geo::IP; 7 | use Storable; 8 | 9 | my $cfg = Config::IniFiles->new(-file => $ARGV[0]); 10 | my ($host, $db, $user, $passwd, $port) = map { $cfg->val('db', $_) } qw(host db user passwd port); 11 | my $dbh = DBI->connect("dbi:Pg:dbname=$db;host=$host;port=$port", $user, $passwd, { RaiseError => 1, AutoCommit => 0 }); 12 | 13 | my %turris_addresses; 14 | open my $addresses, '<', $ARGV[1] or die "Could not read address list '$ARGV[1]': $!\n"; 15 | while (my $line = <$addresses>) { 16 | chomp $line; 17 | my ($id, $addr) = split /;/, $line; 18 | $turris_addresses{$addr} = 1; 19 | } 20 | close $addresses; 21 | 22 | my $whois_cache; 23 | eval { 24 | $whois_cache = retrieve 'whois.cache'; 25 | }; 26 | 27 | my $t = time - 3600*24*10; # If the data is older than 10 days, it is considered expired 28 | delete @$whois_cache{grep { $whois_cache->{$_}->{time} < $t } keys %$whois_cache}; 29 | 30 | sub getwhois($) { 31 | my ($ip) = @_; 32 | if (not exists $whois_cache->{$ip}) { 33 | my $data = whoisip_query($ip); 34 | my %whois; 35 | while (my ($k, $v) = each %$data) { 36 | $v =~ s/\r//; 37 | chomp $v; 38 | $whois{lc $k} = $v; 39 | } 40 | $whois_cache->{$ip} = { 41 | time => time, 42 | data => \%whois 43 | }; 44 | } 45 | return $whois_cache->{$ip}->{data}; 46 | } 47 | 48 | my $forbidden; 49 | 50 | my @private = ( 51 | # Private IPv4 52 | qr/^10\./, 53 | (map qr/^172\.$_\./, 16..31), 54 | qr/^192\.168\./, 55 | # Link local IPv4 56 | qr/^169\.254\./, 57 | # IPv6 private and link local (almost) 58 | qr/^f[ed]..:/ 59 | ); 60 | 61 | my $result = $dbh->selectall_arrayref('SELECT server, remote, clients, score, mode FROM fake_blacklist'); 62 | $dbh->rollback; 63 | $dbh->disconnect; 64 | #local $\ = "\n"; 65 | #local $, = ","; 66 | my $gi = Geo::IP->new(GEOIP_MEMORY_CACHE); 67 | print "server,ip,clients,score,mode,as,nic,email,country\n"; 68 | for my $row (@$result) { 69 | my ($server, $remote, @rest) = @$row; 70 | if ($turris_addresses{$remote}) { 71 | warn "Address $remote belongs to a turris router\n"; 72 | $forbidden = 1; 73 | next; 74 | } 75 | if (grep { $remote =~ $_ } @private) { 76 | warn "Address $remote is private\n"; 77 | $forbidden = 1; 78 | next; 79 | } 80 | my $whois = getwhois $remote; 81 | my %whois = map { lc $_ => $whois->{$_} } keys %$whois; 82 | print (join ',', $server, $remote, @rest, '"' . ($whois{origin} // $whois{originas}) . '"', $whois{source}, $whois{"abuse-mailbox"} // $whois{"e-mail"} // $whois{orgabuseemail} // $whois{rabusemail}, $gi->country_code_by_addr($remote)); 83 | print "\n"; 84 | } 85 | 86 | store $whois_cache, 'whois.cache'; 87 | 88 | exit $forbidden; 89 | -------------------------------------------------------------------------------- /src/master/authenticator/Makefile: -------------------------------------------------------------------------------- 1 | RESTRICT := src/master/authenticator 2 | RELATIVE := ../../../ 3 | 4 | include $(RELATIVE)/Makefile 5 | -------------------------------------------------------------------------------- /src/master/authenticator/Makefile.dir: -------------------------------------------------------------------------------- 1 | ifndef NO_MASTER 2 | SCRIPTS += src/master/authenticator/authenticator 3 | #DOCS += src/master/authenticator/authenticator 4 | LIBRARIES += src/master/authenticator/atsha204 5 | 6 | authenticator_SOURCE = authenticator.py 7 | 8 | atsha204_MODULES := atsha204 9 | atsha204_SO_LIBS := crypto ssl unbound atsha204 10 | 11 | ifneq (,$(findstring ^$(RESTRICT),^src/master)) 12 | clean: clean-authenticator 13 | endif 14 | 15 | .PHONY: clean-authenticator 16 | 17 | clean-authenticator: 18 | rm -f $(wildcard $(S)/src/master/authenticator/*.pyc) 19 | endif 20 | -------------------------------------------------------------------------------- /src/master/authenticator/atsha204.c: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2013 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #include 21 | #include 22 | #include 23 | 24 | /* 25 | * As linking to other libraries from python extensions is tricky, 26 | * we do fully dynamic loading of the library with dlopen. 27 | * 28 | * Also, most of the error checking is done by assert instead of 29 | * python exceptions, out of laziness. 30 | */ 31 | 32 | static PyObject *atsha_do_hmac(PyObject *self, PyObject *args) { 33 | (void) self; 34 | int size_serial, size_key, size_challenge; 35 | unsigned char slot_id; 36 | const uint8_t *serial, *key, *challenge; 37 | if (!PyArg_ParseTuple(args, "bs#s#s#", &slot_id, &serial, &size_serial, &key, &size_key, &challenge, &size_challenge)) 38 | return NULL; 39 | assert(size_key == 32); 40 | assert(size_challenge == 32); 41 | struct atsha_handle *crypto = atsha_open_server_emulation(slot_id, serial, key); 42 | atsha_big_int challenge_s, response_s; 43 | challenge_s.bytes = 32; 44 | memcpy(challenge_s.data, challenge, 32); 45 | int result = atsha_challenge_response(crypto, challenge_s, &response_s); 46 | assert(result == ATSHA_ERR_OK); 47 | atsha_close(crypto); 48 | return Py_BuildValue("s#", response_s.data, (int) response_s.bytes); 49 | } 50 | 51 | static PyMethodDef atsha_methods[] = { 52 | {"hmac", atsha_do_hmac, METH_VARARGS, NULL}, 53 | {NULL} 54 | }; 55 | 56 | PyMODINIT_FUNC initatsha204(void) { 57 | Py_InitModule("atsha204", atsha_methods); 58 | } 59 | -------------------------------------------------------------------------------- /src/master/authenticator/authenticator.conf: -------------------------------------------------------------------------------- 1 | [main] 2 | dbuser: authenticator 3 | dbpasswd: 123456 4 | db: ucollect 5 | port: 8888 6 | -------------------------------------------------------------------------------- /src/master/authenticator/gen-email-passwd.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python2 2 | # 3 | # Ucollect - small utility for real-time analysis of network data 4 | # Copyright (C) 2013 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 5 | # 6 | # This program is free software; you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation; either version 2 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License along 17 | # with this program; if not, write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 19 | # 20 | 21 | from twisted.internet import protocol, reactor 22 | from twisted.protocols import basic 23 | import re 24 | import pgdb 25 | import ConfigParser 26 | import atsha204 27 | import sys 28 | 29 | # Command for challenge-response auth. It is "auth ID Challenge Response" or "half ..." 30 | auth = re.compile(r'^\s*(auth|half)\s+([0-9a-f]+)\s+([0-9a-f]+)\s+([0-9a-f]+)\s*$', re.IGNORECASE) 31 | 32 | if len(sys.argv) != 2: 33 | print "./authenticator.py config_file" 34 | sys.exit(1) 35 | 36 | config_data = ConfigParser.RawConfigParser() 37 | with open(sys.argv[1]) as f: 38 | config_data.readfp(f, sys.argv[1]) 39 | db = pgdb.connect(database=config_data.get('main', 'db'), user=config_data.get('main', 'dbuser'), password=config_data.get('main', 'dbpasswd')) 40 | cursor = db.cursor() 41 | 42 | for s in ['a']: 43 | cursor.execute("SELECT name, passwd, slot_id FROM clients WHERE name LIKE '0000000" + s + "%'") 44 | log_info = cursor.fetchone() 45 | challenge = ('EF' * 32).decode('hex') 46 | while log_info: 47 | print(log_info[0].upper() + ' ' + atsha204.hmac(log_info[2], log_info[0].decode('hex'), log_info[1].decode('hex'), challenge).encode('hex').lower()) 48 | log_info = cursor.fetchone() 49 | 50 | -------------------------------------------------------------------------------- /src/master/ca.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIGBzCCA++gAwIBAgIJANFfZI2j2HaYMA0GCSqGSIb3DQEBCwUAMIGZMQswCQYD 3 | VQQGEwJDWjEXMBUGA1UECAwOQ3plY2ggcmVwdWJsaWMxDzANBgNVBAcMBlByYWd1 4 | ZTEPMA0GA1UECgwGQ1ouTklDMQ0wCwYDVQQLDARMYWJzMRwwGgYDVQQDDBNUdXJy 5 | aXMgRW1lcmdlbmN5IENBMSIwIAYJKoZIhvcNAQkBFhNtaWNoYWwudmFuZXJAbmlj 6 | LmN6MB4XDTE2MDQwODA4MDAxNloXDTI2MDQwNjA4MDAxNlowgZkxCzAJBgNVBAYT 7 | AkNaMRcwFQYDVQQIDA5DemVjaCByZXB1YmxpYzEPMA0GA1UEBwwGUHJhZ3VlMQ8w 8 | DQYDVQQKDAZDWi5OSUMxDTALBgNVBAsMBExhYnMxHDAaBgNVBAMME1R1cnJpcyBF 9 | bWVyZ2VuY3kgQ0ExIjAgBgkqhkiG9w0BCQEWE21pY2hhbC52YW5lckBuaWMuY3ow 10 | ggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDJknvUe3uFUCyvr8+Ishcb 11 | 2NyW3lYYay5NTAfMFR9VvQaAESIKaUwQ4GNXo0E7DCMKEYxvPDvZZ0nSOaLFHws5 12 | gJ0QS62sG+pwZMQJnRhYhCFuEcBFajLG5bmkZcDYIJOiuarr52VVMKMEnTSL0DEI 13 | O7GXlEPSfwKYBcefDkOGTHTQM+eIoWppL1I0UuzWdNUHcvHB0alR2NTG240dsG4I 14 | mPr8RmfEO3L6wYT5E4WlQwRVcdFvQqcRU2lbO+brDS+1TiVatt/mKHzQrbvflkJI 15 | 9kBTGiED6I+xDPwKwfCOiHotEMSdgUx4eLjAxBX7bVDW+CuXkke5munlUeOaz8q7 16 | nwCmlwlBvzaS4ua4Tn8eQNEYlt00To5nUgt81OCa7WF/ylAfw3Ec/xsuOZ1Idv+N 17 | ld54U2lOJ3NfynIOvsLkHeIf8+i76eFAIm9ivby+dqEkCVbgMnelNP40M0UDZKW8 18 | z5ceFLWoyJ1bA/MXi2gYB2g+kA4ZgeQAXhLgdiUHxhuZ538cTH4d6iOWyS3MOHob 19 | 9XU+0lp2OExYiSus04crab+fjvCTz4kduDB8sLY1do+u/1d6gAyDPh2kLAuiOfLA 20 | nD331CglPmRdneyTrLHcQ7prYOcTNMmF2ZETV5HVd1+yYCr/8OSVcTGhgAFajgBN 21 | /Z+cXVGOO0DEHQ5YbidHJQIDAQABo1AwTjAdBgNVHQ4EFgQUrCj0B4Fr74JHAiG4 22 | 0LGd4c30oB4wHwYDVR0jBBgwFoAUrCj0B4Fr74JHAiG40LGd4c30oB4wDAYDVR0T 23 | BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAgEACzPel+BhqY2d6lfbQdCwdNGdoV7f 24 | hbdk3Yhd2Te03t7oU6k02welgIWRg86TtYxzQCi8HWmlWoDhq1UdYnLjheYjUQSR 25 | pT7EXTxhBnDKQRiSYsAZWpETSidQ/ipSffIO7VGVcpdJPH0BTd3BqwunUnfXhd1e 26 | DW3HBOOk5k6FUMBt9uM34NZ4S99klQDvu8q6EEBaNtOMgeYEPh5mnxqhuZICxmeh 27 | 4AtLagmOoLlypOKZgoJNGz+43MPPbXRb8eqj6N2E8pTBt1iB96qvYd/B1O5N38P2 28 | lTz4jLNBXfWieqV+yRnRMENsR9PW+7mEAn6WtM7IlFbDwMtgygh5Xj6PGTzwPCXl 29 | 186ljFV0W/2y0KCey4TWVqliKwymJPhQ7uRCbAjG22eY2GgoMSz6EIHIPJMfd5mL 30 | W0FiZUR/pJJe7+ljVKweNqfLs657/mXGGf+mArbe/Yv58B7XWLZrys6aLnXzfIa/ 31 | sb+EXqLrd431l6kmmylbPFSg5OwBjmo9rOQxcUOTXuiuKGGuzqMGsd7rhMnYtLKm 32 | 3AfcQPYEQIjhjxQDirKPx5LM15U0T2gP2hCkQ86KoWEGIG8tLxzgQD7FWoDYqEAv 33 | HTdnnlsjzPd9hJqyyo0eQcynI75rMdpKI5crvD3l9xJ04IRPRXipKwoX5XnWuAYy 34 | e3NhUtRgGGzR1ek= 35 | -----END CERTIFICATE----- 36 | -------------------------------------------------------------------------------- /src/master/collect-master.conf: -------------------------------------------------------------------------------- 1 | [main] 2 | ; How to connect to the DB 3 | dbuser: updater 4 | dbpasswd: 12345 5 | db: ucollect 6 | dbhost: localhost 7 | ; Port to listen on 8 | port: 5678 9 | port_compression: 5679 10 | ; The logging format. See http://docs.python.org/2/library/logging.html 11 | log_format: %(name)s@%(module)s:%(lineno)s %(asctime)s %(levelname)s %(message)s 12 | syslog_format: ucollect: %(name)s@%(module)s:%(lineno)s %(asctime)s %(levelname)s %(message)s 13 | ; Severity of the logs. One of TRACE, DEBUG, INFO, WARN, ERROR, CRITICAL 14 | log_severity: TRACE 15 | ; Where to log. - means stderr. 16 | log_file: master.log 17 | ; Maximum size of log file (in bytes) 18 | log_file_size: 134217728 19 | ; Maximum number of backup log files when rotated 20 | log_file_count: 5 21 | ; The SSL certificate 22 | cert = server.cert 23 | key = server.key 24 | ca = server.cert 25 | ; Where the authenticator lives 26 | authenticator_host: localhost 27 | authenticator_port: 8888 28 | fastpings: 29 | 0000000500000842 30 | ; rate limiter - bucket capacity 31 | ; actual capacity is rate_limit_number (specified for plugins) * this number 32 | rate_limiter_bucket_capacity: 5 33 | 34 | ; The plugins to load follow. Each name is the class to load and instantiate. 35 | 36 | [spoof_plugin.SpoofPlugin] 37 | 38 | [count_plugin.CountPlugin] 39 | ; The plugin that counts some stuff (packets of various properties, amount of data, ...) 40 | interval: 60 ; How often to store a snapshot, seconds. 41 | aggregate_delay: 5 ; How long to wait for answers after sending the query to store data into DB. 42 | 43 | [sniff.main.SniffPlugin] 44 | taskers = sniff.cert.Cert 45 | sniff.ping.Pinger 46 | parallel_limit = 20 47 | task_timeout = 1 48 | interval = 1 49 | # 15 seconds between starting tasks, so they don't flood all at once 50 | start_interval = 15 51 | ping_interval = 3600 52 | ping_batchsize = 20 53 | cert_interval = 3600 54 | cert_batchsize = 20 55 | 56 | [bandwidth_plugin.BandwidthPlugin] 57 | interval: 900 ; How often to store a snapshot, seconds. 58 | aggregate_delay: 5 ; How long to wait for answers from clients before working on them. 59 | 60 | [flow_plugin.FlowPlugin] 61 | # number of messages that client is allowed to send... 62 | rate_limit_number = 5 63 | # ... in some interval [seconds] 64 | rate_limit_interval = 60 65 | [fwup_plugin.FWUpPlugin] 66 | 67 | [refused_plugin.RefusedPlugin] 68 | version = 1 69 | finished_limit = 10 70 | send_limit = 3 71 | undecided_limit = 50 72 | timeout = 30000 73 | max_age = 120000 74 | # number of messages that client is allowed to send... 75 | rate_limit_number = 5 76 | # ... in some interval [seconds] 77 | rate_limit_interval = 60 78 | 79 | [fake_plugin.FakePlugin] 80 | version = 1 81 | max_age = 60000 82 | max_size = 2048 83 | max_attempts = 2 84 | throttle_holdback = 120000 85 | # number of messages that client is allowed to send... 86 | rate_limit_number = 5 87 | # ... in some interval [seconds] 88 | rate_limit_interval = 60 89 | 90 | -------------------------------------------------------------------------------- /src/master/crash_plugin.py: -------------------------------------------------------------------------------- 1 | # 2 | # Ucollect - small utility for real-time analysis of network data 3 | # Copyright (C) 2013 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | # 5 | # This program is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation; either version 2 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License along 16 | # with this program; if not, write to the Free Software Foundation, Inc., 17 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | # 19 | 20 | import plugin 21 | 22 | class CrashPlugin(plugin.Plugin): 23 | def client_connected(self, client): 24 | self.send('', client.cid()) 25 | 26 | def name(self): 27 | return 'Crash' 28 | -------------------------------------------------------------------------------- /src/master/dbscripts/AddrStoreBuild.pm: -------------------------------------------------------------------------------- 1 | package AddrStoreBuild; 2 | use common::sense; 3 | use base 'Exporter'; 4 | use DBI; 5 | use Config::IniFiles; 6 | use Fcntl ':flock'; 7 | use Socket qw(getaddrinfo getnameinfo NI_NUMERICHOST); 8 | 9 | our @EXPORT = qw(single_instance blacklist_load db_connect addr_store_content $cfg); 10 | 11 | # Global, so it stays open for the lifetime of the script 12 | my $lockfile; 13 | 14 | # Ensure there's only one instance of the script running. Any following one 15 | # will exit. It is expected the next run of the script will catch up by creating 16 | # slightly larger batch in the DB. 17 | sub single_instance($) { 18 | my ($lock_file_name) = @_; 19 | open $lockfile, '>>', $lock_file_name or die "Could not open lock file '$lock_file_name': $!\n"; 20 | flock $lockfile, LOCK_EX | LOCK_NB or exit; 21 | } 22 | 23 | our $cfg = Config::IniFiles->new(-file => $ARGV[0]); 24 | 25 | # Read the blacklist file, resolve all the names there and provide hashref with addresses 26 | sub blacklist_load() { 27 | my $blacklist_file = $cfg->val('blacklist', 'file'); 28 | my %blacklist; 29 | open my $black_open, '<', $blacklist_file or die "Couldn't read blacklist from $blacklist_file: $!\n"; 30 | 31 | while (my $line = <$black_open>) { 32 | # Remove comments and whitespace 33 | chomp $line; 34 | $line =~ s/#.*//; 35 | $line =~ s/^\s*//; 36 | $line =~ s/\s*$//; 37 | next unless length $line; 38 | 39 | my ($err, @addresses) = getaddrinfo $line, 0; 40 | die "Couldn't resolve $line: $err\n" if $err; 41 | for my $addr (@addresses) { 42 | my ($err, $ip) = getnameinfo $addr->{addr}, NI_NUMERICHOST; 43 | die "Couldn't print address for $line: $err\n" if $err; 44 | $blacklist{$ip} = 1; 45 | } 46 | } 47 | return \%blacklist; 48 | } 49 | 50 | my $dbh; 51 | 52 | # Create connection to the database and return the handle 53 | sub db_connect() { 54 | # Already connected. Just reuse the connection. 55 | return $dbh if $dbh; 56 | # Make a new connection 57 | my ($host, $db, $user, $passwd, $port) = map { $cfg->val('db', $_) } qw(host db user passwd port); 58 | return $dbh = DBI->connect("dbi:Pg:dbname=$db;host=$host;port=$port", $user, $passwd, { RaiseError => 1, AutoCommit => 0 }); 59 | } 60 | 61 | # Read the current content of an address store. Return the maximal epoch and a hashref containing the addresses 62 | sub addr_store_content($$$) { 63 | my ($table, $name_column, $name) = @_; 64 | my $dbh = db_connect; 65 | my ($max_epoch) = $dbh->selectrow_array("SELECT COALESCE(MAX(epoch), 1) FROM $table WHERE $name_column = ?", undef, $name); 66 | my $existing = $dbh->selectall_arrayref(" 67 | SELECT 68 | $table.address 69 | FROM 70 | $table 71 | JOIN 72 | (SELECT 73 | MAX(epoch) AS epoch, 74 | MAX(version) AS version, 75 | address 76 | FROM 77 | $table 78 | WHERE 79 | $name_column = ? AND 80 | epoch = ? 81 | GROUP BY 82 | address) 83 | AS selector 84 | ON 85 | $table.version = selector.version AND 86 | $table.epoch = selector.epoch AND 87 | $table.address = selector.address 88 | WHERE 89 | $table.add AND 90 | $name_column = ?", undef, $name, $max_epoch, $name); 91 | my %existing = map { $_->[0] => 1 } @$existing; 92 | return ($max_epoch, \%existing); 93 | } 94 | 95 | 1; 96 | -------------------------------------------------------------------------------- /src/master/dbscripts/Tagger/Flows.pm: -------------------------------------------------------------------------------- 1 | package Tagger::Flows; 2 | use common::sense; 3 | use AddrStoreBuild qw($cfg); 4 | 5 | sub prepare_tags($@) { 6 | my ($dbh, @tags) = @_; 7 | my %tags; 8 | # Read tags stored in passed files 9 | for my $fname (@tags) { 10 | open my $f, '<', $fname or die "Couldn't read $fname: $!\n"; 11 | while (<$f>) { 12 | chomp; 13 | next if /^\s*(|#.*)$/; # Empty lines and comments 14 | next if /^\s*create\s/; 15 | if (my ($rule, $ip, $port) = /^\s*add\s+(\S+)\s+(\S+),(?:udp|tcp):(\d+)\s*$/) { 16 | $tags{$ip}->{ports}->{$port} = $rule; 17 | $tags{$ip}->{values} //= $rule; # Only if there wasn't a global one before 18 | } elsif (my ($rule, $ip) = /^\s*add\s+(\S+)\s+(\S+)\s*$/) { 19 | $tags{$ip}->{values} = $rule; # Override one from port-specific 20 | } else { 21 | die "Can't parse $_\n"; 22 | } 23 | } 24 | } 25 | # Read the fake server generated blacklist and use it as tags as well 26 | my $fake_blacklist = $dbh->prepare('SELECT server, remote FROM fake_blacklist_tmp'); 27 | $fake_blacklist->execute; 28 | while (my ($server, $ip) = $fake_blacklist->fetchrow_array) { 29 | $tags{$ip}->{values} = "fake-$server"; 30 | } 31 | print "Prepared flow tags\n"; 32 | return \%tags; 33 | } 34 | 35 | sub perform($$) { 36 | my ($dbh, $tags) = @_; 37 | 38 | my $read = $dbh->prepare('SELECT id, ip_remote, port_remote FROM biflows WHERE tagged_on IS NULL LIMIT 100000'); 39 | my $update = $dbh->prepare('UPDATE biflows SET tag = ?, tagged_on = ? WHERE id = ?'); 40 | 41 | my $found = 1; 42 | my $count = 0; 43 | while ($found) { 44 | my $batch = 0; 45 | my $tstamp = $dbh->selectrow_array("SELECT CURRENT_TIMESTAMP AT TIME ZONE 'UTC'"); 46 | $read->execute; 47 | undef $found; 48 | while (my ($id, $ip, $port) = $read->fetchrow_array) { 49 | my $tag = $tags->{$ip}->{ports}->{$port} // $tags->{$ip}->{values} // '?'; 50 | $update->execute($tag, $tstamp, $id); 51 | $found = 1; 52 | $count ++; 53 | $batch ++; 54 | } 55 | $dbh->commit; 56 | print "Done $count flows\n"; 57 | last if $batch < 100000; 58 | } 59 | } 60 | 61 | 1; 62 | -------------------------------------------------------------------------------- /src/master/dbscripts/activities: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | . ./dbconfig 4 | 5 | echo 'SELECT clients.name, activities.timestamp, activity_types.name FROM activities JOIN clients ON activities.client = clients.id JOIN activity_types ON activities.activity = activity_types.id ORDER BY timestamp;' | psql -U "$DBREAD" -d "$DB" -F' ' -q -t -A 6 | -------------------------------------------------------------------------------- /src/master/dbscripts/blacklist: -------------------------------------------------------------------------------- 1 | api.turris.cz 2 | 8.8.8.8 3 | # These are shodan's. 4 | shodan.io 5 | census1.shodan.io 6 | census2.shodan.io 7 | census3.shodan.io 8 | census4.shodan.io 9 | census5.shodan.io 10 | census6.shodan.io 11 | census7.shodan.io 12 | census8.shodan.io 13 | census9.shodan.io 14 | census10.shodan.io 15 | census11.shodan.io 16 | census12.shodan.io 17 | atlantic.census.shodan.io 18 | atlantic.dns.shodan.io 19 | pacific.census.shodan.io 20 | rim.census.shodan.io 21 | blog.shodan.io 22 | hello.data.shodan.io 23 | www.shodan.io 24 | -------------------------------------------------------------------------------- /src/master/dbscripts/capture_stats: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | . ./dbconfig 4 | 5 | echo 'SELECT clients.name, count_snapshots.timestamp, capture_stats.interface, capture_stats.captured, capture_stats.dropped, capture_stats.dropped_driver FROM capture_stats JOIN count_snapshots ON count_snapshots.id = capture_stats.snapshot JOIN clients ON clients.id = count_snapshots.client ORDER BY count_snapshots.timestamp, clients.name;' | psql -U "$DBREAD" -d "$DB" -F' ' -q -t -A 6 | -------------------------------------------------------------------------------- /src/master/dbscripts/counts: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | . ./dbconfig 4 | 5 | echo 'SELECT clients.name, count_snapshots.timestamp, count_types.name, counts.count, counts.size FROM counts JOIN count_snapshots ON counts.snapshot = count_snapshots.id JOIN clients ON clients.id = count_snapshots.client JOIN count_types ON count_types.id = counts.type ORDER BY count_snapshots.timestamp, clients.name, count_types.id;' | psql -U "$DBREAD" -d "$DB" -F' ' -q -t -A 6 | -------------------------------------------------------------------------------- /src/master/dbscripts/cvut-export.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | REFRESH MATERIALIZED VIEW permutation; 3 | REFRESH MATERIALIZED VIEW biflow_loc_ip_permutation; 4 | \copy (SELECT * FROM biflows_anonymized) TO 'biflows.csv' WITH (FORMAT CSV); 5 | \copy (SELECT session_id, ts AS timestamp, success, command FROM ssh_commands) TO 'ssh_commands.csv' WITH (FORMAT CSV); 6 | \copy (SELECT * FROM ssh_sessions_anonymized) TO 'ssh_sessions.csv' WITH (FORMAT CSV); 7 | \copy (SELECT * FROM fake_logs_anonymized) TO 'fake_logs.csv' WITH (FORMAT CSV); 8 | REFRESH MATERIALIZED VIEW permutation WITH NO DATA; 9 | REFRESH MATERIALIZED VIEW biflow_loc_ip_permutation WITH NO DATA; 10 | END; 11 | -------------------------------------------------------------------------------- /src/master/dbscripts/dbconfig: -------------------------------------------------------------------------------- 1 | DBPARAMS= 2 | # The user for reading things from the database 3 | DBREAD=read 4 | # User allowed to do anything (eg. creating tables, etc). 5 | DBADMIN=ucollect 6 | # User used for removing old things from database 7 | DBCLEANER=cleaner 8 | # User for updating from ucollect 9 | DBUPDATER=updater 10 | # User for jenkins ‒ updates the passwords 11 | DBJENKINS=jenkins 12 | # Authenticator that has access to the keys 13 | DBAUTHENTICATOR=authenticator 14 | # Something that aggregates the data to archives 15 | DBARCHIVIST=archivist 16 | # Tagger of flows according to firewall rules 17 | DBFLOWTAGGER=ucollect 18 | # The Kippo feeder 19 | DBKIPPO=ucollect 20 | 21 | # Name of the database 22 | DB=ucollect 23 | 24 | # Purge items older than this many days 25 | CLEAN_DAYS=14 26 | 27 | -------------------------------------------------------------------------------- /src/master/dbscripts/dupaddr.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | # Ucollect - small utility for real-time analysis of network data 3 | # Copyright (C) 2015 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | # 5 | # This program is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation; either version 2 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License along 16 | # with this program; if not, write to the Free Software Foundation, Inc., 17 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | 19 | # This script checks the database if there are duplicate addresses (eg. address 20 | # added twice or removed twice) in one of the addr_store tables. Of course it 21 | # considers each store separately (and doesn't consider sequences of 22 | # add-remove-add a problem). 23 | use common::sense; 24 | use DBI; 25 | use Config::IniFiles; 26 | 27 | my $cfg = Config::IniFiles->new(-file => $ARGV[0]); 28 | my ($host, $db, $user, $passwd, $port) = map { $cfg->val('db', $_) } qw(host db user passwd port); 29 | my $dbh = DBI->connect("dbi:Pg:dbname=$db;host=$host;port=$port", $user, $passwd, { RaiseError => 1, AutoCommit => 0 }); 30 | 31 | sub check_table($$) { 32 | my ($table, $name_column) = @_; 33 | # Get all names of the stores and their corresponding epochs 34 | my $stores = $dbh->selectall_hashref("SELECT DISTINCT $name_column, epoch FROM $table", [$name_column, 'epoch']); 35 | # For each in the above tupple... 36 | for my $name (sort keys %$stores) { 37 | my $store = $stores->{$name}; 38 | for my $epoch (sort { $a <=> $b } keys %$store) { 39 | print "Doing $name/$epoch\n"; 40 | # To track what is active AND when it got there or removed 41 | my (%added, %removed); 42 | # Get the content in chronological order and go through them, simulating additions and removals 43 | my $query = $dbh->prepare("SELECT address, add, version FROM $table WHERE $name_column = ? AND epoch = ? ORDER BY version"); 44 | $query->execute($name, $epoch); 45 | while (my ($address, $add, $version) = $query->fetchrow) { 46 | if ($add) { 47 | print "+ $address $version/$added{$address}\n" if exists $added{$address}; 48 | $added{$address} = $version; 49 | delete $removed{$address}; 50 | } else { 51 | print "- $address $version/", ($removed{$address} // "-----"), "\n" unless exists $added{$address}; 52 | $removed{$address} = $version; 53 | delete $added{$address} 54 | } 55 | } 56 | } 57 | } 58 | } 59 | 60 | check_table 'flow_filters', 'filter'; 61 | check_table 'fwup_addresses', 'set'; 62 | -------------------------------------------------------------------------------- /src/master/dbscripts/export_fake: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | DATE=$(date -d 'yesterday' +%Y-%m-%d) 4 | DIR=$(dirname "$0") 5 | if [ "$DIR" = "." ] ; then 6 | DIR=$(pwd) 7 | fi 8 | 9 | cd ~/fake-exports 10 | # It is generated with the date +1, so eat it one day sooner 11 | find -type f -mtime +7 -delete 12 | "$DIR"/export_fake.pl "$DIR"/db.ini $DATE 13 | chmod a+r *.csv 14 | -------------------------------------------------------------------------------- /src/master/dbscripts/export_fake.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | use common::sense; 3 | use DBI; 4 | 5 | # Look for libraries also in the same directory as the script lives 6 | use FindBin; 7 | use lib $FindBin::Bin; 8 | 9 | use AddrStoreBuild; 10 | 11 | my @priv_rex = ( 12 | qr/^192\.168\./, 13 | qr/^10\./, 14 | qr/^169\.254\./, 15 | map { 16 | my $num = $_; 17 | qr/^172\.$num\./ 18 | } 16..31 19 | ); 20 | 21 | # Don't confuse with a blacklist we're building. This is a blacklist for analysis ‒ „ignore these addresses when looking for bad guys“ 22 | my $omit_addresses = blacklist_load; 23 | 24 | my $dbh = db_connect; 25 | 26 | shift @ARGV; # Eat the config file path 27 | 28 | my $stm = $dbh->prepare("SELECT server, remote, remote_port, local, local_port, start_time_utc, end_time_utc FROM fake_bad_connections WHERE DATE(COALESCE(end_time_utc, start_time_utc)) = ? ORDER BY server, remote, local"); 29 | for my $d (@ARGV) { 30 | my %files; 31 | $stm->execute($d); 32 | LINE: 33 | while (my ($server, @data) = $stm->fetchrow_array) { 34 | my $remote = $data[0]; 35 | my $local = $data[2]; 36 | for my $rex (@priv_rex) { 37 | next LINE if $local =~ $rex; 38 | } 39 | next LINE if exists $omit_addresses->{$remote}; 40 | if (not exists $files{$server}) { 41 | open my $file, '>:utf8', "$server-$d.csv" or die "Couldn't write file '$server-$d.csv': $!\n"; 42 | print $file "remote,remote_port,local,local_port,start,end\n"; 43 | $files{$server} = $file; 44 | } 45 | my $file = $files{$server}; 46 | print $file (join ",", @data), "\n"; 47 | } 48 | close $_ for values %files; 49 | } 50 | 51 | # The raw blacklist with everything 52 | my $stm = $dbh->prepare("SELECT server, remote, clients, score, mode FROM fake_blacklist ORDER BY server, remote"); 53 | $stm->execute; 54 | open my $file, '>:utf8', "blacklist.csv" or die "Couln't write the blacklist file: $!\n"; 55 | print $file "server,remote,clients,score,mode\n"; 56 | while (my @data = $stm->fetchrow_array) { 57 | my $remote = $data[1]; 58 | next if exists $omit_addresses->{$remote}; 59 | print $file (join ",", @data), "\n"; 60 | } 61 | close $file; 62 | $dbh->rollback; 63 | -------------------------------------------------------------------------------- /src/master/dbscripts/groups.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | use common::sense; 3 | use DBI; 4 | use Data::Dumper; 5 | 6 | my $dbh = DBI->connect("dbi:Pg:dbname=turris", '', '', { RaiseError => 1, AutoCommit => 0 }); 7 | 8 | my @levels = (2, 4, 8, 16, 32); 9 | my @groups; 10 | for my $level (@levels) { 11 | push @groups, map "rand-$level-$_", (1..$level); 12 | } 13 | print Dumper \@groups; 14 | 15 | #$dbh->do('INSERT INTO groups (name) VALUES (?)', undef, $_) for @groups; 16 | 17 | my $ids = $dbh->selectall_arrayref("SELECT id FROM clients WHERE name LIKE '0000000a%'"); 18 | 19 | for my $id_row (@$ids) { 20 | my ($id) = @$id_row; 21 | #$dbh->do('INSERT INTO group_members (client, in_group) SELECT ?, id FROM groups WHERE name = ?', undef, $id, 'all'); 22 | for my $level (@levels) { 23 | my $rand = 1 + int(rand($level)); 24 | print "$id into $level-$rand\n"; 25 | die "Rand wrong" if $rand > $level; 26 | $dbh->do('INSERT INTO group_members (client, in_group) SELECT ?, id FROM groups WHERE name = ?', undef, $id, "rand-$level-$rand"); 27 | } 28 | } 29 | 30 | $dbh->commit; 31 | -------------------------------------------------------------------------------- /src/master/dbscripts/log_uptime.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | use common::sense; 3 | use DBI; 4 | use Config::IniFiles; 5 | use Data::Dumper; 6 | use DateTime::Duration; 7 | use List::Util qw(sum); 8 | 9 | my $cfg = Config::IniFiles->new(-file => $ARGV[0]); 10 | shift @ARGV; 11 | my ($host, $db, $user, $passwd, $port) = map { $cfg->val('db', $_) } qw(host db user passwd port); 12 | my $column = $cfg->val('activities', 'time_column') // 'timestamp'; 13 | my $dbh = DBI->connect("dbi:Pg:dbname=$db;host=$host;port=$port", $user, $passwd, { RaiseError => 1, AutoCommit => 0 }); 14 | 15 | my $sub_query; 16 | $sub_query = " AND clients.name IN (" . (join ', ', map "%", @ARGV) . ")" if (@ARGV); 17 | my $query = "SELECT date_part('epoch', $column), clients.name, activity_types.name FROM activities JOIN clients ON clients.id = activities.client JOIN activity_types ON activity_types.id = activities.activity WHERE$sub_query activity_types.name IN ('login', 'logout') ORDER BY clients.name, $column"; 18 | 19 | my $compiled = $dbh->prepare($query); 20 | $compiled->execute(@ARGV); 21 | 22 | my ($last_client, $last_login); 23 | my @client; 24 | my @total; 25 | 26 | sub client_push($) { 27 | push @client, $_[0] - $last_login if $last_login; 28 | } 29 | 30 | my $epoch = DateTime->from_epoch(epoch => 0); 31 | 32 | sub form($@) { 33 | my ($client, @data) = @_; 34 | return unless @data; 35 | @data = sort { $a <=> $b } @data; 36 | my ($min, $max, $avg, $mean) = map { 37 | my @d = (DateTime->from_epoch(epoch => $_) - $epoch)->in_units(qw(days hours)); 38 | sprintf "%2ddays %2dhours", @d; 39 | } map { 40 | $_->(@data) 41 | } sub { $_[0] }, sub { $_[-1] }, sub { 42 | (sum @_) / (scalar @_); 43 | }, sub { 44 | ($_[@_ / 2] + $_[(@_ - 1) / 2]) / 2; 45 | }; 46 | print "$client\t$min\t$max\t$avg\t$mean\n"; 47 | } 48 | 49 | sub client_process() { 50 | client_push time; 51 | form $last_client, @client; 52 | push @total, @client; 53 | @client = (); 54 | } 55 | 56 | while (my ($time, $client, $action) = $compiled->fetchrow_array) { 57 | if ($client ne $last_client) { 58 | client_process; 59 | $last_client = $client; 60 | undef $last_login; 61 | } 62 | client_push $time; 63 | undef $last_login; 64 | $last_login = $time if $action eq 'login'; 65 | } 66 | 67 | client_process; 68 | 69 | form "Total\t\t", @total; 70 | 71 | $dbh->rollback; 72 | -------------------------------------------------------------------------------- /src/master/dbscripts/online.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | use common::sense; 3 | use Term::ANSIColor; 4 | use Date::Parse; 5 | use Getopt::Long; 6 | use DBI; 7 | 8 | $ENV{TZ} = 'UTC'; 9 | 10 | my $dbh = DBI->connect("dbi:Pg:dbname=routers", "ucollect", "", { RaiseError => 1, AutoCommit => 0 }); 11 | 12 | my $green = color 'bold green'; 13 | my $red = color 'bold red'; 14 | my $blue = color 'bold blue'; 15 | my $reset = color 'reset'; 16 | 17 | my $forever; 18 | my $all; 19 | my $ocount; 20 | 21 | GetOptions 22 | forever => \$forever, 23 | all => \$all, 24 | ocount => \$ocount 25 | or die "Bad parameters\n"; 26 | 27 | # TODO: This query is probably slower than we would like (especially the second nested 28 | # SELECT, because it goes through a lot of counts data). No idea what this'll do on 29 | # the real DB with thousand of clients, but maybe the DB will handle it. If not, we'll 30 | # see what can be done about it. 31 | my $stmt = $dbh->prepare(<<'ENDSQL'); 32 | SELECT 33 | aggregate.name, 34 | aggregate.devel_note, 35 | aggregate.tag, 36 | aggregate.last, 37 | activity_types.name 38 | FROM ( 39 | SELECT 40 | clients.id, 41 | clients.name, 42 | clients.devel_note, 43 | clients.tag, 44 | MAX(activities.timestamp) AS last 45 | FROM 46 | clients 47 | LEFT OUTER JOIN 48 | activities ON clients.id = activities.client 49 | GROUP BY clients.id 50 | ) AS aggregate 51 | LEFT OUTER JOIN activities ON 52 | activities.client = aggregate.id 53 | AND activities.timestamp = aggregate.last 54 | LEFT OUTER JOIN activity_types ON 55 | activities.activity = activity_types.id 56 | ORDER BY 57 | aggregate.tag ASC, 58 | aggregate.devel_note ASC, 59 | aggregate.name ASC; 60 | ENDSQL 61 | while (1) { 62 | my ($online, $offline) = (0, 0); 63 | $stmt->execute; 64 | print "${blue}ID\t\t\tStatus\tAct.\tTime\t\t\t\tNote$reset\n" unless $ocount; 65 | my $now = time; 66 | my $stuck = $now - 3600; 67 | while (my ($name, $note, $tag, $last, $activity) = $stmt->fetchrow_array) { 68 | my $status = (defined $activity && $activity ne 'logout') ? "${green}online" : "${red}offline"; 69 | $activity //= '----'; 70 | my $time = str2time($last // '0'); 71 | if ($time <= $stuck) { 72 | $status = "${red}stuck" unless $status eq "${red}offline"; 73 | $activity = "$red$activity"; 74 | } 75 | if ($status eq "${green}online") { 76 | $online ++; 77 | } else { 78 | $offline ++; 79 | } 80 | next unless $all or defined $last; 81 | $last //= '----'; 82 | printf "%-16s\t%s$reset\t%s\t%-30s\t%10s$reset\t%10s\t%s\n", $name, $status, $activity, $last, $note unless $ocount; 83 | } 84 | print "${green}Online:\t\t\t$online\n${red}Offline:\t\t$offline$reset\n" unless $ocount; 85 | 86 | print "$online\n" if $ocount; 87 | $dbh->rollback; 88 | last if not $forever; 89 | sleep 15; 90 | print "\033[2J" unless $ocount; 91 | } 92 | -------------------------------------------------------------------------------- /src/master/dbscripts/parse-atsha.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | use common::sense; 3 | use Data::Dumper; 4 | 5 | $/ = "\n\n"; 6 | my @input = ; 7 | 8 | my $slot = 0; 9 | my $tag = 'omnia-v1.0'; 10 | my $start = 3; 11 | 12 | print "BEGIN;\n"; 13 | print "Be sure to use the atshakeys user.\n"; 14 | print "\\q\n"; 15 | 16 | for my $input (@input) { 17 | my @data = split /\n/, $input; 18 | my $name = $data[0]; 19 | my @keys = @data[1..16]; 20 | s/\s//g for @keys; 21 | my $id = join '', @data[17..18]; 22 | $id =~ s/[: ]//g; 23 | $id = lc($id); 24 | my $data = $data[$slot + 1]; 25 | print "INSERT INTO clients (name, passwd, mechanism, slot_id, tag, devel_note) VALUES('$id', '$keys[$slot]', 'A', $slot, '$tag', '$name');\n"; 26 | # print "INSERT INTO groups (name) VALUES('T$start');\n"; 27 | print "INSERT INTO group_members (client, in_group) SELECT clients.id, groups.id FROM clients CROSS JOIN groups WHERE clients.name = '$id' AND groups.name = 'all';\n"; 28 | # print "INSERT INTO group_members (client, in_group) SELECT clients.id, groups.id FROM clients CROSS JOIN groups WHERE clients.name = '$id' AND groups.name = 'turris';\n"; 29 | # print "INSERT INTO group_members (client, in_group) SELECT clients.id, groups.id FROM clients CROSS JOIN groups WHERE clients.name = '$id' AND groups.name = 'T$start';\n"; 30 | $start ++; 31 | } 32 | print "COMMIT;\n"; 33 | -------------------------------------------------------------------------------- /src/master/dbscripts/pkg-unpack: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | DIR="$1" 6 | NAME="$2" 7 | VERSION="$3" 8 | 9 | mkdir -p "$DIR" 10 | cd "$DIR" 11 | gunzip -c | tar x 12 | tar xf data.tar.gz 13 | LIBNAME="$(echo $NAME | tr - _)" 14 | LIBNAME="usr/lib/libplugin_${LIBNAME}_$VERSION.so" 15 | sha256sum "$LIBNAME" 16 | cd / 17 | rm -rf "$DIR" 18 | -------------------------------------------------------------------------------- /src/master/dbscripts/plugin_hashes: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | cd $(dirname "$0") 5 | ./plugin_hashes.pl --base-url https://repo.turris.cz --board turris --board omnia \ 6 | --branch dev-honza --branch dev-japa --branch dev-karel --branch dev-michal --branch dev-miska --branch dev-robin --branch dev-stepan --branch dev-tms --branch master --branch next --branch nightly --branch rc --branch test --branch dev-jirka --branch dev-martin --branch stable \ 7 | --suffix packages --suffix packages/turrispackages \ 8 | --package ucollect-flow --package ucollect-counts --package ucollect-bandwidth --package ucollect-fake --package ucollect-refused --package ucollect-sniff --package ucollect-spoof --package ucollect-count --package ucollect-fwup 9 | -------------------------------------------------------------------------------- /src/master/dbscripts/purge: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -ex 4 | 5 | . ./dbconfig 6 | 7 | TABLES='activities count_snapshots bandwidth bandwidth_stats refused fake_logs plugin_history' 8 | BATCH_TABLES='pings certs spoof' 9 | DATE=$(date -d "$CLEAN_DAYS days ago" "+'%Y-%m-%d'") 10 | 11 | ( 12 | echo "BEGIN;" 13 | # Duplicate the last plugin history event. This way, even if 14 | # the client is connected for a long time, we won't lose all 15 | # the events and we shall know what its state is now or a day ago 16 | echo "INSERT INTO plugin_history (client, name, timestamp, version, hash, active) SELECT DISTINCT ph.client, ph.name, CURRENT_TIMESTAMP AT TIME ZONE 'UTC', ph.version, ph.hash, ph.active FROM plugin_history AS ph JOIN (SELECT client, name, MAX(timestamp) AS timestamp FROM plugin_history GROUP BY client, name) AS latest ON ph.client = latest.client AND ph.name = latest.name AND ph.timestamp = latest.timestamp;" 17 | for TABLE in $TABLES ; do 18 | echo "DELETE FROM $TABLE WHERE timestamp < $DATE;" 19 | done 20 | for TABLE in $BATCH_TABLES ; do 21 | echo "DELETE FROM $TABLE WHERE batch < $DATE;" 22 | done 23 | echo "DELETE FROM celery_taskmeta WHERE date_done < $DATE;" 24 | echo "DELETE FROM router_loggedpacket WHERE created_at < $DATE;" 25 | echo "DELETE FROM router_registrationcode WHERE date < $DATE;" 26 | echo "DELETE FROM biflows WHERE COALESCE(start_in, start_out) < $DATE;" 27 | echo "DELETE FROM ssh_sessions WHERE start_time < $DATE;" 28 | echo 'COMMIT;' 29 | ) | psql -U "$DBCLEANER" -d "$DB" $DBPARAMS 30 | -------------------------------------------------------------------------------- /src/master/dbscripts/tagger: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | cd /home/tagger/dist-fw-data/ 5 | wget https://www.turris.cz/greylist-data/greylist-latest.csv -q -O - | tail -n+2 | cut -f1 -d, | sed -e 's/^/add GREYLIST /' >greylist 6 | /home/tagger/ucollect/src/master/dbscripts/tagger.pl /home/tagger/tagger.ini -f greylist 7 | -------------------------------------------------------------------------------- /src/master/dbscripts/tagger.ini.example: -------------------------------------------------------------------------------- 1 | [db] 2 | db=dynamic_firewall 3 | user=tagger 4 | passwd=sM0k3meAk1pp3R 5 | host=db_host.example.org 6 | port=5432 7 | 8 | [anomalies] 9 | client_treshold=4 10 | 11 | [blacklist] 12 | file=/home/tagger/ucollect/src/master/dbscripts/blacklist 13 | 14 | [keep_blocked_limits] 15 | routers = 5 16 | packets = 15 17 | -------------------------------------------------------------------------------- /src/master/dbscripts/tagger.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | 3 | # Look for libraries also in the same directory as the script lives 4 | use FindBin; 5 | use lib $FindBin::Bin; 6 | 7 | use AddrStoreBuild; 8 | #use Tagger::Flows; 9 | use Tagger::FlowFilter; 10 | use Tagger::FwUp; 11 | use Getopt::Long; 12 | use Data::Dumper; 13 | 14 | # Don't start parallel instances of the script 15 | single_instance '/tmp/tagger.lock'; 16 | 17 | my @flows; 18 | GetOptions 19 | "flows=s" => \@flows 20 | or die "Error in command line arguments\n"; 21 | 22 | my $dbh = db_connect; 23 | 24 | my $blacklist = blacklist_load; 25 | 26 | # Prepare a shared precomputed blacklist, so all the loggers can reuse the result of the rather expensive query 27 | $dbh->do('CREATE TEMPORARY TABLE fake_blacklist_tmp AS (SELECT remote, server, mode FROM fake_blacklist) WITH DATA'); 28 | $dbh->do('ANALYZE fake_blacklist_tmp'); 29 | print "Prepared the fake blacklist\n"; 30 | 31 | # Prepare the tags to be used for tagging the flows 32 | #my $flow_tags = Tagger::Flows::prepare_tags($dbh, @flows); 33 | 34 | Tagger::FlowFilter::perform($dbh, $blacklist); 35 | print "Stored flow filters\n"; 36 | 37 | Tagger::FwUp::perform($dbh, $blacklist); 38 | print "Stored FWUp filters\n"; 39 | 40 | # We no longer need the temporary table, commit will kill it. Also, commit the filters and sets. 41 | $dbh->commit; 42 | 43 | #Tagger::Flows::perform($dbh, $flow_tags); 44 | -------------------------------------------------------------------------------- /src/master/dbscripts/testdb: -------------------------------------------------------------------------------- 1 | CREATE DATABASE ucollect; 2 | CREATE USER ucollect WITH PASSWORD '12345'; 3 | GRANT ALL ON DATABASE ucollect TO ucollect; 4 | CREATE USER cleaner; 5 | CREATE USER updater; 6 | CREATE USER jenkins; 7 | CREATE USER archivist; 8 | CREATE USER authenticator; 9 | -------------------------------------------------------------------------------- /src/master/fwup_plugin.py: -------------------------------------------------------------------------------- 1 | # 2 | # Ucollect - small utility for real-time analysis of network data 3 | # Copyright (C) 2015 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | # 5 | # This program is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation; either version 2 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License along 16 | # with this program; if not, write to the Free Software Foundation, Inc., 17 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | # 19 | 20 | import plugin 21 | import logging 22 | import database 23 | import diff_addr_store 24 | import struct 25 | from protocol import extract_string 26 | 27 | logger = logging.getLogger(name='FWUp') 28 | 29 | class FWUpPlugin(plugin.Plugin, diff_addr_store.DiffAddrStore): 30 | """ 31 | Plugin for remotely updating firewall IP sets. 32 | """ 33 | def __init__(self, plugins, config): 34 | plugin.Plugin.__init__(self, plugins) 35 | self.__sets = {} 36 | diff_addr_store.DiffAddrStore.__init__(self, logger, "fwup", "fwup_addresses", "set") 37 | 38 | def __build_config(self): 39 | def convert(name): 40 | return struct.pack('!I' + str(len(name)) + 'scII', len(name), name, self.__sets[name][0], self.__sets[name][1], self.__sets[name][2]) 41 | return ''.join(['C', struct.pack('!II', int(self._conf.get('version', 0)), len(self.__sets))] + map(convert, self.__sets.keys())) 42 | 43 | def _broadcast_config(self): 44 | # Read the rest of the config 45 | with database.transaction() as t: 46 | t.execute("SELECT name, type, maxsize, hashsize FROM fwup_sets") 47 | self.__sets = dict(map(lambda (name, tp, maxsize, hashsize): (name, (tp, maxsize, hashsize)), t.fetchall())) 48 | self.__config_message = self.__build_config() 49 | self.broadcast(self.__config_message) 50 | 51 | def __build_version_info(self, name, epoch, version): 52 | return 'V' + struct.pack('!II' + str(len(name)) + 'sII', int(self._conf.get('version', 0)), len(name), name, epoch, version) 53 | 54 | def _broadcast_version(self, name, epoch, version): 55 | self.broadcast(self.__build_version_info(name, epoch, version)) 56 | 57 | def message_from_client(self, message, client): 58 | if message[0] == 'C': 59 | logger.debug('Sending config to %s', client) 60 | self.send(self.__config_message, client) 61 | elif message[0] == 'A': 62 | (name, rest) = extract_string(message[1:]) 63 | if rest: 64 | logger.warn("Extra info after version query of %s from %s: %s", name, client, repr(rest)) 65 | version = self._addresses.get(name, (0, 0)) 66 | self.send(self.__build_version_info(name, version[0], version[1]), client) 67 | elif message[0] == 'U': 68 | self._provide_diff(message[1:], client, struct.pack('!I', int(self._conf.get('version', 0)))) 69 | else: 70 | logger.warn('Unknown message opcode %s', message[0]) 71 | 72 | def name(self): 73 | return 'Fwup' 74 | -------------------------------------------------------------------------------- /src/master/log_extra.py: -------------------------------------------------------------------------------- 1 | # 2 | # Ucollect - small utility for real-time analysis of network data 3 | # Copyright (C) 2013 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | # 5 | # This program is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation; either version 2 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License along 16 | # with this program; if not, write to the Free Software Foundation, Inc., 17 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | # 19 | 20 | import logging 21 | 22 | """ 23 | Enhance the built in logging a little bit. 24 | 25 | Add a TRACE level (which is more verbose than DEBUG). 26 | """ 27 | 28 | TRACE_LEVEL = 5 29 | 30 | logging.addLevelName(TRACE_LEVEL, "TRACE") 31 | def trace(self, message, *args, **kwargs): 32 | if self.isEnabledFor(TRACE_LEVEL): 33 | self._log(TRACE_LEVEL, message, args, **kwargs) 34 | logging.Logger.trace = trace 35 | def trace_module(message, *args, **kwargs): 36 | trace(logging.getLogger(), message, *args, **kwargs) 37 | logging.trace = trace_module 38 | -------------------------------------------------------------------------------- /src/master/master_config.py: -------------------------------------------------------------------------------- 1 | # 2 | # Ucollect - small utility for real-time analysis of network data 3 | # Copyright (C) 2013 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | # 5 | # This program is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation; either version 2 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License along 16 | # with this program; if not, write to the Free Software Foundation, Inc., 17 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | # 19 | 20 | import ConfigParser 21 | import sys 22 | 23 | if len(sys.argv) != 2: 24 | raise Exception('There must be exactly 1 argument - config file name') 25 | 26 | config_data = ConfigParser.RawConfigParser() 27 | with open(sys.argv[1]) as f: 28 | config_data.readfp(f, sys.argv[1]) 29 | 30 | def get(name): 31 | global config_data 32 | return config_data.get('main', name) 33 | 34 | def getint(name): 35 | global config_data 36 | return config_data.getint('main', name) 37 | 38 | def plugins(): 39 | global config_data 40 | sections = set(config_data.sections()) 41 | sections.remove('main') 42 | return dict(map(lambda plugin: (plugin, dict(config_data.items(plugin))), sections)) 43 | -------------------------------------------------------------------------------- /src/master/protocol.py: -------------------------------------------------------------------------------- 1 | # 2 | # Ucollect - small utility for real-time analysis of network data 3 | # Copyright (C) 2013 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | # 5 | # This program is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation; either version 2 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License along 16 | # with this program; if not, write to the Free Software Foundation, Inc., 17 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | # 19 | 20 | import struct 21 | 22 | def format_string(string): 23 | length = len(string) 24 | return struct.pack('!L' + str(length) + 's', length, string) 25 | 26 | def extract_string(buf): 27 | (slen,) = struct.unpack('!L', buf[:4]) 28 | return (buf[4:slen + 4], buf[slen + 4:]) 29 | -------------------------------------------------------------------------------- /src/master/sniff/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CZ-NIC/ucollect/cfbb9edb99ef514852d29afbfe885ce5d70adb06/src/master/sniff/__init__.py -------------------------------------------------------------------------------- /src/master/sniff/task.py: -------------------------------------------------------------------------------- 1 | # 2 | # Ucollect - small utility for real-time analysis of network data 3 | # Copyright (C) 2014 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | # 5 | # This program is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation; either version 2 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License along 16 | # with this program; if not, write to the Free Software Foundation, Inc., 17 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | # 19 | 20 | class Task: 21 | def __init__(self): 22 | self.active_clients = {} 23 | self.finished_clients = set() 24 | self.task_id = None 25 | self.code = '?' 26 | 27 | def name(self): 28 | return 'Generic task' 29 | 30 | def message(self, client): 31 | return '' 32 | 33 | def finished(self): 34 | pass 35 | 36 | def failure(self, client, message): 37 | pass 38 | 39 | def success(self, client, message): 40 | pass 41 | -------------------------------------------------------------------------------- /src/master/soxy/conn.h: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2014 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #ifndef CONN_H 21 | #define CONN_H 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | static const int COMPRESSION_LEVEL = 9; 31 | static const unsigned int COMPRESSION_BUFFSIZE = 4096; 32 | 33 | class Connection : public QObject { 34 | Q_OBJECT 35 | public: 36 | static bool enableCompression; 37 | Connection(int socket, QSslConfiguration &config); 38 | ~Connection(); 39 | private: 40 | QSslSocket remote; 41 | QLocalSocket local; 42 | QTimer timer; 43 | QByteArray inBuf, outBuf; 44 | unsigned char compressOutBuffer[COMPRESSION_BUFFSIZE]; 45 | unsigned char decompressOutBuffer[COMPRESSION_BUFFSIZE]; 46 | z_stream zStreamCompress; 47 | z_stream zStreamDecompress; 48 | bool inReady, outReady; 49 | void touch(); 50 | private slots: 51 | void incoming(); 52 | void error(QAbstractSocket::SocketError); 53 | void error(QLocalSocket::LocalSocketError); 54 | void error(const QList &); 55 | void error(const char *); 56 | void connectedRemote(); 57 | void tryWriteRemote(); 58 | void outgoing(); 59 | void connectedLocal(); 60 | void tryWriteLocal(); 61 | }; 62 | 63 | #endif 64 | -------------------------------------------------------------------------------- /src/master/soxy/handler.h: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2014 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #ifndef HANDLER_H 21 | #define HANDLER_H 22 | 23 | #include 24 | #include 25 | 26 | class Receiver : public QObject { 27 | Q_OBJECT 28 | public: 29 | Receiver(); 30 | private slots: 31 | void handleConnection(int fd); 32 | private: 33 | QSslConfiguration config; 34 | }; 35 | 36 | class Handler : public QThread { 37 | Q_OBJECT 38 | public: 39 | Receiver *receiver; 40 | Handler(); 41 | void putFd(int fd); 42 | protected: 43 | virtual void run(); 44 | signals: 45 | void handleConnection(int fd); 46 | }; 47 | 48 | #endif 49 | -------------------------------------------------------------------------------- /src/master/soxy/soxy.pro: -------------------------------------------------------------------------------- 1 | TEMPLATE = app 2 | TARGET = soxy 3 | DEPENDPATH += . 4 | INCLUDEPATH += . 5 | LIBS += -lz 6 | 7 | # Input 8 | SOURCES += main.cpp 9 | HEADERS += handler.h conn.h 10 | QT += network 11 | QT -= gui 12 | -------------------------------------------------------------------------------- /src/master/timers.py: -------------------------------------------------------------------------------- 1 | # 2 | # Ucollect - small utility for real-time analysis of network data 3 | # Copyright (C) 2015 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | # 5 | # This program is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation; either version 2 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License along 16 | # with this program; if not, write to the Free Software Foundation, Inc., 17 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | # 19 | 20 | from twisted.internet.task import LoopingCall 21 | import logging 22 | import traceback 23 | 24 | logger = logging.getLogger(name='timers') 25 | 26 | def timer(callback, time, startnow=False): 27 | def protected(): 28 | try: 29 | callback() 30 | except Exception as e: 31 | logger.error("Exception in timer call: %s", traceback.format_exc()) 32 | result = LoopingCall(protected) 33 | result.start(time, startnow) 34 | return result 35 | -------------------------------------------------------------------------------- /src/plugins/Makefile: -------------------------------------------------------------------------------- 1 | RESTRICT := src/plugins 2 | RELATIVE := ../../ 3 | 4 | include $(RELATIVE)/Makefile 5 | -------------------------------------------------------------------------------- /src/plugins/Makefile.dir: -------------------------------------------------------------------------------- 1 | UCOLLECT_PLUGINS:= \ 2 | count \ 3 | crash \ 4 | badconf \ 5 | sniff \ 6 | bandwidth \ 7 | flow \ 8 | majordomo \ 9 | cfgtest \ 10 | spoof \ 11 | refused \ 12 | fake \ 13 | plugtest \ 14 | fwup 15 | 16 | include $(patsubst %,$(S)/src/plugins/%/Makefile.dir,$(UCOLLECT_PLUGINS)) 17 | -------------------------------------------------------------------------------- /src/plugins/badconf/Makefile: -------------------------------------------------------------------------------- 1 | RESTRICT := src/plugins/badconf 2 | RELATIVE := ../../../ 3 | 4 | include $(RELATIVE)/Makefile 5 | -------------------------------------------------------------------------------- /src/plugins/badconf/Makefile.dir: -------------------------------------------------------------------------------- 1 | LIBRARIES += src/plugins/badconf/libplugin_badconf 2 | libplugin_badconf_MODULES := badconf 3 | -------------------------------------------------------------------------------- /src/plugins/badconf/badconf.c: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2013 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #include "../../core/plugin.h" 21 | #include "../../core/packet.h" 22 | #include "../../core/context.h" 23 | #include "../../core/loop.h" 24 | #include "../../core/mem_pool.h" 25 | #include "../../core/util.h" 26 | 27 | #include 28 | 29 | // Warn at most every 15 minutes 30 | #define WARN_TIMEOUT 15 * 60 * 1000 31 | 32 | // At least so many packets before actually warning 33 | #define WARN_COUNT 10 34 | 35 | enum warn_type { 36 | W_PPPOE, 37 | W_LAYER, 38 | W_DIRECTION, 39 | W_MAX 40 | }; 41 | 42 | struct user_data { 43 | size_t count; 44 | uint64_t start; 45 | }; 46 | 47 | void initialize(struct context *context) { 48 | context->user_data = mem_pool_alloc(context->permanent_pool, W_MAX * sizeof *context->user_data); 49 | memset(context->user_data, 0, W_MAX * sizeof *context->user_data); 50 | } 51 | 52 | #define WARN(WTYPE, ...) do {\ 53 | uint64_t now = loop_now(context->loop); \ 54 | if (now - WARN_TIMEOUT > context->user_data[WTYPE].start) { \ 55 | context->user_data[WTYPE] = (struct user_data) { \ 56 | .count = 1, \ 57 | .start = now \ 58 | }; \ 59 | } else if (++ context->user_data[WTYPE].count == WARN_COUNT) { \ 60 | const char *message = mem_pool_printf(context->temp_pool, __VA_ARGS__); \ 61 | ulog(LLOG_WARN, "Possible misconfiguration on interface %s: %s\n", info->interface, message); \ 62 | } \ 63 | } while (0) 64 | 65 | static void packet_handle(struct context *context, const struct packet_info *info) { 66 | for (; info; info = info->next) { 67 | if (info->layer == '?') 68 | WARN(W_LAYER, "packet on unknown layer %d", info->layer_raw); 69 | if (info->direction >= DIR_UNKNOWN) 70 | WARN(W_DIRECTION, "packet of unknown direction"); 71 | if (info->app_protocol == 'P') 72 | WARN(W_PPPOE, "a PPPoE packet seen"); 73 | } 74 | } 75 | 76 | #ifdef STATIC 77 | struct plugin *plugin_info_badconf(void) { 78 | #else 79 | struct plugin *plugin_info(void) { 80 | #endif 81 | static struct plugin plugin = { 82 | .name = "Badconf", 83 | .packet_callback = packet_handle, 84 | .init_callback = initialize, 85 | .version = 1 86 | }; 87 | return &plugin; 88 | } 89 | -------------------------------------------------------------------------------- /src/plugins/bandwidth/Makefile: -------------------------------------------------------------------------------- 1 | RESTRICT := src/plugins/bandwidth 2 | RELATIVE := ../../../ 3 | 4 | include $(RELATIVE)/Makefile 5 | -------------------------------------------------------------------------------- /src/plugins/bandwidth/Makefile.dir: -------------------------------------------------------------------------------- 1 | LIBRARIES += src/plugins/bandwidth/libplugin_bandwidth 2 | libplugin_bandwidth_MODULES := bandwidth 3 | 4 | DOCS += src/plugins/bandwidth/bandwidth 5 | -------------------------------------------------------------------------------- /src/plugins/bandwidth/bandwidth.txt: -------------------------------------------------------------------------------- 1 | The `bandwidth` plugin 2 | ====================== 3 | 4 | Bandwidth is ucollect plugin for passive measurement of available bandwidth of 5 | network connection. 6 | 7 | Collected data 8 | -------------- 9 | 10 | Bandwidth is trying to get count of bytes per second. Plugin uses multiple 11 | 'windows'. Each window has different length in microseconds. The final number is 12 | count of bytes per window's length and the bps value must be computed. 13 | 14 | Count and length of windows is configurable in source code. 15 | 16 | The algorithm places every incoming packet into sliding window that slides at 17 | time line. In fact, it is a group of sliding windows (of the same length) and 18 | they are working together as buffer. In some point the sliding window falls out 19 | from buffer and count of bytes per its length is taken as the candidate for 20 | global maximum. 21 | 22 | The main idea is that the global maximum should represent bandwidth of user's 23 | internet connection. Different length of windows is just tool to evaluate the 24 | stability of connection. 25 | 26 | The protocol 27 | ------------ 28 | 29 | Communication protocol of bandwidth plugin is very simple. Server sends request 30 | to dump data. The format of request is just one number - timestamp. Plugin (as 31 | the response) generates 3 64-bit (uint64_t) numbers for every window length: 32 | 33 | 1) Window length in microseconds 34 | 2) Global maximum of download speed in bytes per 1) 35 | 3) Global maximum of upload speed in bytes per 1) 36 | 37 | Ucollect's binary protocol defines that the head of every message contains the 38 | length of message in bytes. 64-bit number uses 8 bytes. Message length divided 39 | by 8 is count of transmitted numbers. Count of transmitted numbers divided by 3 40 | is the count of different windows defined in plugin. 41 | -------------------------------------------------------------------------------- /src/plugins/cfgtest/Makefile: -------------------------------------------------------------------------------- 1 | RESTRICT := src/plugins/cfgtest 2 | RELATIVE := ../../../ 3 | 4 | include $(RELATIVE)/Makefile 5 | -------------------------------------------------------------------------------- /src/plugins/cfgtest/Makefile.dir: -------------------------------------------------------------------------------- 1 | LIBRARIES += src/plugins/cfgtest/libplugin_cfgtest 2 | libplugin_cfgtest_MODULES := cfgtest 3 | -------------------------------------------------------------------------------- /src/plugins/cfgtest/cfgtest.c: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2014 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #include "../../core/plugin.h" 21 | #include "../../core/context.h" 22 | #include "../../core/loop.h" 23 | #include "../../core/util.h" 24 | 25 | static bool check(struct context *context) { 26 | ulog(LLOG_INFO, "Check called\n"); 27 | const struct config_node *conf = loop_plugin_option_get(context, "test2"); 28 | ulog(LLOG_INFO, "There are %zu options\n", conf ? conf->value_count : 0); 29 | if (conf) { 30 | for (size_t i = 0; i < conf->value_count; i ++) 31 | ulog(LLOG_INFO, "Val: %s\n", conf->values[i]); 32 | } 33 | conf = loop_plugin_option_get(context, "Test3"); 34 | if (conf) { 35 | ulog(LLOG_ERROR, "Test3 is available\n"); 36 | return false; 37 | } 38 | return true; 39 | } 40 | 41 | static void finish(struct context *context, bool activate) { 42 | (void)context; 43 | ulog(LLOG_INFO, "Finish called, activate: %d\n", (int)activate); 44 | } 45 | 46 | #ifdef STATIC 47 | struct plugin *plugin_info_cfgtest(void) { 48 | #else 49 | struct plugin *plugin_info(void) { 50 | #endif 51 | static struct plugin plugin = { 52 | .name = "CfgTest", 53 | .config_check_callback = check, 54 | .config_finish_callback = finish, 55 | .version = 1 56 | }; 57 | return &plugin; 58 | } 59 | -------------------------------------------------------------------------------- /src/plugins/count/Makefile: -------------------------------------------------------------------------------- 1 | RESTRICT := src/plugins/count 2 | RELATIVE := ../../../ 3 | 4 | include $(RELATIVE)/Makefile 5 | -------------------------------------------------------------------------------- /src/plugins/count/Makefile.dir: -------------------------------------------------------------------------------- 1 | LIBRARIES += src/plugins/count/libplugin_count 2 | libplugin_count_MODULES := count 3 | 4 | DOCS += src/plugins/count/count 5 | -------------------------------------------------------------------------------- /src/plugins/count/count.txt: -------------------------------------------------------------------------------- 1 | The `count` plugin 2 | ================== 3 | 4 | It is a very simple plugin, that just counts packets of various 5 | properties and their total sizes. Aside from having this basic 6 | information available, it can also serve as example implementation of 7 | plugin. 8 | 9 | From time to time, the server requests the statistics and they are 10 | reset to zeroes. 11 | 12 | The protocol 13 | ------------ 14 | 15 | All data are encoded in network byte order. 16 | 17 | The server requests data by sending its timestamp (`uint64_t`). 18 | 19 | The client answers with the data. There are four parts to the message: 20 | 21 | Timestamp:: 22 | Encoded as `uint64_t`, it is the timestamp sent by server in the 23 | previous request, or 0 on the first request. This can be used to 24 | guess the length of interval. 25 | Number of interfaces:: 26 | The number of interfaces being captured in the client, encoded as 27 | `uint64_t`. 28 | Interface statistics:: 29 | For each interface, there is a triple of `uint64_t` numbers, meaning 30 | number of packets captured on the interface, dropped by PCAP because 31 | the software didn't keep up and dropped by the interface driver. 32 | Property statistics:: 33 | For each property, there's a pair of `uint64_t` numbers, meaning the 34 | count of packets with the property and the total size of these 35 | packets. 36 | 37 | The properties counted 38 | ---------------------- 39 | 40 | These are the kinds of packets counted, in order they appear in the 41 | message. More kinds are expected to appear, but they'll be appended to 42 | the list, so the beginning of the list is fixed. 43 | 44 | All:: 45 | Any packet is included in this category. 46 | IPv4:: 47 | Packets sent over the IPv4 protocol. 48 | IPv6:: 49 | Packets sent over the IPv6 protocol. 50 | In:: 51 | Packets sent over either IPv4 or IPv6 that have the source address 52 | outside of the local network and the destination inside. 53 | Out:: 54 | Similar as In, but with the destination and source locations 55 | reversed. 56 | TCP:: 57 | IPv4/TCP and IPv6/TCP packets. 58 | UDP:: 59 | IPv4/UDP and IPv6/UDP packets. 60 | ICMP:: 61 | Both ICMP and ICMPv6 packets. 62 | Low port:: 63 | TCP or UDP packets with the remote port less or equal to 1024. 64 | SYN:: 65 | TCP packets with SYN flag set. 66 | FIN:: 67 | TCP packets with FIN flag set. 68 | SYN+ACK:: 69 | TCP packets with both SYN and ACK flags set. This may be a good 70 | heuristics for number of created TCP connections. 71 | ACK:: 72 | TCP packets with ACK flag set. 73 | PUSH:: 74 | TCP packets with PUSH flag set. 75 | SERVER:: 76 | The communication with the uplink server. 77 | V6TUNNEL:: 78 | IPv6 communication sent through a tunnel (in contrast with IPv4 or 79 | IPv6 native communication). 80 | 81 | Note that the related categories don't necessarily form complements. 82 | There may be, for example, packets that are not considered In nor Out 83 | -- packets that are not sent over any IP protocol, or packets that 84 | have both addresses outside or both addresses inside the local 85 | network. 86 | -------------------------------------------------------------------------------- /src/plugins/crash/Makefile: -------------------------------------------------------------------------------- 1 | RESTRICT := src/plugins/crash 2 | RELATIVE := ../../../ 3 | 4 | include $(RELATIVE)/Makefile 5 | -------------------------------------------------------------------------------- /src/plugins/crash/Makefile.dir: -------------------------------------------------------------------------------- 1 | LIBRARIES += src/plugins/crash/libplugin_crash 2 | libplugin_crash_MODULES := crash 3 | -------------------------------------------------------------------------------- /src/plugins/crash/crash.c: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2013 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #include "../../core/plugin.h" 21 | #include "../../core/loop.h" 22 | #include "../../core/context.h" 23 | 24 | #include 25 | 26 | /* 27 | * WARNING: This is a testing plugin only. The only thing it does is to crash 28 | * after 2 seconds of runtime. It is not expected to be used in production. 29 | */ 30 | 31 | static void crash(struct context *context_unused, void *data_unused, size_t id_unused) { 32 | (void) context_unused; 33 | (void) data_unused; 34 | (void) id_unused; 35 | abort(); 36 | } 37 | 38 | static void prepare_crash(struct context *context) { 39 | loop_timeout_add(context->loop, 2000, context, NULL, crash); 40 | } 41 | 42 | static void packet_crash(struct context *context_unused, const struct packet_info *info_unused) { 43 | (void) context_unused; 44 | (void) info_unused; 45 | abort(); 46 | } 47 | 48 | static void data_crash(struct context *context_unused, const uint8_t *data_unused, size_t length_unused) { 49 | (void) context_unused; 50 | (void) data_unused; 51 | (void) length_unused; 52 | abort(); 53 | } 54 | 55 | struct plugin *plugin_info(void) { 56 | static struct plugin plugin = { 57 | .name = "Crash", 58 | .uplink_data_callback = data_crash, 59 | .init_callback = prepare_crash, 60 | .packet_callback = packet_crash 61 | }; 62 | return &plugin; 63 | } 64 | -------------------------------------------------------------------------------- /src/plugins/fake/Makefile: -------------------------------------------------------------------------------- 1 | RESTRICT := src/plugins/fake 2 | RELATIVE := ../../../ 3 | 4 | include $(RELATIVE)/Makefile 5 | -------------------------------------------------------------------------------- /src/plugins/fake/Makefile.dir: -------------------------------------------------------------------------------- 1 | LIBRARIES += src/plugins/fake/libplugin_fake 2 | libplugin_fake_MODULES := main server log telnet websrv base64 3 | DOCS += src/plugins/fake/fake_firewall \ 4 | src/plugins/fake/fake 5 | -------------------------------------------------------------------------------- /src/plugins/fake/base64.c: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2016 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #include "base64.h" 21 | 22 | #include 23 | #include 24 | 25 | /* 26 | * The code is slightly inspired by the one from libucw, but 27 | * rewritten from scratch to suit our needs better. 28 | */ 29 | 30 | /* 31 | * This is OK, since all the characters here are <128 and therefore 32 | * have the same value as signed and unsigned. 33 | */ 34 | const uint8_t *allowed = (const uint8_t *) 35 | "ABCDEFGHIJKLMNOPQRSTUVWXYZ" 36 | "abcdefghijklmnopqrstuvwxyz" 37 | "0123456789+/"; 38 | 39 | size_t base64_decode_inplace(uint8_t *buffer) { 40 | // Mapping character → index 41 | static uint8_t reverse[265]; 42 | static bool init = false; 43 | if (!init) { 44 | // Mark all positions as invalid first 45 | memset(reverse, 0xFF, sizeof reverse); 46 | // Go through the allowed characters and place a value to each, so we can index by them 47 | for (size_t i = 0; allowed[i]; i ++) 48 | reverse[allowed[i]] = i; 49 | init = true; 50 | } 51 | // Positions in the buffer (in moves faster, so they have separate counters) 52 | size_t in = 0; 53 | size_t out = 0; 54 | // Count the valid input characters (we ignore invalid ones, so it may be different than in) 55 | size_t vin = 0; 56 | uint8_t val; 57 | while ((val = buffer[in ++])) { 58 | val = reverse[val]; 59 | if (val == 0xFF) 60 | // Invalid character. Move to the next. 61 | continue; 62 | switch (vin ++ % 4) { 63 | case 0: 64 | buffer[out] = val << 2; 65 | break; 66 | case 1: 67 | buffer[out ++] |= val >> 4; 68 | buffer[out] = (val & 0x0F) << 4; 69 | break; 70 | case 2: 71 | buffer[out ++] |= val >> 2; 72 | buffer[out] = (val & 0x03) << 6; 73 | break; 74 | case 3: 75 | buffer[out ++] |= val; 76 | break; 77 | } 78 | } 79 | // NULL-terminate for convenience 80 | buffer[out] = '\0'; 81 | return out; 82 | } 83 | -------------------------------------------------------------------------------- /src/plugins/fake/base64.h: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2016 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #ifndef UCOLLECT_FAKE_BASE64_H 21 | #define UCOLLECT_FAKE_BASE64_H 22 | 23 | #include 24 | #include 25 | 26 | /* 27 | * Decode the buffer from base64 to normal data. The data are overwritten. 28 | * The input is NULL-terminated string and it returns the size of decoded 29 | * data. The output is NULL-terminated for convenience (however, the output 30 | * size does not include this NULL-terminator and the data may contain 31 | * other NULL-bytes). 32 | * 33 | * Note that the base64 decoded data are never larger than the encoded 34 | * equivalent, so it's OK to reuse that buffer. 35 | * 36 | * Any invalid characters are ignored. 37 | */ 38 | size_t base64_decode_inplace(uint8_t *buffer) __attribute__((nonnull)); 39 | 40 | #endif 41 | -------------------------------------------------------------------------------- /src/plugins/fake/firewall-add: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | while read line ; do 4 | eval $(echo "$line" | sed -ne 's/.*ucollect-fake-open-\([^:]*\).*SRC=\([^ ]*\).*DST=\([^ ]*\).*PROTO=\([^ ]*\).*DPT=\([^ ]*\).*/ipset add ucollect_fake_exc_\1 \2,\4:\5,\3/p') 5 | done 6 | -------------------------------------------------------------------------------- /src/plugins/fake/firewall-syslog: -------------------------------------------------------------------------------- 1 | destination ucollect_fake { 2 | program("/usr/share/ucollect/scripts/ucollect-add-firewall"); 3 | }; 4 | 5 | filter f_ucollect_fake { 6 | match(".*ucollect-fake-open-.*: .*" value(MESSAGE)); 7 | }; 8 | 9 | log { 10 | source(kernel); 11 | filter(f_ucollect_fake); 12 | destination(ucollect_fake); 13 | }; 14 | -------------------------------------------------------------------------------- /src/plugins/fake/main.h: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2014 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #ifndef UCOLLECT_TELNET_MAIN_H 21 | #define UCOLLECT_TELNET_MAIN_H 22 | 23 | #include 24 | 25 | struct context *context; 26 | struct fd_tag *tag; 27 | 28 | /* 29 | * Report that the connection was closed by either side. 30 | * 31 | * The error indicates if it was in some error (no matter what) or 32 | * if it was clean shutdown. 33 | */ 34 | void conn_closed(struct context *context, struct fd_tag *tag, bool error, const char *reason); 35 | /* 36 | * Log that the other side attemted to authenticate or use the service in some way. 37 | */ 38 | void conn_log_attempt(struct context *context, struct fd_tag *tag, const char *username, const char *password, const char *method, const char *host, const char *uri); 39 | 40 | #endif 41 | -------------------------------------------------------------------------------- /src/plugins/fake/server.c: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2014 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #include "server.h" 21 | 22 | #include "telnet.h" 23 | #include "websrv.h" 24 | 25 | #include 26 | #include 27 | 28 | #define SECOND (1000) 29 | 30 | const struct server_desc server_descs_intern[] = { 31 | { 32 | .name = "telnet", 33 | .code = 'T', 34 | .sock_type = SOCK_STREAM, 35 | .default_port = 23, 36 | // No server-scope data, so skip server_alloc and server_set_fd 37 | .conn_alloc_cb = telnet_conn_alloc, 38 | .conn_set_fd_cb = telnet_conn_set_fd, 39 | .server_ready_cb = telnet_data, 40 | .max_conn = 20, 41 | .conn_timeout = 30 * SECOND 42 | }, 43 | { // An alternative telnet port 44 | .name = "telnet_alt", 45 | .code = 't', 46 | .sock_type = SOCK_STREAM, 47 | .default_port = 2323, 48 | .conn_alloc_cb = telnet_conn_alloc, 49 | .conn_set_fd_cb = telnet_conn_set_fd, 50 | .server_ready_cb = telnet_data, 51 | .max_conn = 20, 52 | .conn_timeout = 30 * SECOND 53 | }, 54 | { 55 | .name = "http", 56 | .code = 'H', 57 | .sock_type = SOCK_STREAM, 58 | .default_port = 80, 59 | .server_alloc_cb = alloc_websrv, 60 | .conn_alloc_cb = http_conn_alloc, 61 | .conn_set_fd_cb = http_conn_set_fd, 62 | .server_ready_cb = http_data, 63 | .max_conn = 20, 64 | .conn_timeout = 30 * SECOND 65 | }, 66 | { 67 | .name = "squid_http_proxy", 68 | .code = 'P', 69 | .sock_type = SOCK_STREAM, 70 | .default_port = 3128, 71 | .server_alloc_cb = alloc_proxy, 72 | .conn_alloc_cb = http_conn_alloc, 73 | .conn_set_fd_cb = http_conn_set_fd, 74 | .server_ready_cb = http_data, 75 | .max_conn = 20, 76 | .conn_timeout = 30 * SECOND 77 | }, 78 | { 79 | .name = "http_proxy", 80 | .code = 'h', 81 | .sock_type = SOCK_STREAM, 82 | .default_port = 8080, 83 | .server_alloc_cb = alloc_proxy, 84 | .conn_alloc_cb = http_conn_alloc, 85 | .conn_set_fd_cb = http_conn_set_fd, 86 | .server_ready_cb = http_data, 87 | .max_conn = 20, 88 | .conn_timeout = 30 * SECOND 89 | }, 90 | { 91 | .name = "polipo_http_proxy", 92 | .code = 'p', 93 | .sock_type = SOCK_STREAM, 94 | .default_port = 8123, 95 | .server_alloc_cb = alloc_proxy, 96 | .conn_alloc_cb = http_conn_alloc, 97 | .conn_set_fd_cb = http_conn_set_fd, 98 | .server_ready_cb = http_data, 99 | .max_conn = 20, 100 | .conn_timeout = 30 * SECOND 101 | }, 102 | { 103 | .name = NULL 104 | } 105 | }; 106 | 107 | const struct server_desc *server_descs = server_descs_intern; 108 | -------------------------------------------------------------------------------- /src/plugins/fake/server.h: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2014 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #ifndef UCOLLECT_FAKE_SERVER_H 21 | #define UCOLLECT_FAKE_SERVER_H 22 | 23 | #include 24 | 25 | struct server_data; 26 | struct server_desc; 27 | struct conn_data; 28 | struct context; 29 | struct mem_pool; 30 | struct fd_tag; 31 | 32 | typedef struct server_data *(*server_alloc)(struct context *context, struct fd_tag *tag, struct mem_pool *pool, const struct server_desc *desc); 33 | typedef void (*server_set_fd)(struct context *context, struct fd_tag *tag, struct server_data *server, int fd, uint16_t port); 34 | typedef struct conn_data *(*conn_alloc)(struct context *context, struct fd_tag *tag, struct mem_pool *pool, struct server_data *server); 35 | // Also accept... 36 | typedef void (*conn_set_fd)(struct context *context, struct fd_tag *tag, struct server_data *server, struct conn_data *conn, int fd); 37 | typedef void (*server_ready)(struct context *context, struct fd_tag *tag, struct server_data *server, struct conn_data *conn); 38 | 39 | /* 40 | * There are two modes in which this may operate. A connected mode and unconnected one. 41 | * 42 | * In the connected mode, certain number of simultaneous connections can be accepted 43 | * (specifid by max_conn). A main FD is allocated in the beginning and set with 44 | * server_set_fd. Bunch of conn structures is allocated and whenever new connection 45 | * is accepted, the conn_set_fd is called on one of the conn structures with the new 46 | * fd. The server_ready is called on it every time data is available. 47 | * 48 | * In the unconnected mode, single conn structure is allocated, the main FD is set into 49 | * it and then the server_ready is called with it whenever it is readable. This mode 50 | * is enabled by setting max_conn to 0. The conn_set_fd callback is never used here. 51 | * 52 | * Note that even the main FD can change during the lifetime of the server. 53 | * 54 | * Only the server_ready is mandatory. 55 | */ 56 | struct server_desc { 57 | const char *name; 58 | char code; // Code in uplink communication 59 | int sock_type; // like SOCK_STREAM or SOCK_DGRAM 60 | uint16_t default_port; 61 | server_alloc server_alloc_cb; 62 | server_set_fd server_set_fd_cb; 63 | conn_alloc conn_alloc_cb; 64 | conn_set_fd conn_set_fd_cb; 65 | server_ready server_ready_cb; 66 | unsigned max_conn; // Maximum number of parallel connections. If more are to be opened, new ones are immediatelly closed after accepting. 67 | unsigned conn_timeout; // Timeout in milliseconds ‒ if nothing comes over the socket in this time, it is dropped 68 | }; 69 | 70 | extern const struct server_desc *server_descs; 71 | 72 | #endif 73 | -------------------------------------------------------------------------------- /src/plugins/fake/telnet.h: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2014 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #ifndef UCOLLECT_FAKE_TELNET_H 21 | #define UCOLLECT_FAKE_TELNET_H 22 | 23 | struct context; 24 | struct mem_pool; 25 | 26 | struct server_data; // Not used here at all. No need to coordinate through central data structure 27 | struct conn_data; 28 | struct fd_tag; 29 | 30 | struct conn_data *telnet_conn_alloc(struct context *context, struct fd_tag *tag, struct mem_pool *pool, struct server_data *server); 31 | void telnet_conn_set_fd(struct context *context, struct fd_tag *tag, struct server_data *server, struct conn_data *conn, int fd); 32 | void telnet_data(struct context *context, struct fd_tag *tag, struct server_data *server, struct conn_data *conn); 33 | 34 | #endif 35 | -------------------------------------------------------------------------------- /src/plugins/fake/websrv.h: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2016 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #ifndef UCOLLECT_FAKE_WEBSRV_H 21 | #define UCOLLECT_FAKE_WEBSRV_H 22 | 23 | struct context; 24 | struct mem_pool; 25 | 26 | struct server_data; 27 | struct conn_data; 28 | struct fd_tag; 29 | struct server_desc; 30 | 31 | struct server_data *alloc_websrv(struct context *context, struct fd_tag *tag, struct mem_pool *pool, const struct server_desc *desc); 32 | struct server_data *alloc_proxy(struct context *context, struct fd_tag *tag, struct mem_pool *pool, const struct server_desc *desc); 33 | struct conn_data *http_conn_alloc(struct context *context, struct fd_tag *tag, struct mem_pool *pool, struct server_data *server); 34 | void http_conn_set_fd(struct context *context, struct fd_tag *tag, struct server_data *server, struct conn_data *conn, int fd); 35 | void http_data(struct context *context, struct fd_tag *tag, struct server_data *server, struct conn_data *conn); 36 | 37 | #endif 38 | -------------------------------------------------------------------------------- /src/plugins/flow/Makefile: -------------------------------------------------------------------------------- 1 | RESTRICT := src/plugins/flow 2 | RELATIVE := ../../../ 3 | 4 | include $(RELATIVE)/Makefile 5 | -------------------------------------------------------------------------------- /src/plugins/flow/Makefile.dir: -------------------------------------------------------------------------------- 1 | LIBRARIES += src/plugins/flow/libplugin_flow 2 | libplugin_flow_MODULES := main filter flow 3 | 4 | DOCS += src/plugins/flow/flow 5 | -------------------------------------------------------------------------------- /src/plugins/flow/filter.h: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2014 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #ifndef UCOLLECT_FLOW_FILTER_H 21 | #define UCOLLECT_FLOW_FILTER_H 22 | 23 | #include "../../libs/diffstore/diff_store.h" 24 | 25 | #include 26 | #include 27 | #include 28 | 29 | struct filter; 30 | struct packet_info; 31 | struct mem_pool; 32 | 33 | bool filter_apply(struct mem_pool *tmp_pool, const struct filter *filter, const struct packet_info *packet) __attribute__((nonnull)); 34 | struct filter *filter_parse(struct mem_pool *pool, const uint8_t *desc, size_t size) __attribute__((nonnull)); 35 | 36 | // Decide how to react to a change on the server. Orig-version is used as an out-parameter in case of FILTER_INCREMENTAL 37 | enum diff_store_action filter_action(struct filter *filter, const char *name, uint32_t epoch, uint32_t version, uint32_t *orig_version); 38 | /* 39 | * Apply a difference to a filter of given name. 40 | * 41 | * It can use the memory pool to allocate permanent data (in the filter). 42 | * 43 | * It usually returns FILTER_NO_ACTION, which means the change was successful. 44 | * However, it can request a full update instead of incremental, if the epoch is different, or say FILTER_UNKNOWN if the filter is not known (or whatever other value ‒ see orig_version in filter_action). 45 | */ 46 | enum diff_store_action filter_diff_apply(struct mem_pool *tmp_pool, struct filter *filter, const char *name, bool full, uint32_t epoch, uint32_t from, uint32_t to, const uint8_t *diff, size_t diff_size, uint32_t *orig_version); 47 | 48 | #endif 49 | -------------------------------------------------------------------------------- /src/plugins/flow/flow.h: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2014-2017 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #ifndef UCOLLECT_FLOW_FLOW_H 21 | #define UCOLLECT_FLOW_FLOW_H 22 | 23 | #include 24 | #include 25 | #include 26 | 27 | struct packet_info; 28 | struct mem_pool; 29 | 30 | enum flow_ipv { 31 | FLOW_V4 = 0, 32 | FLOW_V6 = 1 33 | }; 34 | 35 | enum flow_proto { 36 | FLOW_TCP = 0, 37 | FLOW_UDP = 2// Maybe others? 38 | }; 39 | 40 | enum seen_event { 41 | SEEN_START_IN, 42 | SEEN_START_OUT, 43 | SEEN_FIN_IN, 44 | SEEN_FIN_OUT, 45 | SEEN_RST, 46 | }; 47 | 48 | static const uint8_t const flow_events[5] = { 4, 8, 16, 32, 64 }; 49 | 50 | typedef uint8_t flow_addr_t[16]; 51 | 52 | // Order is in, out, resp local/remote 53 | struct flow { 54 | uint32_t count[2]; 55 | uint64_t size[2]; 56 | uint64_t first_time[2], last_time[2]; 57 | uint16_t ports[2]; 58 | flow_addr_t addrs[2]; 59 | enum flow_ipv ipv; 60 | enum flow_proto proto; 61 | // Detected an initialization of the communication. Curretly, this is done only for TCP and it means a packet with only SYN was seen. 62 | bool seen_flow_event[5]; 63 | }; 64 | 65 | void flow_parse(struct flow *target, const struct packet_info *packet) __attribute__((nonnull)); 66 | size_t flow_size(const struct flow *flow) __attribute__((nonnull)); 67 | void flow_render(uint8_t *dst, size_t dst_size, const struct flow *flow) __attribute__((nonnull)); 68 | uint8_t *flow_key(const struct packet_info *packet, size_t *size, struct mem_pool *pool) __attribute__((nonnull, malloc)); 69 | 70 | #endif 71 | -------------------------------------------------------------------------------- /src/plugins/fwup/Makefile: -------------------------------------------------------------------------------- 1 | RESTRICT := src/plugins/fwup 2 | RELATIVE := ../../../ 3 | 4 | include $(RELATIVE)/Makefile 5 | -------------------------------------------------------------------------------- /src/plugins/fwup/Makefile.dir: -------------------------------------------------------------------------------- 1 | LIBRARIES += src/plugins/fwup/libplugin_fwup 2 | libplugin_fwup_MODULES := main type queue 3 | DOCS += src/plugins/fwup/fwup 4 | -------------------------------------------------------------------------------- /src/plugins/fwup/queue.h: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2015 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #ifndef UCOLLECT_FWUP_QUEUE_H 21 | #define UCOLLECT_FWUP_QUEUE_H 22 | 23 | #include 24 | 25 | struct queue; 26 | 27 | struct context; 28 | 29 | typedef void (*reload_callback_t)(struct context *context); 30 | 31 | /* 32 | * Create a queue for the commands. It will manage 33 | * running the ipset command and feed commands to it. 34 | * 35 | * The structure is allocated from the context's permanent 36 | * pool. It is not expected to be destroyed during the lifetime 37 | * of the plugin. 38 | * 39 | * The ipset command is launched on-demand when data are 40 | * set to it. It is stopped either by explicit flush or 41 | * by a short timeout. 42 | * 43 | * The reload callback is used whenever the ipset command dies 44 | * with an error. The queue is disabled for a while and after 45 | * a short time, it is retried. 46 | */ 47 | struct queue *queue_alloc(struct context *context, reload_callback_t reload_callback) __attribute__((nonnull)) __attribute__((malloc)) __attribute__((returns_nonnull)); 48 | /* 49 | * Enqueue another command. The ipset command is launched 50 | * or previous one is reused. Due to internal OS buffering, 51 | * the command may be delayed for several seconds, but not 52 | * after flush is performed. 53 | */ 54 | void enqueue(struct context *context, struct queue *queue, const char *command) __attribute__((nonnull)); 55 | /* 56 | * Make sure there are no more waiting commands (close 57 | * the ipset command if there's one kept running). 58 | */ 59 | void queue_flush(struct context *context, struct queue *queue) __attribute__((nonnull)); 60 | 61 | /* 62 | * Callback for when there are data on our FD. 63 | */ 64 | void queue_fd_data(struct context *context, int fd, void *userdata) __attribute__((nonnull)); 65 | 66 | /* 67 | * Callback when some child of ucollect dies. Examines it to see if it 68 | * is the ipset command and if so, if it terminated with error or successfully. 69 | */ 70 | void queue_child_died(struct context *context, int status, pid_t pid, struct queue *queue) __attribute__((nonnull)); 71 | 72 | #endif 73 | -------------------------------------------------------------------------------- /src/plugins/fwup/type.h: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2015 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #ifndef UCOLLECT_FWUP_TYPE_H 21 | #define UCOLLECT_FWUP_TYPE_H 22 | 23 | #include 24 | #include 25 | 26 | struct mem_pool; 27 | 28 | typedef const char *(*addr2str_t)(const uint8_t *addr, size_t size, struct mem_pool *pool); 29 | 30 | struct set_type { 31 | const char *desc; 32 | const char *family; 33 | addr2str_t addr2str; 34 | }; 35 | 36 | extern const struct set_type set_types[256]; 37 | 38 | #endif 39 | -------------------------------------------------------------------------------- /src/plugins/majordomo/Makefile: -------------------------------------------------------------------------------- 1 | RESTRICT := src/plugins/majordomo 2 | RELATIVE := ../../../ 3 | 4 | include $(RELATIVE)/Makefile 5 | -------------------------------------------------------------------------------- /src/plugins/majordomo/Makefile.dir: -------------------------------------------------------------------------------- 1 | LIBRARIES += src/plugins/majordomo/libplugin_majordomo 2 | libplugin_majordomo_MODULES := majordomo 3 | 4 | DOCS += src/plugins/majordomo/majordomo 5 | -------------------------------------------------------------------------------- /src/plugins/majordomo/majordomo.txt: -------------------------------------------------------------------------------- 1 | The `majordomo` plugin 2 | ====================== 3 | 4 | `Majordomo` is the `lcollect` plugin for analysing flows of connected clients 5 | in LAN. 6 | 7 | Collected data are just small subset of hypothetical netflow. Eg. time 8 | dimension is not considered. 9 | 10 | Important notes to deployment 11 | ----------------------------- 12 | 13 | There are 2 possible memory limits: 14 | 15 | 1) Maximum flows per client - this value is limited by SOURCE_SIZE_LIMIT 16 | constant. 17 | 18 | 2) Maximum number of clients - this value is currently unlimited. Anyway, 19 | expected production area is in SOHO routers with small number of real connected 20 | clients. In this case aren't any limits needed. 21 | 22 | For bigger networks is recommended to consider the introduction of the maximum 23 | number of clients. 24 | -------------------------------------------------------------------------------- /src/plugins/majordomo/scripts/majordomo-cron: -------------------------------------------------------------------------------- 1 | MAILTO="" 2 | # m h dom mon dow user command 3 | */5 * * * * root majordomo_db.sh downsize 4 | 1 * * * * root majordomo_db.sh genhour 5 | 4 * * * * root majordomo_locked_precache.sh 6 | -------------------------------------------------------------------------------- /src/plugins/majordomo/scripts/majordomo.conf: -------------------------------------------------------------------------------- 1 | package majordomo 2 | 3 | config 'db' 4 | option 'path' '/tmp/majordomo_db' 5 | option 'max_items_per_client' '6000' 6 | option 'store_hourly_files' '96' 7 | option 'store_daily_files' '60' 8 | option 'store_monthly_files' '12' 9 | 10 | config 'lookup' 11 | option 'make_lookup_dns' '1' 12 | -------------------------------------------------------------------------------- /src/plugins/majordomo/scripts/majordomo_locked_precache.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | LOCK_DIR="/tmp/majordomo_precache_lock" 4 | 5 | if mkdir "$LOCK_DIR" 2>/dev/null; then 6 | trap 'kill $PID ; rm -rf "$LOCK_DIR" ; exit 2' INT QUIT TERM ABRT HUP ILL TRAP BUS FPE SEGV 7 | timeout 45m majordomo_cache.lua precache & 8 | PID=$! 9 | wait 10 | STATUS=$? 11 | rm -rf "$LOCK_DIR" 12 | exit $STATUS 13 | fi 14 | -------------------------------------------------------------------------------- /src/plugins/majordomo/scripts/majordomo_show.lua: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env lua 2 | 3 | --[[ 4 | Copyright 2014, CZ.NIC z.s.p.o. (http://www.nic.cz/) 5 | 6 | This script is part of majordomo plugin for ucollect 7 | 8 | NUCI is free software: you can redistribute it and/or modify 9 | it under the terms of the GNU General Public License as published by 10 | the Free Software Foundation, either version 3 of the License, or 11 | (at your option) any later version. 12 | 13 | NUCI is distributed in the hope that it will be useful, 14 | but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | GNU General Public License for more details. 17 | 18 | You should have received a copy of the GNU General Public License 19 | along with NUCI. If not, see . 20 | ]] 21 | 22 | package.path = package.path .. ';/usr/share/lcollect/lua/?.lua' 23 | require("majordomo_lib"); 24 | 25 | function main() 26 | local _, make_lookup_mac, make_lookup_dns, _ = majordomo_get_configuration(); 27 | if make_lookup_mac then 28 | macdb = get_inst_macdb(); 29 | macdb:deserialize(); 30 | end 31 | if #arg ~= 1 then 32 | io.stderr:write(string.format("Usage: %s file_to_dump\n", arg[0])); 33 | os.exit(1); 34 | end 35 | 36 | db = {} 37 | if not read_file(db, arg[1]) then 38 | io.stderr:write(string.format("Cannot open file %s\n", arg[1])); 39 | os.exit(2); 40 | end 41 | 42 | for addr, items in pairs(db) do 43 | local sorted = get_sorted_items(items, "u_count"); 44 | if make_lookup_mac and macdb:lookup(addr) then 45 | io.stdout:write(string.format("%s (%s)\n", addr, macdb:lookup(addr))); 46 | else 47 | io.stdout:write(string.format("%s\n", addr)); 48 | end 49 | for _, item in ipairs(sorted) do 50 | local proto, _, dst, port = split_key(item.key); 51 | if item.value.resolved_name and item.value.resolved_name ~= CACHE_EMPTY_NAME then 52 | dst = table.concat(item.value.names:items(), ", "); 53 | end 54 | io.stdout:write(string.format("\t - %s - (%s/%s) - (%f/%f/%f) - (%f/%f/%f)\n", 55 | dst, port, proto, 56 | item.value.d_count, item.value.d_size, item.value.d_data_size, 57 | item.value.u_count, item.value.u_size, item.value.u_data_size) 58 | ); 59 | end 60 | end 61 | 62 | if make_lookup_mac then 63 | macdb:serialize(); 64 | end 65 | 66 | end 67 | 68 | main(); 69 | -------------------------------------------------------------------------------- /src/plugins/plugtest/Makefile: -------------------------------------------------------------------------------- 1 | RESTRICT := src/plugins/plugtest 2 | RELATIVE := ../../../ 3 | 4 | include $(RELATIVE)/Makefile 5 | -------------------------------------------------------------------------------- /src/plugins/plugtest/Makefile.dir: -------------------------------------------------------------------------------- 1 | LIBRARIES += src/plugins/plugtest/libplugin_plugtest 2 | libplugin_plugtest_MODULES := plugtest 3 | -------------------------------------------------------------------------------- /src/plugins/plugtest/plugtest.c: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2015 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #include "../../core/plugin.h" 21 | #include "../../core/loop.h" 22 | #include "../../core/context.h" 23 | 24 | #define PLUGLIB_DO_IMPORT PLUGLIB_LOCAL 25 | #include "../../libs/test/test.h" 26 | 27 | static void timeout(struct context *context, void *data __attribute__((unused)), size_t id __attribute__((unused))) { 28 | hello_world(); 29 | loop_timeout_add(context->loop, 1000, context, NULL, timeout); 30 | } 31 | 32 | static void init(struct context *context) { 33 | timeout(context, NULL, 0); 34 | } 35 | 36 | #ifndef STATIC 37 | unsigned api_version() { 38 | return UCOLLECT_PLUGIN_API_VERSION; 39 | } 40 | #endif 41 | 42 | #ifdef STATIC 43 | struct plugin *plugin_info_plugtest(void) { 44 | #else 45 | struct plugin *plugin_info(void) { 46 | #endif 47 | static struct pluglib_import *imports[] = { 48 | &hello_world_import, 49 | NULL 50 | }; 51 | static struct plugin plugin = { 52 | .name = "PlugTest", 53 | .version = 1, 54 | .imports = imports, 55 | .init_callback = init 56 | }; 57 | return &plugin; 58 | } 59 | -------------------------------------------------------------------------------- /src/plugins/refused/Makefile: -------------------------------------------------------------------------------- 1 | RESTRICT := src/plugins/refused 2 | RELATIVE := ../../../ 3 | 4 | include $(RELATIVE)/Makefile 5 | -------------------------------------------------------------------------------- /src/plugins/refused/Makefile.dir: -------------------------------------------------------------------------------- 1 | LIBRARIES += src/plugins/refused/libplugin_refused 2 | libplugin_refused_MODULES := refused icmp 3 | 4 | DOCS += src/plugins/refused/refused 5 | -------------------------------------------------------------------------------- /src/plugins/refused/icmp.h: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2014 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | // Parsing of destination unreachable from ICMP and ICMPv6 packets 21 | 22 | #ifndef UCOLLECT_REFUSED_ICMP_H 23 | #define UCOLLECT_REFUSED_ICMP_H 24 | 25 | #include 26 | #include 27 | 28 | struct packet_info *info; 29 | 30 | /* 31 | * Look into a packet and decide if it is a destination unreachable ICMP packet 32 | * for a TCP connection. 33 | * 34 | * If it is, return the type by the return value and provide info about the 35 | * connection it belongs to. The types are: 36 | * - 'N': Network unreachable. 37 | * - 'H': Host unreachable. 38 | * - 'P': Port unreachable. 39 | * - 'O': Some other reasone. 40 | * - 'A': Administratively prohibited. 41 | * 42 | * In case it is not such NAK packet, it returns '\0'. 43 | */ 44 | char nak_parse(const struct packet_info *packet, size_t *addr_len, const uint8_t **addr, uint16_t *loc_port, uint16_t *dest_port); 45 | 46 | #endif 47 | -------------------------------------------------------------------------------- /src/plugins/sniff/Makefile: -------------------------------------------------------------------------------- 1 | RESTRICT := src/plugins/sniff 2 | RELATIVE := ../../../ 3 | 4 | include $(RELATIVE)/Makefile 5 | -------------------------------------------------------------------------------- /src/plugins/sniff/Makefile.dir: -------------------------------------------------------------------------------- 1 | LIBRARIES += src/plugins/sniff/libplugin_sniff 2 | SNIFF_TASKS := ping cert 3 | libplugin_sniff_MODULES := sniff task fork nop parse $(SNIFF_TASKS) 4 | 5 | DOCS += src/plugins/sniff/sniff 6 | 7 | # Make hack. Used below in the foreach, we can't really put an empty line there otherwise 8 | define N 9 | 10 | 11 | endef 12 | 13 | $(O)/.objs/%.inc: $(S)/src/plugins/sniff/% 14 | $(M) SED $@ 15 | $(Q)sed -e '/^#/d;s/\([\'\''"]\)/\\\1/g;s/^/"/;s/$$/\\n"/' $< >$@ 16 | 17 | $(eval $(foreach NAME,$(SNIFF_TASKS),$(O)/.objs/src/plugins/sniff/$(NAME).o: $(O)/.objs/sniff-$(NAME).inc${N})) 18 | -------------------------------------------------------------------------------- /src/plugins/sniff/cert.h: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2014 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #ifndef UCOLLECT_SNIFF_CERT_H 21 | #define UCOLLECT_SNIFF_CERT_H 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | struct task_data; 29 | struct context; 30 | struct mem_pool; 31 | 32 | struct task_data *start_cert(struct context *context, struct mem_pool *pool, const uint8_t *message, size_t message_size, int *output, pid_t *pid); 33 | const uint8_t *finish_cert(struct context *context, struct task_data *data, uint8_t *output, size_t output_size, size_t *result_size, bool *ok); 34 | 35 | 36 | #endif 37 | -------------------------------------------------------------------------------- /src/plugins/sniff/fork.c: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2014-2015 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #include "fork.h" 21 | 22 | #include "../../core/util.h" 23 | #include "../../core/loop.h" 24 | 25 | #include 26 | #include 27 | #include 28 | 29 | bool fork_task(struct loop *loop, const char *program, char **argv, const char *name, int *output, pid_t *pid) { 30 | int pipes[2]; 31 | if (pipe(pipes) == -1) { 32 | ulog(LLOG_ERROR, "Couldn't create %s pipes: %s\n", name, strerror(errno)); 33 | return false; 34 | } 35 | pid_t new_pid = loop_fork(loop); 36 | if (new_pid == -1) { 37 | ulog(LLOG_ERROR, "Couldn't create new %s process: %s\n", name, strerror(errno)); 38 | if (close(pipes[0]) == -1) 39 | ulog(LLOG_ERROR, "Failed to close %s read pipe: %s\n", name, strerror(errno)); 40 | if (close(pipes[1]) == -1) 41 | ulog(LLOG_ERROR, "Failed to close %s write pipe: %s\n", name, strerror(errno)); 42 | return false; 43 | } 44 | if (new_pid == 0) { // We are the child now. 45 | sanity(close(pipes[0]) != -1, "Failed to close %s read pipe in child: %s\n", name, strerror(errno)); 46 | sanity(dup2(pipes[1], 1) != -1, "Failed to assign stdout of %s: %s\n", name, strerror(errno)); 47 | sanity(close(pipes[1]) != -1, "Failed to close copy of %s write pipe: %s\n", name, strerror(errno)); 48 | execv(program, argv); 49 | sanity(false, "Failed to execute %s (%s): %s\n", name, program, strerror(errno)); 50 | } else { 51 | if (close(pipes[1]) == -1) 52 | ulog(LLOG_ERROR, "Couldn't close %s write pipe: %s\n", name, strerror(errno)); 53 | ulog(LLOG_DEBUG, "Task %s (%s) started with FD %d and PID %d\n", name, program, pipes[0], (int) new_pid); 54 | *output = pipes[0]; 55 | *pid = new_pid; 56 | } 57 | return true; 58 | } 59 | -------------------------------------------------------------------------------- /src/plugins/sniff/fork.h: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2014 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #ifndef UCOLLECT_SNIFF_FORK 21 | #define UCOLLECT_SNIFF_FORK 22 | 23 | #include 24 | #include 25 | #include 26 | 27 | struct loop; 28 | 29 | bool fork_task(struct loop *loop, const char *program, char **argv, const char *name, int *output, pid_t *pid); 30 | 31 | #endif 32 | -------------------------------------------------------------------------------- /src/plugins/sniff/getip.cgi: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Ucollect - small utility for real-time analysis of network data 4 | # Copyright (C) 2014 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 5 | # 6 | # This program is free software; you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation; either version 2 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License along 17 | # with this program; if not, write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 19 | 20 | echo 'Content-Type: text/plain' 21 | echo 22 | echo "$REMOTE_ADDR" 23 | -------------------------------------------------------------------------------- /src/plugins/sniff/nop.c: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2014 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #include "nop.h" 21 | 22 | struct task_data *start_nop(struct context *context, struct mem_pool *pool, const uint8_t *message, size_t message_size, int *output, pid_t *pid) { 23 | (void) context; 24 | (void) pool; 25 | (void) message; 26 | (void) message_size; 27 | *output = 0; 28 | *pid = 0; 29 | return NULL; 30 | } 31 | 32 | const uint8_t *finish_nop(struct context *context, struct task_data *data, uint8_t *output, size_t output_size, size_t *result_size, bool *ok) { 33 | (void) context; 34 | (void) data; 35 | (void) output; 36 | (void) output_size; 37 | *result_size = 0; 38 | *ok = true; 39 | return NULL; 40 | } 41 | -------------------------------------------------------------------------------- /src/plugins/sniff/nop.h: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2014 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #ifndef UCOLLECT_SNIFF_NOP_H 21 | #define UCOLLECT_SNIFF_NOP_H 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | struct task_data; 29 | struct context; 30 | struct mem_pool; 31 | 32 | /* 33 | * Implementation of task that does nothing at all, just finishes successfully. 34 | * 35 | * The start_nop signals nothing was started and that the finish should be called right away. 36 | * The finish successfully produces empty output. 37 | * 38 | * No parameters are checked for sanity, only the output parameters and results are set. 39 | */ 40 | 41 | struct task_data *start_nop(struct context *context, struct mem_pool *pool, const uint8_t *message, size_t message_size, int *output, pid_t *pid); 42 | const uint8_t *finish_nop(struct context *context, struct task_data *data, uint8_t *output, size_t output_size, size_t *result_size, bool *ok); 43 | 44 | #endif 45 | -------------------------------------------------------------------------------- /src/plugins/sniff/parse.c: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2014 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #include "parse.h" 21 | #include "fork.h" 22 | 23 | #include "../../core/mem_pool.h" 24 | #include "../../core/util.h" 25 | #include "../../core/context.h" 26 | 27 | #include 28 | #include 29 | 30 | struct task_data *input_parse(struct context *context, struct mem_pool *pool, const uint8_t *message, size_t message_size, int *output, pid_t *pid, const char *program, const char *name, size_t params_per_target, size_t target_size, task_parse_t task_parse) { 31 | *output = 0; // Not running yet 32 | struct task_data *data = mem_pool_alloc(pool, sizeof *data); 33 | *data = (struct task_data) { 34 | .input_ok = true, 35 | .system_ok = true 36 | }; 37 | uint16_t target_count; 38 | if (message_size < sizeof target_count) { 39 | ulog(LLOG_ERROR, "%s input broken: Message too short to contain even the number of hosts (%zu bytes)\n", name, message_size); 40 | data->input_ok = false; 41 | return data; 42 | } 43 | memcpy(&target_count, message, sizeof target_count); 44 | message += sizeof target_count; 45 | message_size -= sizeof target_count; 46 | target_count = ntohs(target_count); 47 | char **argv = mem_pool_alloc(context->temp_pool, (6 + params_per_target * target_count) * sizeof *argv); 48 | argv[0] = "/bin/busybox"; 49 | argv[1] = "ash"; 50 | argv[2] = "-c"; 51 | argv[3] = mem_pool_strdup(context->temp_pool, program); 52 | argv[4] = mem_pool_printf(context->temp_pool, "sniff-%s", name); 53 | argv[5 + params_per_target * target_count] = NULL; 54 | data->target_count = target_count; 55 | data->targets = mem_pool_alloc(pool, target_count * target_size); 56 | for (size_t i = 0; i < target_count; ++ i) 57 | if (!task_parse(pool, context->temp_pool, (struct target *)((uint8_t *) data->targets + i * target_size), argv + 5 + params_per_target * i, &message, &message_size, i)) { 58 | data->input_ok = false; 59 | return data; 60 | } 61 | data->system_ok = fork_task(context->loop, "/bin/busybox", argv, name, output, pid); 62 | return data; 63 | } 64 | -------------------------------------------------------------------------------- /src/plugins/sniff/parse.h: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2014 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #ifndef UCOLLECT_SNIFF_PARSE_H 21 | #define UCOLLECT_SNIFF_PARSE_H 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | struct target; 29 | 30 | struct task_data { 31 | bool input_ok; 32 | bool system_ok; 33 | size_t target_count; 34 | struct target *targets; 35 | }; 36 | 37 | struct context; 38 | struct mem_pool; 39 | 40 | typedef bool (*task_parse_t)(struct mem_pool *task_pool, struct mem_pool *tmp_pool, struct target *target, char **args, const uint8_t **message, size_t *message_size, size_t index); 41 | 42 | struct task_data *input_parse(struct context *context, struct mem_pool *pool, const uint8_t *message, size_t message_size, int *output, pid_t *pid, const char *program, const char *name, size_t params_per_target, size_t target_size, task_parse_t task_parse); 43 | 44 | #endif 45 | -------------------------------------------------------------------------------- /src/plugins/sniff/ping.h: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2014 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #ifndef UCOLLECT_SNIFF_PING_H 21 | #define UCOLLECT_SNIFF_PING_H 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | struct task_data; 29 | struct context; 30 | struct mem_pool; 31 | 32 | struct task_data *start_ping(struct context *context, struct mem_pool *pool, const uint8_t *message, size_t message_size, int *output, pid_t *pid); 33 | const uint8_t *finish_ping(struct context *context, struct task_data *data, uint8_t *output, size_t output_size, size_t *result_size, bool *ok); 34 | 35 | #endif 36 | -------------------------------------------------------------------------------- /src/plugins/sniff/sniff-cert: -------------------------------------------------------------------------------- 1 | #!/bin/busybox ash 2 | 3 | # Ucollect - small utility for real-time analysis of network data 4 | # Copyright (C) 2014 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 5 | # 6 | # This program is free software; you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation; either version 2 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License along 17 | # with this program; if not, write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 19 | 20 | DIR="/tmp/sniff-cert-$$" 21 | trap 'rm -rf "$DIR"' EXIT ABRT QUIT TERM INT 22 | 23 | while [ "$1" ] ; do 24 | HOST="$1" 25 | PORT="$2" 26 | STARTTLS="$3" 27 | PARAMS="" 28 | if [ "$STARTTLS" ] ; then 29 | PARAMS="-starttls $STARTTLS" 30 | fi 31 | mkdir -p "$DIR" 32 | cd "$DIR" 33 | echo -n '' | openssl s_client -servername "$HOST" -connect "$HOST":"$PORT" $PARAMS -showcerts >out 2>/dev/null 34 | awk -v c=-1 '/-----BEGIN CERTIFICATE-----/{inc=1;c++} inc {print > (c ".pem")}' session 36 | echo '-----BEGIN HOST-----' 37 | if [ -s out ] ; then 38 | echo '-----CIPHER-----' 39 | sed -ne 's/ *Cipher *: \(.*\)/\1/p' session 40 | echo '-----PROTOCOL-----' 41 | sed -ne 's/ *Protocol *: \(.*\)/\1/p' session 42 | ls *.pem | sort -n | while read cert ; do 43 | openssl x509 -in "$cert" 44 | echo '-----FINGERPRINT-----' 45 | openssl x509 -in "$cert" -noout -fingerprint -sha1 | sed -e 's/.*=//;s/://g' 46 | echo '-----NAME-----' 47 | ( 48 | openssl x509 -in "$cert" -noout -text | sed -ne 's/ *Subject:.*CN=\([^ \/]*\).*/\1/p' 49 | openssl x509 -in "$cert" -noout -text | grep -A 1 'X509v3 Subject Alternative Name' | grep -o 'DNS:[^ ,]*' | sed -e 's/DNS://' 50 | ) | sort -u 51 | echo '-----EXPIRY-----' 52 | 53 | openssl x509 -in "$cert" -noout -text | sed -ne 's/ *Not After *: *\([^ ].*[^ ]\) */\1/p' 54 | done 55 | fi 56 | echo '-----END HOST-----' 57 | cd /tmp 58 | rm -rf "$DIR" 59 | shift 3 60 | done 61 | -------------------------------------------------------------------------------- /src/plugins/sniff/sniff-nat: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | LOCIP=$(ip addr show | sed -ne '/ *inet6\? *\([^ ]*\) .*/s//\1/p') 4 | 5 | test() { 6 | IP=$(curl -skf$1 https://api.turris.cz/getip.cgi) 7 | if [ "$?" = "0" ] ; then 8 | if echo "$LOCIP" | grep -qF "$IP" ; then 9 | echo 'DIRECT' 10 | else 11 | echo 'NAT' 12 | fi 13 | else 14 | echo 'NONE' 15 | fi 16 | } 17 | 18 | test 4 19 | test 6 20 | -------------------------------------------------------------------------------- /src/plugins/sniff/sniff-ping: -------------------------------------------------------------------------------- 1 | #!/bin/busybox ash 2 | 3 | # Ucollect - small utility for real-time analysis of network data 4 | # Copyright (C) 2014 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 5 | # 6 | # This program is free software; you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation; either version 2 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License along 17 | # with this program; if not, write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 19 | 20 | while [ "$1" ] ; do 21 | PROTO="$1" 22 | COUNT="$2" 23 | SIZE="$3" 24 | HOST="$4" 25 | shift 4 26 | if echo "$HOST" | grep -q "'" ; then 27 | echo 'BAD INPUT'; 28 | continue 29 | fi 30 | PARAMS='' 31 | case "$PROTO" in 32 | 4) 33 | PARAMS="$PARAMS -4" 34 | ;; 35 | 6) 36 | PARAMS="$PARAMS -6" 37 | ;; 38 | X) 39 | ;; 40 | *) 41 | echo 'BAD INPUT' 42 | continue 43 | ;; 44 | esac 45 | PARAMS="$PARAMS -c $COUNT -s $SIZE '$HOST'" 46 | eval "busybox ping $PARAMS" 2>/dev/null | sed -ne 's/.*(\(.*\)):.*/\1/p;s/.*seq=\([^ ]*\).*time=\([^ ]*\).*/\1:\2/p' | tr '\n' ' ' ; echo END 47 | done 48 | -------------------------------------------------------------------------------- /src/plugins/sniff/task.c: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2014 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #include "task.h" 21 | #include "nop.h" 22 | #include "ping.h" 23 | #include "cert.h" 24 | 25 | struct task_desc task_descs[] = { 26 | { 27 | .name = 'N', 28 | .label = "NOP", 29 | .start = start_nop, 30 | .finish = finish_nop 31 | }, 32 | { 33 | .name = 'P', 34 | .label = "Ping", 35 | .start = start_ping, 36 | .finish = finish_ping 37 | }, 38 | { 39 | .name = 'C', 40 | .label = "Certificate", 41 | .start = start_cert, 42 | .finish = finish_cert 43 | }, 44 | { 45 | .name = '\0' 46 | } 47 | }; 48 | -------------------------------------------------------------------------------- /src/plugins/sniff/task.h: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2014 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #ifndef UCOLLECT_SNIFF_TASK_H 21 | #define UCOLLECT_SNIFF_TASK_H 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | struct task_data; 29 | struct context; 30 | struct mem_pool; 31 | 32 | /* 33 | * Description of task type. 34 | * 35 | * A server requests a task to be run, it is then started in background process. 36 | * The output is then sent from the subtask to its stdout and read. It is then 37 | * postprocessed and answer sent to server. 38 | */ 39 | struct task_desc { 40 | char name; // Single-character name of the task type, as sent over the protocol. 41 | char *label; // Human readable label of the type, for logs. 42 | /* 43 | * Start the task. 44 | * 45 | * Context is the plugin's context. 46 | * The pool is for data that can be needed during the time of the task, it is guaranteed to be valid until finish is called. 47 | * Message is what arrived from the server, parameters of the task. Interpretation is up to the task type. If you need to preserve it for future, make a copy, it will be invalidated soon. 48 | * The output is FD where the output will appear and pid is PID of the process. 49 | * 50 | * The return value is then passed to finish unmodified, it is meant as private task data. 51 | * 52 | * If you return 0 as out, it is considered the task failed to start. The finish is called right away to generate output. 53 | */ 54 | struct task_data *(*start)(struct context *context, struct mem_pool *pool, const uint8_t *message, size_t message_size, int *output, pid_t *pid); 55 | /* 56 | * Postprocess the task. It is run after the output FD is closed. 57 | * 58 | * The context is context of the plugin. 59 | * Data is what was returned from start. 60 | * Output (and output size) is what got out of the output FD. NULL means there was an error ‒ either start signaled one or it wasn't possible to read the output. If there's no data output from out FD, but it is closed correctly, output_size is 0, but non-NULL pointer is passed. 61 | * The result_size is size of the message to be sent to the server. 62 | * OK is output parameter signalling if all went well. 63 | * 64 | * Return pointer to data to be sent to server (it may be allocated either from the memory pool passed to start or temporary pool in the context, either is fine). 65 | * 66 | * Note, however, that finish might be ommited by the plugin (for example, when the task gets aborted). 67 | */ 68 | const uint8_t *(*finish)(struct context *context, struct task_data *data, uint8_t *output, size_t output_size, size_t *result_size, bool *ok); 69 | }; 70 | 71 | extern struct task_desc task_descs[]; 72 | 73 | #endif 74 | -------------------------------------------------------------------------------- /src/plugins/spoof/Makefile: -------------------------------------------------------------------------------- 1 | RESTRICT := src/plugins/spoof 2 | RELATIVE := ../../../ 3 | 4 | include $(RELATIVE)/Makefile 5 | -------------------------------------------------------------------------------- /src/plugins/spoof/Makefile.dir: -------------------------------------------------------------------------------- 1 | LIBRARIES += src/plugins/spoof/libplugin_spoof 2 | libplugin_spoof_MODULES := spoof 3 | 4 | DOCS += src/plugins/spoof/spoof 5 | -------------------------------------------------------------------------------- /src/plugins/spoof/spoof.txt: -------------------------------------------------------------------------------- 1 | The spoof plugin 2 | ================ 3 | 4 | The goal of this plugin is to check if ISPs properly block packets 5 | with spoofed addresses (addresses not in their ranges). 6 | 7 | Upon request, the plugin sends two packets to the server. One is with 8 | the correct source address, the other is spoofed (the correct one is 9 | to check reachability). The server than looks into the content of the 10 | address and pairs it to the client, storing the result to database. 11 | 12 | Currently, it works only on IPv4. 13 | 14 | The request 15 | ----------- 16 | 17 | The requests starts with the opcode `'4'` (character '4', not byte 18 | with number 4). This is to provide possibility of future extension to 19 | other types of requests. 20 | 21 | Then there are these fields: 22 | 23 | source address:: This is an `uint32_t` field, and it is the IP address 24 | that is set as source address on the spoofed packet. 25 | destination address:: Similarly, an `uint32_t` field, meaning the 26 | address both packets are sent to. 27 | port:: An `uint16_t` field, specifying the destination UDP port. 28 | token:: An `uint64_t` token used for pairing packets to clients. It is 29 | randomly generated number. 30 | 31 | The packet 32 | ---------- 33 | 34 | The packet is a UDP packet, with given port and addresses. It 35 | contains: 36 | 37 | Magic signature:: This is `uint32_t` number, always set to 38 | `0x17ACEE43`. This helps to verify the packet is ours and not 39 | something stray wandering the Internet. 40 | Token:: An `uint64_t` number, copied verbatim from the request. It is 41 | used to pair the packet on the server side when received. 42 | Spoofed:: A `bool` value, set to `true` on the packet that has spoofed 43 | address, `false` on the other one. 44 | Message:: There's a character string explaining the packet is testing 45 | one. It has no meaning for the application itself, so it is ignored 46 | by the server. 47 | 48 | Generation of packet 49 | -------------------- 50 | 51 | The ordinary packet is sent first, the usual way (by opening a IP/UDP 52 | socket and using `sendto`). The packet is then captured when going out 53 | by WAN interface. The packet is examined and used as a template for 54 | the forged packet. This saves a lot of sniffing through the OS to find 55 | out needed information (like the MAC address of the gate, the correct 56 | interface to send the packet through, etc). The source address is 57 | modified according to the request and it is sent out though a 58 | AF_PACKET socket. 59 | -------------------------------------------------------------------------------- /src/ucollect/Makefile: -------------------------------------------------------------------------------- 1 | RESTRICT := src/ucollect 2 | RELATIVE := ../../ 3 | 4 | include $(RELATIVE)/Makefile 5 | -------------------------------------------------------------------------------- /src/ucollect/Makefile.dir: -------------------------------------------------------------------------------- 1 | BINARIES += src/ucollect/ucollect 2 | 3 | ucollect_MODULES := \ 4 | main 5 | ucollect_LOCAL_LIBS := ucollect_core 6 | # TODO: Make the build system take this from the ucollect_core somehow 7 | ucollect_SYSTEM_LIBS := pcap rt dl uci crypto ssl unbound atsha204 8 | 9 | DOCS += src/ucollect/ucollect 10 | -------------------------------------------------------------------------------- /src/ucollect/main.c: -------------------------------------------------------------------------------- 1 | /* 2 | Ucollect - small utility for real-time analysis of network data 3 | Copyright (C) 2013 CZ.NIC, z.s.p.o. (http://www.nic.cz/) 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | */ 19 | 20 | #include "../core/loop.h" 21 | #include "../core/util.h" 22 | #include "../core/uplink.h" 23 | #include "../core/configure.h" 24 | #include "../core/startup.h" 25 | #include "../core/tunable.h" 26 | #include "../core/mem_pool.h" 27 | 28 | #include 29 | #include 30 | 31 | static void dump_stats(struct context *context, void *data, size_t id) { 32 | (void)context; 33 | (void)data; 34 | (void)id; 35 | char *stats = mem_pool_stats(loop_temp_pool(loop)); 36 | char *tok; 37 | while ((tok = strtok(stats, ","))) { 38 | stats = NULL; 39 | while (*tok == ' ') 40 | tok ++; 41 | ulog(LLOG_INFO, "Mempool stats: %s\n", tok); 42 | } 43 | ulog(LLOG_INFO, "Mempool stats done\n"); 44 | loop_timeout_add(loop, STAT_DUMP_TIMEOUT, NULL, NULL, dump_stats); 45 | } 46 | 47 | int main(int argc, const char* argv[]) { 48 | (void) argc; 49 | openlog("ucollect", LOG_CONS | LOG_NDELAY | LOG_PID, LOG_DAEMON); 50 | 51 | if (argv[1]) { 52 | ulog(LLOG_DEBUG, "Setting config dir to %s\n", argv[1]); 53 | config_set_dir(argv[1]); 54 | } 55 | config_set_package("ucollect"); 56 | // Create the loop. 57 | loop = loop_create(); 58 | 59 | loop_timeout_add(loop, STAT_DUMP_TIMEOUT, NULL, NULL, dump_stats); 60 | 61 | // Connect upstream 62 | uplink = uplink_create(loop); 63 | // FIXME: Is it OK to hardcore it here? 64 | uplink_set_status_file(uplink, "/tmp/ucollect-status"); 65 | 66 | set_stop_signals(); 67 | 68 | if (!load_config(loop)) 69 | die("No configuration available\n"); 70 | 71 | // Run until a stop signal comes. 72 | loop_run(loop); 73 | 74 | system_cleanup(); 75 | return 0; 76 | } 77 | -------------------------------------------------------------------------------- /src/ucollect/main.cfg: -------------------------------------------------------------------------------- 1 | 2 | config uplink 3 | option name api.turris.cz 4 | option service 5679 5 | option cert "/etc/ssl/ucollect-server.pem" 6 | -------------------------------------------------------------------------------- /src/ucollect/ucollect-turris.cfg: -------------------------------------------------------------------------------- 1 | package 'ucollect' 2 | 3 | config interface 4 | option ifname 'eth2' 5 | -------------------------------------------------------------------------------- /src/ucollect/ucollect.cfg: -------------------------------------------------------------------------------- 1 | package 'ucollect' 2 | 3 | config interface 4 | option ifname 'eth0.2' 5 | -------------------------------------------------------------------------------- /src/ucollect/ucollect.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIFuDCCA6ACCQDrDx0WMy6WbTANBgkqhkiG9w0BAQUFADCBnTELMAkGA1UEBhMC 3 | Q1oxFzAVBgNVBAgMDkN6ZWNoIHJlcHVibGljMQ8wDQYDVQQHDAZQcmFndWUxDzAN 4 | BgNVBAoMBkNaLk5JQzENMAsGA1UECwwETGFiczEgMB4GA1UEAwwXc2VjdXJ0LXRl 5 | c3QubGFicy5uaWMuY3oxIjAgBgkqhkiG9w0BCQEWE21pY2hhbC52YW5lckBuaWMu 6 | Y3owHhcNMTMwODMwMTMyMjU5WhcNMTYwNjE5MTMyMjU5WjCBnTELMAkGA1UEBhMC 7 | Q1oxFzAVBgNVBAgMDkN6ZWNoIHJlcHVibGljMQ8wDQYDVQQHDAZQcmFndWUxDzAN 8 | BgNVBAoMBkNaLk5JQzENMAsGA1UECwwETGFiczEgMB4GA1UEAwwXc2VjdXJ0LXRl 9 | c3QubGFicy5uaWMuY3oxIjAgBgkqhkiG9w0BCQEWE21pY2hhbC52YW5lckBuaWMu 10 | Y3owggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCwq98ZW3Hf7PRTnP9i 11 | ytNJ1bnJID+mClK3MJfyXBe19Lw64m+Fw+uYMwreuFR20heLZfD6N0Vj87wBUv8U 12 | u5jB+pZVskja3gS6MHgEd+CLDX8wUKYlXntXHlcZmznvzMAc1OeF6OqrNDwvgePr 13 | k4wmduUvgqG01UNCIB0WIb5fyjbJe33d/3dr+wKfIwZ6Ewdak8laSfvtIHVPsZGz 14 | xriSYigobvw9IiyVPP9LWuNpVFHDN2Esu8m35GpQZbElXFVTqj7LBqfJ7cPlHAQv 15 | Zgwbqp7GgjqeewGcIThoq5Fk87PgP7XcFG5TwKVYYzkQ7rrbpmhfc+M2YVEhjSTE 16 | y5iZxW7iroPWKCoLZ1rRP6hFe6nWtVH1tfVAh8M+SSYpoB9a5ALQqrGz+l32uOrL 17 | r9PAfLmQ1APlqPSU4nESN6FKNNMaclZealBzWxPxHxyi5MRtD8FasEItQ2y+jieC 18 | gO/4+7MtPrNPaJHk7AJBt0PxOqDNbnbLGLfUTvC9nlBtJ+732uL5j7pvTlVBd4yA 19 | IpuNcCDxwsDjos8DyWUXi14QY8ajYpob58L3Wnq/H4KWtbTBq0gCp/eHvvPH0B7R 20 | aLVp5lIf6mibl6k83BBASku2jaZDLQD3cfSs5eAivQknx6zm9v78CzQsbHAIpgQy 21 | aQO4wnEXE12BDUArcnpFnG5WPwIDAQABMA0GCSqGSIb3DQEBBQUAA4ICAQBUXg51 22 | Xd0EdQhfngYuRRSl4GODcmRVHaF2XtzIe3uUQ6XNKrJLeD93EPvWDvgVE/h01VOk 23 | xKz5S80dQweSgi5h2TcMANMK7i32T07dXlXUCcBWAcRfzSMoHnf3AXpWJqZfWzgK 24 | Xwn0uMfzEnCVkEfrEAhbHU3zXI7A00jtK6RocwGDhhiT9nkqOu/aXcUUBw9r9inB 25 | goCTSvqciOhNMck5dfHsVlDjk/h97osiINKQHJux9Ct92rI1r/D84A32nWW+ntH6 26 | FRdGHw9XEIsAIwX6ygwl4tJJJ/Igm72pFx52MUxdALX1JCoxj6KJMqt3YuOagSs3 27 | I/UCTQW1UbJu4ktnq5JrMBu2fhfQDxEZ4bJxzwUf+ArV0PgRrmUJdOGOJqmiBLVB 28 | nDM15vbQ9PFVOzuCKlpCbnJDHcZMX9ZNSCJHfuyfVr9z1PBQdrzuHOBB1MXG7GJE 29 | s+IgZoKSn/vRFgMRYffNQiz3S+RMZw60aFD9GCMyj17wcQzu0U/yjmd2Sj/sz+37 30 | 9H+DXSXuRs8ddJJpsd2VyspHDA0jEsfu3lIU7hV/jc78OEEIfYd/ybNit+nVKvzR 31 | uCBz0gXFFRCreZta+lFUJqx2AMi13BR5qMqWvPun2F9qy10jB7N6lk7v+kapC+7o 32 | 44Feg66tiYZ0B/+j2vUBQIW2n5sl0kLys7NNng== 33 | -----END CERTIFICATE----- 34 | -------------------------------------------------------------------------------- /src/ucollect/ucollect.txt: -------------------------------------------------------------------------------- 1 | The `ucollect` binary 2 | ===================== 3 | 4 | The `ucollect` is the actual binary that watches some network 5 | interfaces and analyzes the packets by several plugins. It then sends 6 | the gathered data to the central server for processing. 7 | 8 | Invocation 9 | ---------- 10 | 11 | A configuration directory with UCI configs may be specified as the 12 | first (and only) parameter. If no parameter is set, the directory of 13 | `/etc/config` is used. 14 | 15 | The rest of configuration is read from there. 16 | 17 | Configuration 18 | ------------- 19 | 20 | The configuration is read from `/etc/config/ucollect` (or file 21 | `ucollect` from the directory set on command line). It uses the 22 | http://wiki.openwrt.org/doc/uci[uci] system and syntax. An example 23 | config would look like this: 24 | 25 | package 'ucollect' 26 | 27 | config interface 28 | option ifname 'eth0' 29 | option promiscuous 1 30 | 31 | config plugin 32 | option libname 'libplugin_count.so' 33 | 34 | config plugin 35 | option libname 'libplugin_buckets.so' 36 | list pluglib 'libpluglib_test.so' 37 | option some_opt 42 38 | 39 | config uplink 40 | option name localhost 41 | option service 5678 42 | option login some_name 43 | option password "A password" 44 | 45 | The `package 'ucollect'` is always the same and should not be changed. 46 | Then there are three kinds of sections. 47 | 48 | The `interface` section 49 | ~~~~~~~~~~~~~~~~~~~~~~~ 50 | 51 | This describes one interface to listen on. The option `ifname` specifies 52 | the name of the interface on the system. The option `promiscuous` may 53 | be set to 0 or 1 and specifies if promiscuous mode should be on on the 54 | device. It defaults to 1. 55 | 56 | The `plugin` section 57 | ~~~~~~~~~~~~~~~~~~~~ 58 | 59 | Each such section describes one analysis plugin to load. The option 60 | `libname` specifies the file name of the library to load as the 61 | plugin. 62 | 63 | The list `pluglib` lists all the needed plugin libraries. 64 | 65 | All options and lists (even ones not covered here) are preserved and 66 | provided to the plugin. Therefore, it allows for plugin-specific 67 | configuration. 68 | 69 | The `uplink` section 70 | ~~~~~~~~~~~~~~~~~~~~ 71 | 72 | It describes the server it should connect to. It contains the dns name 73 | (or ip address) in the `name` option and the port or service name in 74 | in the `service` option. 75 | 76 | There's also the login info. The login name is in the `login` option 77 | and password in `password`. However, default compilation 78 | authenticates through libatsha204 and doesn't understant these options 79 | (just omit them). 80 | 81 | There should be exactly one instance of this config section. 82 | 83 | Signals 84 | ------- 85 | 86 | Apart from normal signal handling (like terminanig on `SIGINT`), these 87 | two work with configuration: 88 | 89 | SIGHUP:: 90 | Re-reads the configuration and applies the changes. 91 | SIGUSR1:: 92 | Reinitialize all the plugins and interfaces according to current 93 | configuration. The difference from `SIGHUP` is that `SIGHUP` does 94 | not change a plugin or interface that is the same in the old and new 95 | versions. This one unloads it first and then loads again. 96 | -------------------------------------------------------------------------------- /tools/hammer.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | use strict; 3 | use warnings; 4 | use Socket qw(PF_INET SOCK_DGRAM pack_sockaddr_in inet_aton); 5 | use Getopt::Long; 6 | use Time::HiRes qw(gettimeofday usleep); 7 | 8 | # This script can be used to hammer a device with large amounts of UDP packets. 9 | # It is used to see when the device starts dropping packets and 10 | # performance-check it in that way. 11 | # 12 | # Each $interval seconds a burst of $burst packets is sent, each having $size 13 | # payload (and having a header too). 14 | # 15 | # It sends only over IPv4. It seems unimportant which protocol is used when 16 | # the only purpose is to send many packets. 17 | # 18 | # It is not expected the other side will listen to the packets. They would be 19 | # simply dropped, but any analysis tool will get them with pcap. 20 | 21 | # Get the options. 22 | my $port = 12345; # Just arbitrary port. We don't expect the other side to listen there. 23 | my $address = 'localhost'; # Which host to spam 24 | my $burst = 10; # How many packets in each burst 25 | my $size = 100; # Size of the payload of the packet 26 | my $interval = 0.01; # Time between starts of bursts. 27 | my $srccount = 1; 28 | 29 | GetOptions 30 | 'port=i' => \$port, 31 | 'address=s' => \$address, 32 | 'burst=i' => \$burst, 33 | 'size=i' => \$size, 34 | 'interval=f' => \$interval, 35 | 'origincount=i' => \$srccount, 36 | or exit 1; 37 | 38 | # Some information about how much will be sent. 39 | my $speed = ($size + 20) * 8; # Size of one packet with IP headers, in bits 40 | $speed *= $burst; # Size of one burst 41 | $speed /= $interval; # Size per second 42 | $speed /= (1024 ** 2); 43 | print "Expected throughput: $speed MBits, ", $burst / $interval, " packets/s\n"; 44 | 45 | # Connect the socket (which only sets the address with UDP packet, but we don't want 46 | # to provide it each time). 47 | my @sockets; 48 | for (0..$srccount) { 49 | socket(my $socket, PF_INET, SOCK_DGRAM, 0) or die "Could not create socket ($!)\n"; 50 | connect($socket, pack_sockaddr_in($port, inet_aton($address))) or die "Could not connect the UDP socket ($!)\n"; 51 | push @sockets, $socket; 52 | } 53 | 54 | # Payload of the packet 55 | my $payload = ' ' x $size; 56 | 57 | # Time when the current burst started. 58 | my $start_time = gettimeofday; 59 | 60 | # Loop forever. Each loop is one burst of packets. 61 | while (1) { 62 | # Send burst of packets first. 63 | for (1..$burst) { 64 | my $result = send $sockets[int rand $srccount], $payload, 0; 65 | if (defined $result) { 66 | if ($result < $size) { 67 | warn "Packet only $result of $size bytes\n"; 68 | } 69 | } else { 70 | # We don't expect the other side to listen, so the 'Connection refused' is expected. 71 | warn "Error sending packet ($!)\n" unless $! eq 'Connection refused'; 72 | } 73 | } 74 | # Wait for the rest of the interval. 75 | my $time = gettimeofday; 76 | my $elapsed = $time - $start_time; 77 | if ($elapsed < $interval) { 78 | usleep 1000000 * ($interval - $elapsed); 79 | $time = gettimeofday; 80 | } 81 | $start_time = $time; 82 | } 83 | -------------------------------------------------------------------------------- /tools/update_hash.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | use common::sense; 3 | use Digest::SHA; 4 | use File::Path; 5 | use Getopt::Long; 6 | 7 | my (@plugins, %branches, $url, $pass_file, $default, $serie, $categories); 8 | 9 | GetOptions 10 | 'url=s' => \$url, 11 | 'default=s' => \$default, 12 | 'serie=s' => \$serie, 13 | 'pass-file=s' => \$pass_file, 14 | "branch=s%" => \%branches, 15 | 'categories=s' => \$categories, 16 | "plugins=s" => \@plugins or die "Bad params"; 17 | 18 | sub parse($) { 19 | my ($hexes) = @_; 20 | return map hex, $hexes =~ /(.{2})/g; 21 | } 22 | 23 | # Read the compiled-in password 24 | open my $pf, '<', $pass_file or die "Could not read password file $pass_file: $!\n"; 25 | my $passwd = <$pf>; 26 | close $pf; 27 | chomp $passwd; 28 | $passwd =~ s/[,{}]//g; 29 | $passwd =~ s/\s//g; 30 | $passwd =~ s/0x//g; 31 | 32 | open my $cfile, '<', $categories or die "Could not read category file '$categories': $!\n"; 33 | my %categories; 34 | my $current; 35 | while (<$cfile>) { 36 | chomp; 37 | s/\s*(|#.*)$//; 38 | next unless /\S/; 39 | if (/(\S+):/) { 40 | $current = $1; 41 | } elsif (/(\S+)/) { 42 | $categories{$1} = $current; 43 | } 44 | } 45 | close $cfile; 46 | 47 | sub get_hash($) { 48 | my ($list) = @_; 49 | # Get versions of packages 50 | open my $packages, '-|', 'wget', "$url/lists/$list", '-O', '-' or die "Couldn't download package list: $!\n"; 51 | my @packages = <$packages>; 52 | close $packages or die "Error downloading package list: $!\n"; 53 | 54 | my %packages = map { 55 | my ($name, $version) = split; 56 | $name => "$url/packages/$name-$version.ipk" 57 | } @packages; 58 | 59 | my %paths = map { 60 | my ($name, $version) = split; 61 | my $file = $name; 62 | $file =~ s/-/_/; 63 | $name => "usr/lib/libplugin_${file}_${version}.so" 64 | } @packages; 65 | 66 | my @passwd = parse $passwd; 67 | 68 | my $had_plugin; 69 | 70 | for my $plugin (@plugins) { 71 | system 'wget', ($packages{$plugin} // next), '-O', 'package.ipk' and die "Couldn't download package $packages{$plugin}\n"; 72 | system 'tar', 'xf', 'package.ipk' and die "Can't unpack package for $plugin\n"; 73 | system 'tar', 'xf', 'data.tar.gz' and die "Can't unpack data for $plugin\n"; 74 | my $library = $paths{$plugin}; 75 | my $sha = Digest::SHA->new(256); 76 | $sha->addfile($library); 77 | my $digest = $sha->hexdigest; 78 | warn "Digest of $plugin is $digest\n"; 79 | my @digest = parse $digest; 80 | for my $i (0..@passwd - 1) { 81 | $passwd[$i] ^= $digest[$i]; 82 | } 83 | rmtree 'usr'; 84 | $had_plugin = 1; 85 | } 86 | 87 | return join '', map sprintf("%02X", $_), @passwd; 88 | } 89 | my $hash = get_hash $default; 90 | my %hashes = map { 91 | $_ => get_hash $_ 92 | } values %branches; 93 | say "UPDATE clients SET builtin_passwd = '$hash' WHERE name like '$serie%';"; 94 | say "UPDATE clients SET builtin_passwd = '$hashes{$branches{$categories{$_}}}' WHERE name = '" . lc($_) . "';" for grep { /^$serie/ } keys %categories; 95 | --------------------------------------------------------------------------------