├── archive ├── vortex │ ├── 2.7 │ │ ├── vortex-2.7.tgz │ │ ├── vortex-2.7-41.LM.src.rpm │ │ └── vortex-2.7-41.LM.x86_64.rpm │ ├── 2.7.1 │ │ ├── vortex-2.7.1.tgz │ │ ├── vortex-2.7.1-44.el5.src.rpm │ │ └── vortex-2.7.1-44.el5.x86_64.rpm │ ├── 2.8.0 │ │ ├── vortex-2.8.0.tgz │ │ ├── vortex-2.8.0-46.el5.src.rpm │ │ └── vortex-2.8.0-46.el5.x86_64.rpm │ ├── 2.8.1 │ │ ├── vortex-2.8.1.tgz │ │ ├── vortex-2.8.1-49.el5.src.rpm │ │ ├── vortex-2.8.1-51.el5.src.rpm │ │ ├── vortex-2.8.1-49.el5.x86_64.rpm │ │ └── vortex-2.8.1-51.el5.x86_64.rpm │ └── 2.9.0 │ │ ├── vortex-2.9.0.tgz │ │ ├── vortex-2.9.0-59.tgz │ │ ├── vortex-2.9.0-57.el5.src.rpm │ │ ├── vortex-2.9.0-59.el5.src.rpm │ │ ├── vortex-2.9.0-57.el5.x86_64.rpm │ │ └── vortex-2.9.0-59.el5.x86_64.rpm └── libbsf │ └── 1.0.1 │ ├── libbsf-1.0.1.tgz │ ├── libbsf-1.0.1-4.LM.src.rpm │ └── libbsf-1.0.1-4.LM.x86_64.rpm ├── README.md ├── libbsf ├── libbsf.spec ├── bsf.h └── libbsf.c ├── vortex ├── vortex.init ├── vortex.conf ├── vortex.spec ├── README ├── LICENSE └── vortex.c ├── xpipes └── xpipes.c └── LICENSE /archive/vortex/2.7/vortex-2.7.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lmco/vortex-ids/HEAD/archive/vortex/2.7/vortex-2.7.tgz -------------------------------------------------------------------------------- /archive/libbsf/1.0.1/libbsf-1.0.1.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lmco/vortex-ids/HEAD/archive/libbsf/1.0.1/libbsf-1.0.1.tgz -------------------------------------------------------------------------------- /archive/vortex/2.7.1/vortex-2.7.1.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lmco/vortex-ids/HEAD/archive/vortex/2.7.1/vortex-2.7.1.tgz -------------------------------------------------------------------------------- /archive/vortex/2.8.0/vortex-2.8.0.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lmco/vortex-ids/HEAD/archive/vortex/2.8.0/vortex-2.8.0.tgz -------------------------------------------------------------------------------- /archive/vortex/2.8.1/vortex-2.8.1.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lmco/vortex-ids/HEAD/archive/vortex/2.8.1/vortex-2.8.1.tgz -------------------------------------------------------------------------------- /archive/vortex/2.9.0/vortex-2.9.0.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lmco/vortex-ids/HEAD/archive/vortex/2.9.0/vortex-2.9.0.tgz -------------------------------------------------------------------------------- /archive/vortex/2.9.0/vortex-2.9.0-59.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lmco/vortex-ids/HEAD/archive/vortex/2.9.0/vortex-2.9.0-59.tgz -------------------------------------------------------------------------------- /archive/vortex/2.7/vortex-2.7-41.LM.src.rpm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lmco/vortex-ids/HEAD/archive/vortex/2.7/vortex-2.7-41.LM.src.rpm -------------------------------------------------------------------------------- /archive/libbsf/1.0.1/libbsf-1.0.1-4.LM.src.rpm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lmco/vortex-ids/HEAD/archive/libbsf/1.0.1/libbsf-1.0.1-4.LM.src.rpm -------------------------------------------------------------------------------- /archive/vortex/2.7/vortex-2.7-41.LM.x86_64.rpm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lmco/vortex-ids/HEAD/archive/vortex/2.7/vortex-2.7-41.LM.x86_64.rpm -------------------------------------------------------------------------------- /archive/vortex/2.7.1/vortex-2.7.1-44.el5.src.rpm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lmco/vortex-ids/HEAD/archive/vortex/2.7.1/vortex-2.7.1-44.el5.src.rpm -------------------------------------------------------------------------------- /archive/vortex/2.8.0/vortex-2.8.0-46.el5.src.rpm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lmco/vortex-ids/HEAD/archive/vortex/2.8.0/vortex-2.8.0-46.el5.src.rpm -------------------------------------------------------------------------------- /archive/vortex/2.8.1/vortex-2.8.1-49.el5.src.rpm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lmco/vortex-ids/HEAD/archive/vortex/2.8.1/vortex-2.8.1-49.el5.src.rpm -------------------------------------------------------------------------------- /archive/vortex/2.8.1/vortex-2.8.1-51.el5.src.rpm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lmco/vortex-ids/HEAD/archive/vortex/2.8.1/vortex-2.8.1-51.el5.src.rpm -------------------------------------------------------------------------------- /archive/vortex/2.9.0/vortex-2.9.0-57.el5.src.rpm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lmco/vortex-ids/HEAD/archive/vortex/2.9.0/vortex-2.9.0-57.el5.src.rpm -------------------------------------------------------------------------------- /archive/vortex/2.9.0/vortex-2.9.0-59.el5.src.rpm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lmco/vortex-ids/HEAD/archive/vortex/2.9.0/vortex-2.9.0-59.el5.src.rpm -------------------------------------------------------------------------------- /archive/libbsf/1.0.1/libbsf-1.0.1-4.LM.x86_64.rpm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lmco/vortex-ids/HEAD/archive/libbsf/1.0.1/libbsf-1.0.1-4.LM.x86_64.rpm -------------------------------------------------------------------------------- /archive/vortex/2.7.1/vortex-2.7.1-44.el5.x86_64.rpm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lmco/vortex-ids/HEAD/archive/vortex/2.7.1/vortex-2.7.1-44.el5.x86_64.rpm -------------------------------------------------------------------------------- /archive/vortex/2.8.0/vortex-2.8.0-46.el5.x86_64.rpm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lmco/vortex-ids/HEAD/archive/vortex/2.8.0/vortex-2.8.0-46.el5.x86_64.rpm -------------------------------------------------------------------------------- /archive/vortex/2.8.1/vortex-2.8.1-49.el5.x86_64.rpm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lmco/vortex-ids/HEAD/archive/vortex/2.8.1/vortex-2.8.1-49.el5.x86_64.rpm -------------------------------------------------------------------------------- /archive/vortex/2.8.1/vortex-2.8.1-51.el5.x86_64.rpm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lmco/vortex-ids/HEAD/archive/vortex/2.8.1/vortex-2.8.1-51.el5.x86_64.rpm -------------------------------------------------------------------------------- /archive/vortex/2.9.0/vortex-2.9.0-57.el5.x86_64.rpm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lmco/vortex-ids/HEAD/archive/vortex/2.9.0/vortex-2.9.0-57.el5.x86_64.rpm -------------------------------------------------------------------------------- /archive/vortex/2.9.0/vortex-2.9.0-59.el5.x86_64.rpm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lmco/vortex-ids/HEAD/archive/vortex/2.9.0/vortex-2.9.0-59.el5.x86_64.rpm -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # vortex-ids 2 | Vortex is a near real time IDS and network surveillance engine for TCP stream data. Vortex decouples packet capture, stream reassembly, and real time constraints from analysis. Vortex is used to provide TCP stream data to a separate analyzer program. 3 | 4 | vortex-ids is comprised of vortex, the core utility, and includes some additional utilites to enhance both its functionality, and its ability to integrate with outside analyzers. 5 | 6 | ## libBSF 7 | libBSF is a stream filtering library based on BPF and tcpdump syntax that vortex can use. 8 | libBSF is an optional component of vortex, however it is included by default in the vortex spec file. 9 | 10 | ## xpipes 11 | xpipes is a simple utility for multiplexing pipes that is similar to parallel and xargs but designed to work in situations where items are multiplexed to long running programs reading from STDIN. 12 | xpipes is intended to be used in conjunction with vortex to create multi-threaded analyzers, but it can also be used as a standalone utility. 13 | -------------------------------------------------------------------------------- /libbsf/libbsf.spec: -------------------------------------------------------------------------------- 1 | Summary: BSF--A stream filtering mechanism 2 | Name: libbsf 3 | Version: 1.0.1 4 | Release: 4.LM 5 | License: GPLv2 6 | Group: System Environment/Libraries 7 | Url: http://vortex-ids.sourceforge.net/ 8 | Source0: bsf.h 9 | Source1: libbsf.c 10 | BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root 11 | Requires: libpcap 12 | BuildRequires: libpcap, libpcap-devel 13 | 14 | 15 | %description 16 | BSF is a stream filtering mechanism based on BPF. 17 | 18 | %prep 19 | cp -fp %{SOURCE0} ./ 20 | cp -fp %{SOURCE1} ./ 21 | 22 | 23 | %build 24 | #gcc %optflags libbsf.c -o libbsf.o 25 | #gcc %optflags -static libbsf.o -o libbsf.a -lpcap 26 | gcc %optflags -fPIC -shared -Wl,-soname,libbsf.so.%{version} libbsf.c -o libbsf.so.%{version} 27 | 28 | 29 | 30 | %install 31 | %{__rm} -rf %{buildroot} 32 | mkdir -p $RPM_BUILD_ROOT/%{_libdir} 33 | mkdir -p $RPM_BUILD_ROOT/%{_includedir} 34 | #install -m 644 -p $RPM_BUILD_DIR/libbsf.a $RPM_BUILD_ROOT%{_libdir}/libbsf.a 35 | install -m 644 -p $RPM_BUILD_DIR/libbsf.so.%{version} $RPM_BUILD_ROOT%{_libdir}/libbsf.so.%{version} 36 | ln -s %{_libdir}/libbsf.so.%{version} $RPM_BUILD_ROOT%{_libdir}/libbsf.so 37 | install -m 644 -p $RPM_SOURCE_DIR/bsf.h $RPM_BUILD_ROOT%{_includedir}/bsf.h 38 | 39 | 40 | %clean 41 | %{__rm} -rf %{buildroot} 42 | 43 | %files 44 | %defattr(-, root, root, 0744) 45 | #%{_libdir}/libbsf.a 46 | %{_libdir}/libbsf.so 47 | %{_libdir}/libbsf.so.%{version} 48 | %{_includedir}/bsf.h 49 | 50 | %changelog 51 | 52 | * Fri Jul 3 2009 Charles Smutz - 1.0.1-4 53 | - Rebuilt 54 | * Fri Mar 23 2009 Charles Smutz - 1.0.1-3 55 | - Package for release as open source 56 | * Fri Jan 01 2009 Charles Smutz - 1.0.0-1 57 | - Initial Package 58 | -------------------------------------------------------------------------------- /vortex/vortex.init: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # chkconfig: - 55 45 4 | # 5 | # Init script for vortex based utilities 6 | # 7 | # 8 | # 9 | 10 | . /etc/init.d/vortex_conf_parser 11 | 12 | 13 | 14 | # Source function library. 15 | . /etc/init.d/functions 16 | 17 | # Source networking configuration. 18 | . /etc/sysconfig/network 19 | 20 | # Check that networking is up. 21 | [ "${NETWORKING}" = "no" ] && exit 0 22 | 23 | 24 | 25 | 26 | 27 | RETVAL=0 28 | 29 | chktmpdir() 30 | { 31 | if [ -e $VORTEX_OPTION_DATA_DIR ] 32 | then 33 | echo "$VORTEX_OPTION_DATA_DIR already exists" > /dev/null 34 | else 35 | echo "$VORTEX_OPTION_DATA_DIR doesn't exist. Creating...." 36 | mkdir -p $VORTEX_OPTION_DATA_DIR 37 | #chown to specified user if specified 38 | if [ -n "$VORTEX_USER" ] 39 | then 40 | chown $VORTEX_USER $VORTEX_OPTION_DATA_DIR 41 | fi 42 | fi 43 | } 44 | 45 | start() { 46 | echo -n $"Starting $VORTEX_ANALYZER_NAME: " 47 | chktmpdir 48 | eval $VORTEX_COMMAND1 49 | daemon $VORTEX_COMMAND2 2>/dev/null 50 | RETVAL=$? 51 | echo 52 | [ $RETVAL -eq 0 ] && touch /var/lock/subsys/$VORTEX_PROGRAM_NAME 53 | return $RETVAL 54 | } 55 | 56 | stop() { 57 | echo -n $"Shutting down $VORTEX_ANALYZER_NAME: " 58 | killproc $VORTEX_PROGRAM_NAME 59 | RETVAL=$? 60 | [ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/$VORTEX_PROGRAM_NAME 61 | echo 62 | return $RETVAL 63 | } 64 | 65 | dostatus() { 66 | status $VORTEX_PROGRAM_NAME 67 | } 68 | 69 | restart() { 70 | stop 71 | start 72 | } 73 | 74 | condrestart() { 75 | [ -e /var/lock/subsys/$VORTEX_PROGRAM_NAME ] && restart || : 76 | } 77 | 78 | # See how we were called. 79 | case "$1" in 80 | start) 81 | start 82 | ;; 83 | stop) 84 | stop 85 | ;; 86 | restart|reload) 87 | restart 88 | ;; 89 | condrestart) 90 | condrestart 91 | ;; 92 | status) 93 | dostatus 94 | ;; 95 | checktemp) 96 | chktmpdir 97 | ;; 98 | *) 99 | echo "Usage: $VORTEX_ANALYZER_NAME {start|stop|restart|reload|condrestart|status|checktemp}" 100 | exit 1 101 | esac 102 | 103 | exit $RETVAL 104 | -------------------------------------------------------------------------------- /vortex/vortex.conf: -------------------------------------------------------------------------------- 1 | # VORTEX CONFIG FILE 2 | 3 | #Name of the analyzer as a whole: 4 | VORTEX_ANALYZER_NAME="null" 5 | 6 | #Command to be executed. Vortex output will be provided to this command on STDIN 7 | VORTEX_ANALYZER_COMMAND="tee -a /tmp/vortex_null/deleted.log | xargs rm" 8 | 9 | #Symbolic link to vortex executable. Must be uniq for proper operation of init scripts, distinguishable entry in process listing. 10 | VORTEX_PROGRAM_PATH="/usr/bin/vortex_null" 11 | 12 | #Number of bytes going from client to server to collect: 13 | VORTEX_OPTION_SERVER_DATA=1048576 14 | 15 | #Number of bytes going from server to client to collect: 16 | VORTEX_OPTION_CLIENT_DATA=0 17 | 18 | #Interface to listen on 19 | VORTEX_OPTION_INTERFACE="bond0" 20 | 21 | #Directory for temporary storage of re-assembled streams. 22 | VORTEX_OPTION_DATA_DIR="/tmp/vortex_null/tmp/" 23 | 24 | #Pipe for vortex ouput. Use mkfifo to create this. 25 | VORTEX_PIPE="/tmp/vortex_null/pipe0" 26 | 27 | #Execute as the specified unpriveledged user. 28 | VORTEX_USER="nobody" 29 | 30 | #File containing TCP-dump style filter expression 31 | VORTEX_OPTION_FILTER_FILE="/etc/vortex/conf/null.filter" 32 | 33 | #Libnids parameter--Connection Hash Table size. Suggested value: 1048576 34 | VORTEX_OPTION_HASH_SIZE=2097152 35 | 36 | #Libnids parameter--IPfrag table size. Suggested value: 65538 37 | VORTEX_OPTION_IPFRAG_SIZE=65538 38 | 39 | #Disable line buffering. Change to any non-null value to disable. Default: null (line buffer is enabled) 40 | VORTEX_OPTION_DISABLE_LINE_BUFFERING= 41 | 42 | #Poll Rate Number of connection from which to sample a connection. In other words, if sample every 1/X connections where X is the poll rate specified below. (Default:1) 43 | VORTEX_OPTION_POLL_RATE= 44 | 45 | #Debug Level. Remember that if you enable this you add addtional processing possibly changing performance and behavior. Default=0 46 | VORTEX_OPTION_DEBUG_LEVEL= 47 | 48 | #Interval for reporting of Performance Statistics in seconds. Default: 0 Recommended 60 to 3600 49 | VORTEX_OPTION_STATS_INTERVAL=600 50 | 51 | #Interval for reporting of Performance Statistics in seconds. Default: 0 Recommended 60 to 3600 52 | VORTEX_OPTION_ERRORS_INTERVAL=600 53 | 54 | 55 | #Any other options to be thrown in raw 56 | VORTEX_OPTION_FREE_FORM= 57 | -------------------------------------------------------------------------------- /libbsf/bsf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2008-2009 Lockheed Martin Corporation 4 | * 5 | * The libBSF program is open source software: you can copy it, redistribute it and/or modify 6 | * it under the terms of the GNU General Public License version 2.0 as published by 7 | * the Free Software Foundation. The libBSF Program and any derivatives of the libBSF program 8 | * must be licensed under GPL version 2.0 and may not be licensed under GPL version 3.0. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY OF ANY KIND, including without limitation the implied warranties of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details at http://www.gnu.org/licenses/gpl-2.0.html. 14 | * 15 | * The term "libBSF" should be taken to also include any portions or derived works of libBSF. 16 | * 17 | * You are highly encouraged to send your changes to the Vortex program to 18 | * opensource.tools.security@lmco.com for possible incorporation into the main distribution. 19 | * By sending these changes to Lockheed Martin, you are granting to Lockheed Martin 20 | * Corporation the unlimited, perpetual, non-exclusive right to reuse, modify, 21 | * and/or relicense the code on a royalty-free basis. 22 | * 23 | * The libraries to which libBSF links are distributed under the terms of their own licenses. 24 | * Please see those libraries for their applicable licenses. 25 | * 26 | */ 27 | /* 28 | * 29 | * libbsf a stream filtering mechanism based on BPF and tcpdump filter syntax 30 | * 31 | */ 32 | 33 | #ifndef _BSF_BSF_H 34 | #define _BSF_BSF_H 35 | 36 | #include 37 | #include 38 | #include 39 | #include 40 | 41 | #include 42 | #include 43 | 44 | 45 | #define BSF_ERROR_NONE 0 46 | #define BSF_ERROR_PCAP_OPEN 1 47 | #define BSF_ERROR_MALLOC 2 48 | #define BSF_ERROR_FILTER 3 49 | 50 | #define BSF_FLAG_NO_OPTIMIZE 1 51 | #define BSF_FLAG_NO_TRANSLATE 2 52 | #define BSF_FLAG_VALIDATE 4 53 | 54 | #define BSF_RESULT_PASS 0 55 | #define BSF_RESULT_FAIL 1 56 | #define BSF_RESULT_ERROR 2 57 | 58 | struct bsf { 59 | unsigned int flags; 60 | struct bpf_program *bpf_prog; 61 | }; 62 | 63 | typedef struct bsf bsf_t; 64 | 65 | //Used to create a new BSF. Must be freed with bsf_destroy 66 | bsf_t *bsf_create(); 67 | 68 | //Used to comple a BSF with the given filter and flags 69 | int bsf_compile(bsf_t *bsf_desc, char *filter, int flags); 70 | 71 | //Free a previously created bsf 72 | void bsf_destroy(bsf_t *bsf_desc); 73 | 74 | //Actually filter a connection 75 | int bsf_filter(bsf_t *bsf_desc, in_addr_t clt_ip, in_port_t clt_port, in_addr_t svr_ip, in_port_t svr_port); 76 | 77 | 78 | 79 | #endif //_BSF_BSF_H 80 | 81 | -------------------------------------------------------------------------------- /vortex/vortex.spec: -------------------------------------------------------------------------------- 1 | # Determine Redhat distribution version for naming and dependency checks 2 | %define dist %{expand:%%(/usr/lib/rpm/redhat/dist.sh --dist)} 3 | 4 | Name: vortex 5 | Version: 2.9.0 6 | Release: 60%dist 7 | Summary: real-time passive network capture and TCP stream reassembly 8 | Vendor: Lockheed Martin 9 | License: GPLv2 10 | Group: Applications/Internet 11 | Url: http://vortex-ids.sourceforge.net/ 12 | Source0: vortex.c 13 | Source1: vortex.init 14 | Source2: vortex.conf 15 | Source3: vortex_conf_parser.init 16 | Source4: vortex.README 17 | Source5: xpipes.c 18 | Source6: vortex.LICENSE 19 | BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) 20 | BuildRequires: libnids libpcap libbsf 21 | Requires: libnids libbsf 22 | 23 | 24 | %description 25 | Vortex is a program that performs real-time passive network capture and TCP stream reassembly. It is designed to be used by near real time, possibly parallel, deep protocol logging and analysis applications. Vortex uses libnids internally to do the brunt of the tcp reassembly. Vortex can also be used for offline analysis. In this case, it is similar to tcpflow or the "Follow TCP Stream" functionality of Wireshark. 26 | 27 | %prep 28 | cp -fp %{SOURCE0} ./ 29 | cp -fp %{SOURCE1} ./ 30 | cp -fp %{SOURCE2} ./ 31 | cp -fp %{SOURCE3} ./ 32 | cp -fp %{SOURCE4} ./ 33 | cp -fp %{SOURCE5} ./ 34 | cp -fp %{SOURCE6} ./ 35 | 36 | %build 37 | gcc %optflags vortex.c -o vortex -lnids -lpthread -lbsf -lpcap -DWITH_BSF 38 | gcc %optflags xpipes.c -o xpipes -lpthread 39 | 40 | %install 41 | rm -rf $RPM_BUILD_ROOT 42 | 43 | # Place binary into correct directory 44 | mkdir -p $RPM_BUILD_ROOT/%{_bindir} 45 | install -m 644 -p $RPM_BUILD_DIR/vortex \ 46 | $RPM_BUILD_ROOT%{_bindir}/vortex 47 | install -m 644 -p $RPM_BUILD_DIR/xpipes \ 48 | $RPM_BUILD_ROOT%{_bindir}/xpipes 49 | 50 | 51 | # Place conf files into position 52 | mkdir -p $RPM_BUILD_ROOT/%{_sysconfdir}/vortex/conf 53 | install -m 644 -p $RPM_SOURCE_DIR/vortex.conf \ 54 | $RPM_BUILD_ROOT%{_sysconfdir}/vortex/conf/vortex.conf 55 | 56 | # install SYSV init stuff 57 | mkdir -p $RPM_BUILD_ROOT/etc/rc.d/init.d 58 | install -m755 $RPM_SOURCE_DIR/vortex.init \ 59 | $RPM_BUILD_ROOT/etc/rc.d/init.d/vortex 60 | 61 | install -m755 $RPM_SOURCE_DIR/vortex_conf_parser.init \ 62 | $RPM_BUILD_ROOT/etc/rc.d/init.d/vortex_conf_parser 63 | 64 | mkdir -p $RPM_BUILD_ROOT/usr/share/vortex 65 | install -m755 $RPM_SOURCE_DIR/vortex.README \ 66 | $RPM_BUILD_ROOT/usr/share/vortex/README 67 | 68 | install -m755 $RPM_SOURCE_DIR/vortex.LICENSE \ 69 | $RPM_BUILD_ROOT/usr/share/vortex/LICENSE 70 | 71 | %clean 72 | %{__rm} -rf %{buildroot} 73 | 74 | %pre 75 | 76 | 77 | %post 78 | # Register the service 79 | # /sbin/chkconfig --add vortex 80 | 81 | 82 | %preun 83 | # only pre-erase 84 | 85 | 86 | %postun 87 | 88 | 89 | %files 90 | %defattr(0755,root,root) 91 | 92 | %config(noreplace) %{_sysconfdir}/vortex/conf/vortex.conf 93 | %{_sysconfdir}/rc.d/init.d/vortex 94 | %{_sysconfdir}/rc.d/init.d/vortex_conf_parser 95 | %{_bindir}/vortex* 96 | %{_bindir}/xpipes 97 | /usr/share/vortex/README 98 | /usr/share/vortex/LICENSE 99 | 100 | %changelog 101 | 102 | * Thu Feb 02 2017 Zach Rasmor 2.9.0-60 103 | - Add -lpcap flag to compilation 104 | * Fri Sep 16 2011 Charles Smutz 2.9.0-59 105 | - Documentation updates: derivative work clarification and community contributions links 106 | * Tue Jan 04 2011 Charles Smutz 2.9.0-58 107 | - Minor fixes--inlcude path for limits.h 108 | * Wed Dec 15 2010 Charles Smutz 2.9.0-57 109 | - Rebuild for public distribution 110 | * Tue Dec 14 2010 Charles Smutz 2.9.0-56 111 | - Significant performance improvements (lower CPU use by ~300%) by making calls to realloc less frequently (-x to tune) 112 | - Set timestamps of output files to match timestamps from pcap (-d to disable) 113 | - Remove arbitrary limit (2 GB) on stream collection sizes, now depends on system 114 | - Wrap 32bit pcap counters with 64bit counters 115 | - Fix cpu locking/affinity and simplify interface 116 | - Addition of error counters for file/IO and memory errors 117 | * Thu Jul 23 2010 Charles Smutz 2.8.1-55 118 | - Added more debug statement for tcp callback and idle queue 119 | * Thu Jul 22 2010 Charles Smutz 2.8.1-54 120 | - Added debug statements for connection open, close, and write 121 | * Wed May 19 2010 William Hoyt 2.8.1-52 122 | - set init script to send messages to /dev/null 123 | * Wed Apr 8 2010 Charles Smutz 2.8.1-51 124 | - Rebuild for public distribution 125 | * Mon Apr 6 2010 Charles Smutz 2.8.1-50 126 | - Rebuild against libnids 1.24 127 | * Mon Feb 22 2010 Charles Smutz 2.8.1-49 128 | - Rebuild for public distribution 129 | * Wed Feb 10 2010 Charles Smutz 2.8.1-48 130 | - Minor fixes to xpipes, added documentation 131 | * Tue Feb 09 2010 Charles Smutz 2.8.1-47 132 | - Addition of xpipes 133 | * Mon Dec 21 2009 Charles Smutz 2.8.0-46 134 | - More documention, cleanup for public release 135 | * Tue Dec 02 2009 Charles Smutz 2.8.0-45 136 | - Changes to defaults 137 | - Ability to dump empty streams (-v) 138 | - Added byte counts to stats 139 | - Report errors and stats on exit and hints if error/stats is not otherwise enabled 140 | * Thu Jul 30 2009 Charles Smutz 2.7.1-44 141 | - Minor Fixes to specfile 142 | * Wed Jul 29 2009 Charles Smutz 2.7.1-43 143 | - Fix to ring buffer implementation (wasn't being used) and output filename formatting 144 | * Fri Jul 3 2009 Samuel Wenck 2.7-42 145 | - Release named with RH distro ID 146 | * Fri Jul 3 2009 Charles Smutz 2.7-41 147 | - Addition of idle timeout 148 | * Fri May 22 2009 Charles Smutz 2.6-40 149 | - Minor fixes to documentation 150 | * Thu May 21 2009 Charles Smutz 2.6-39 151 | - Fixed exiting timestamp error. 152 | * Wed May 20 2009 Charles Smutz 2.6-38 153 | - Refinements to extended output, BSF now optional and off by default 154 | * Tue May 19 2009 Charles Smutz 2.6-37 155 | - Extended extended output, minor fixes 156 | * Fri Apr 10 2009 Charles Smutz 2.5-36 157 | - Addition of extended output 158 | * Fri Apr 10 2009 Charles Smutz 2.4-35 159 | - Make CPU affinity locking off by default 160 | * Fri Apr 10 2009 Charles Smutz 2.4-34 161 | - Addition of CPU affinity locking 162 | * Fri Apr 10 2009 Charles Smutz 2.3-33 163 | - Addition of per thread priorities 164 | * Wed Apr 8 2009 Charles Smutz 2.2-32 165 | - Implementation of output queue. Various thread fixes. 166 | * Fri Mar 23 2009 Charles Smutz 2.1-31 167 | - Package for release as open source 168 | * Wed Jan 14 2009 Charles Smutz 2.1-30 169 | - Add explicit pthread detaches (shouldn't do anything) 170 | * Thu Jan 08 2009 Charles Smutz 2.1-29 171 | - Fix unterminated filter string issue 172 | * Thu Jan 08 2009 Charles Smutz 2.1-26 173 | - Integration of libBSF 174 | * Mon Dec 01 2008 Charles Smutz 2.0-25 175 | - Fixes to improved init script/conf 176 | * Wed Nov 26 2008 Charles Smutz 2.0-24 177 | - Fixes to improved init script/conf 178 | * Mon Nov 24 2008 Charles Smutz 2.0-23 179 | - Fixes to improved init script/conf 180 | * Mon Nov 24 2008 Charles Smutz 2.0-19 181 | - Improved init script/conf 182 | * Wed Nov 19 2008 Charles Smutz 2.0-18 183 | - Better accounting of connections not tracked due to polling 184 | * Mon Nov 17 2008 Charles Smutz 2.0-15 185 | - Improved logging 186 | * Fri Oct 13 2008 Charles Smutz 2.0-10 187 | - Fix to early dumping of data 188 | * Fri Oct 10 2008 Charles Smutz 2.0-9 189 | - Fixes and update of defaults to vortex 190 | * Wed Sep 17 2008 Samuel Wenck = 2.0-6 191 | - Changed debug_level type to int 192 | - Added option flag for libnids TCP workaround processing 193 | - Added ability to disable libnids TCP checksum processing 194 | -------------------------------------------------------------------------------- /libbsf/libbsf.c: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2008-2009 Lockheed Martin Corporation 4 | * 5 | * The libBSF program is open source software: you can copy it, redistribute it and/or modify 6 | * it under the terms of the GNU General Public License version 2.0 as published by 7 | * the Free Software Foundation. The libBSF Program and any derivatives of the libBSF program 8 | * must be licensed under GPL version 2.0 and may not be licensed under GPL version 3.0. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY OF ANY KIND, including without limitation the implied warranties of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details at http://www.gnu.org/licenses/gpl-2.0.html. 14 | * 15 | * The term "libBSF" should be taken to also include any portions or derived works of libBSF. 16 | * 17 | * You are highly encouraged to send your changes to the Vortex program to 18 | * opensource.tools.security@lmco.com for possible incorporation into the main distribution. 19 | * By sending these changes to Lockheed Martin, you are granting to Lockheed Martin 20 | * Corporation the unlimited, perpetual, non-exclusive right to reuse, modify, 21 | * and/or relicense the code on a royalty-free basis. 22 | * 23 | * The libraries to which libBSF links are distributed under the terms of their own licenses. 24 | * Please see those libraries for their applicable licenses. 25 | * 26 | */ 27 | /* 28 | * 29 | * libbsf a stream filtering mechanism based on BPF and tcpdump filter syntax 30 | * 31 | * compile: 32 | * gcc -Wall -fPIC -shared libbsf.c -o libbsf.so 33 | * install: 34 | * copy bsf.h to your include dir (/usr/include) 35 | * copy libbsf.so to your lib dir (/usr/lib) or (/usr/lib64) 36 | */ 37 | 38 | 39 | 40 | #include 41 | #include "bsf.h" 42 | 43 | 44 | #define BSF_SNAP_LEN (BSF_LINK_LEN + BSF_IP_LEN + BSF_TCP_LEN) 45 | #define BSF_LINK_LEN 0 46 | #define BSF_IP_LEN 20 47 | #define BSF_TCP_LEN 4 48 | 49 | #define BSF_LINK_TYPE DLT_RAW 50 | #define BSF_WHITE_SPACE "()\t\n " 51 | 52 | 53 | 54 | 55 | //returns 0 if char is not contained in white_space string, 1 if it is 56 | int is_white_space(char *white_space, char c) 57 | { 58 | if ( strchr(white_space, c) == NULL) 59 | { 60 | return 0; 61 | } else 62 | { 63 | return 1; 64 | } 65 | } 66 | 67 | //replace all instances of to_replace in filter which are bounded by one of the characters in white_space with replacement 68 | //returns 0 on normal, else on error 69 | int word_replace(char *filter, char *white_space, char *string_to_replace, char *string_replacement) 70 | { 71 | //some basic validation 72 | if (strlen(string_replacement) != strlen(string_to_replace)) 73 | { 74 | return 1; 75 | } 76 | if (strlen(string_to_replace) < 1) 77 | { 78 | return 1; 79 | } 80 | 81 | 82 | char *filter_current_pos; 83 | char *filter_current_match; 84 | 85 | int i; 86 | 87 | filter_current_pos = filter; 88 | 89 | while((filter_current_match = strstr(filter_current_pos,string_to_replace))) 90 | { 91 | 92 | //check to see if previous char is white space 93 | //special case for match at first char 94 | if ((is_white_space(white_space, filter[strlen(filter)-strlen(filter_current_match)-1])) || (filter_current_match == filter)) 95 | { 96 | //now try for white space after hit 97 | //special case for hit that contains last car 98 | if( (is_white_space(white_space, filter_current_match[strlen(string_to_replace)])) || (strlen(filter_current_match) == strlen(string_to_replace)) ) 99 | { 100 | //the word is surrounded by white space, replace it 101 | for (i = 0; i < strlen(string_to_replace); i++) 102 | { 103 | filter[strlen(filter)-strlen(filter_current_match)+i] = string_replacement[i]; 104 | } 105 | } 106 | } 107 | filter_current_pos = &filter_current_match[1]; 108 | } 109 | return 0; 110 | } 111 | 112 | 113 | void translate_filter(char *filter) 114 | { 115 | word_replace(filter, BSF_WHITE_SPACE, "clt", "src"); 116 | word_replace(filter, BSF_WHITE_SPACE, "svr", "dst"); 117 | } 118 | 119 | 120 | //creates a new BSF. 121 | bsf_t *bsf_create() 122 | { 123 | return calloc(1,sizeof(bsf_t)); 124 | } 125 | 126 | 127 | //Compile the specified filter into the previously created (non-null) bsf using specified flags 128 | int bsf_compile(bsf_t *bsf_desc, char *filter, int flags) 129 | { 130 | pcap_t *pcap_desc; 131 | struct bpf_program *bpf_prog; 132 | int optimize; 133 | int translate; 134 | char *bpf_filter; 135 | //bsf_t *bsf_desc; 136 | 137 | //modify this if we do translation 138 | bpf_filter = filter; 139 | 140 | 141 | //process the pertinant flags 142 | //optimize 143 | if (flags & BSF_FLAG_NO_OPTIMIZE) 144 | { 145 | optimize = 0; 146 | } else 147 | { 148 | optimize = 1; 149 | } 150 | 151 | //translate 152 | if (flags & BSF_FLAG_NO_TRANSLATE) 153 | { 154 | translate = 0; 155 | } else 156 | { 157 | translate = 1; 158 | } 159 | 160 | 161 | 162 | 163 | //try to create pcap descriptor 164 | pcap_desc = pcap_open_dead(BSF_LINK_TYPE, BSF_SNAP_LEN); 165 | if (pcap_desc == NULL) 166 | { 167 | // free(bsf_desc); 168 | return BSF_ERROR_PCAP_OPEN; 169 | } 170 | 171 | //Malloc a bpf_program 172 | bpf_prog = calloc(1,sizeof(struct bpf_program)); 173 | if (bpf_prog == NULL) 174 | { 175 | // free(bsf_desc); 176 | pcap_close(pcap_desc); 177 | return BSF_ERROR_MALLOC; 178 | } 179 | 180 | //create a copy of the input filter so we can translate it. 181 | bpf_filter = calloc(1,strlen(filter)+1); 182 | if (bpf_filter == NULL) 183 | { 184 | // free(bsf_desc); 185 | pcap_close(pcap_desc); 186 | free(bpf_prog); 187 | return BSF_ERROR_MALLOC; 188 | } 189 | 190 | strncpy( bpf_filter, filter, strlen(filter)); 191 | 192 | //run filter translation routines 193 | if (translate) 194 | { 195 | translate_filter(bpf_filter); 196 | } 197 | 198 | if (pcap_compile(pcap_desc, bpf_prog, bpf_filter, optimize, 0) == -1) 199 | { 200 | // free(bsf_desc); 201 | pcap_close(pcap_desc); 202 | free(bpf_prog); 203 | free(bpf_filter); 204 | return BSF_ERROR_FILTER; 205 | 206 | //we don't do anything with error message, but we could--use pcap_geterr(pcap_desc) to retrieve it. 207 | 208 | } 209 | 210 | bsf_desc->bpf_prog = bpf_prog; 211 | bsf_desc->flags = flags; 212 | //*bsf_desc_ref = bsf_desc; 213 | 214 | free(bpf_filter); 215 | pcap_close(pcap_desc); 216 | return BSF_ERROR_NONE; 217 | } 218 | 219 | void bsf_destroy(bsf_t *bsf_desc) 220 | { 221 | pcap_freecode(bsf_desc->bpf_prog); 222 | free(bsf_desc->bpf_prog); 223 | free(bsf_desc); 224 | return; 225 | } 226 | 227 | 228 | 229 | //Used to populate create a minimalistic ethernet/ip/tcp packet 230 | void populate_packet(unsigned char *packet, in_addr_t clt_ip, in_port_t clt_port, in_addr_t svr_ip, in_port_t svr_port) 231 | { 232 | //this is just easier than setting zeros everywehere necessary 233 | //bzero(packet, BSF_SNAP_LEN); 234 | unsigned int temp; 235 | 236 | //Hardcoded constants 237 | 238 | 239 | //set IP version and hdr len 240 | packet[BSF_LINK_LEN] = 0x45; 241 | //set IP total length 242 | packet[BSF_LINK_LEN+2] = 0x00; 243 | packet[BSF_LINK_LEN+3] = 0x1C; 244 | //set IP prototol (TCP) 245 | packet[BSF_LINK_LEN+9] = 0x06; 246 | 247 | //set UDP/TCP hdr len 248 | //packet[BSF_LINK_LEN+BSF_IP_LEN+8] = 0x20; 249 | 250 | //Set per stream values 251 | 252 | //src ip 253 | temp = clt_ip & 0xFF; 254 | packet[BSF_LINK_LEN + 12] = temp; 255 | temp = clt_ip & 0xFF00; 256 | temp >>= 8; 257 | packet[BSF_LINK_LEN + 13] = temp; 258 | temp = clt_ip & 0xFF0000; 259 | temp >>= 16; 260 | packet[BSF_LINK_LEN + 14] = temp; 261 | temp = clt_ip & 0xFF000000; 262 | temp >>= 24; 263 | packet[BSF_LINK_LEN + 15] = temp; 264 | 265 | 266 | //dst ip 267 | 268 | temp = svr_ip & 0xFF; 269 | packet[BSF_LINK_LEN + 16] = temp; 270 | temp = svr_ip & 0xFF00; 271 | temp >>= 8; 272 | packet[BSF_LINK_LEN + 17] = temp; 273 | temp = svr_ip & 0xFF0000; 274 | temp >>= 16; 275 | packet[BSF_LINK_LEN + 18] = temp; 276 | temp = svr_ip & 0xFF000000; 277 | temp >>= 24; 278 | packet[BSF_LINK_LEN + 19] = temp; 279 | 280 | 281 | 282 | //src port 283 | temp = clt_port & 0xFF; 284 | packet[BSF_LINK_LEN+BSF_IP_LEN+1] = temp; 285 | temp = clt_port & 0xFF00; 286 | temp >>= 8; 287 | packet[BSF_LINK_LEN+BSF_IP_LEN] = temp; 288 | 289 | //dst port 290 | temp = svr_port & 0xFF; 291 | packet[BSF_LINK_LEN+BSF_IP_LEN+3] = temp; 292 | temp = svr_port & 0xFF00; 293 | temp >>= 8; 294 | packet[BSF_LINK_LEN+BSF_IP_LEN+2] = temp; 295 | 296 | } 297 | 298 | 299 | int bsf_filter(bsf_t *bsf_desc, in_addr_t clt_ip, in_port_t clt_port, in_addr_t svr_ip, in_port_t svr_port) 300 | { 301 | unsigned char dummy_packet[BSF_SNAP_LEN]; 302 | 303 | 304 | populate_packet(dummy_packet, clt_ip, clt_port, svr_ip, svr_port); 305 | //validate the bpf program 306 | if (bsf_desc->flags && BSF_FLAG_VALIDATE == BSF_FLAG_VALIDATE) 307 | { 308 | //validate the BPF: 309 | if (bpf_validate(bsf_desc->bpf_prog->bf_insns, bsf_desc->bpf_prog->bf_len) == 0) 310 | { 311 | //validation failed 312 | return BSF_RESULT_ERROR; 313 | } 314 | } 315 | 316 | if (bpf_filter(bsf_desc->bpf_prog->bf_insns, dummy_packet, BSF_SNAP_LEN, BSF_SNAP_LEN) == BSF_SNAP_LEN) 317 | { 318 | return BSF_RESULT_PASS; 319 | } else 320 | { 321 | return BSF_RESULT_FAIL; 322 | } 323 | 324 | 325 | } 326 | -------------------------------------------------------------------------------- /xpipes/xpipes.c: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2009-2011 Lockheed Martin Corporation 4 | * 5 | * The xpipes program is open source software: you can copy it, redistribute it and/or modify 6 | * it under the terms of the GNU General Public License version 2.0 as published by 7 | * the Free Software Foundation. The xpipes Program and any derivatives of the xpipes program 8 | * must be licensed under GPL version 2.0 and may not be licensed under GPL version 3.0. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY OF ANY KIND, including without limitation the implied warranties of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details at http://www.gnu.org/licenses/gpl-2.0.html. 14 | * 15 | * The term "xpipes" should be taken to also include any portions or derived works of xpipes. 16 | * 17 | * You are highly encouraged to send your changes to the xpipes program to 18 | * opensource.tools.security@lmco.com for possible incorporation into the main distribution. 19 | * By sending these changes to Lockheed Martin, you are granting to Lockheed Martin 20 | * Corporation the unlimited, perpetual, non-exclusive right to reuse, modify, 21 | * and/or relicense the code on a royalty-free basis. 22 | * 23 | * The libraries to which xpipes links are distributed under the terms of their own licenses. 24 | * Please see those libraries for their applicable licenses. 25 | * 26 | */ 27 | 28 | /* 29 | * xpipes 30 | * a simple utilitity for multiplexing pipes 31 | * this is intended to be used in conjunction with vortex to create multithreaded analyzers 32 | * gcc xpipes.c -lpthread -Wall -o xpipes -O2 33 | */ 34 | 35 | 36 | 37 | #include 38 | #include 39 | #include 40 | #include 41 | #include 42 | #include 43 | 44 | 45 | #define MAX_NUM_PROCS 1000 46 | #define MAX_NUM_PROCS_DIGITS 16 47 | 48 | 49 | /* 50 | TODO: 51 | set cpu affinity for children (per child or as a group). If you do that, probably have to do setuid also (since root is required for setting affinity). 52 | */ 53 | 54 | 55 | struct list_element_t 56 | { 57 | struct list_element_t *next; 58 | //taken from linux/limits.h 59 | char buffer[PATH_MAX+1]; 60 | }; 61 | 62 | struct list_metadata_t 63 | { 64 | struct list_element_t *head; 65 | int count; 66 | }; 67 | 68 | char *process_name; 69 | 70 | int num_list_elements = 0; 71 | int num_procs = 10; 72 | int num_procs_initialized = 0; 73 | //these are tunables, current defaults 74 | unsigned int consumer_bottleneck_poll_interval = 10; //time in us 75 | unsigned int list_elements_per_proc = 10; 76 | 77 | 78 | int input_done = 0; 79 | int ensure_exit = 0; 80 | 81 | char *command = NULL; 82 | const char popen_type[] = "w"; 83 | 84 | struct list_metadata_t free_list; 85 | struct list_metadata_t ready_list; 86 | 87 | //for getopt 88 | extern int optind; 89 | extern int opterr; 90 | extern int optopt; 91 | extern char *optarg; 92 | 93 | pthread_mutex_t list_mtx; 94 | 95 | pthread_mutex_t data_ready_mtx; 96 | pthread_cond_t data_ready_cv; 97 | 98 | 99 | void grablock(void) 100 | { 101 | if (pthread_mutex_lock(&list_mtx) != 0) 102 | { 103 | fprintf(stderr, "error getting mutex lock\n"); 104 | //exit(2); 105 | } 106 | } 107 | 108 | void releaselock(void) 109 | { 110 | if (pthread_mutex_unlock(&list_mtx) != 0) 111 | { 112 | fprintf(stderr, "error releasing mutex lock\n"); 113 | } 114 | } 115 | 116 | void wait_data_ready(void) 117 | { 118 | pthread_mutex_lock(&data_ready_mtx); 119 | pthread_cond_wait(&data_ready_cv, &data_ready_mtx); 120 | pthread_mutex_unlock(&data_ready_mtx); 121 | 122 | } 123 | 124 | void signal_data_ready(void) 125 | { 126 | pthread_mutex_lock(&data_ready_mtx); 127 | pthread_cond_signal(&data_ready_cv); 128 | pthread_mutex_unlock(&data_ready_mtx); 129 | } 130 | 131 | void broadcast_data_ready(void) 132 | { 133 | pthread_mutex_lock(&data_ready_mtx); 134 | pthread_cond_broadcast(&data_ready_cv); 135 | pthread_mutex_unlock(&data_ready_mtx); 136 | } 137 | 138 | 139 | void print_usage() 140 | { 141 | fprintf(stderr,"Usage: %s -f file | -c command [-h] [-P num procs] [ -e ] [ -Q count ] [ -R usec ]\n\n", process_name); 142 | fprintf(stderr, 143 | " -h Print this help message and exit\n" 144 | " -f file file containing command to execute\n" 145 | " -c command Don't forget to quote if necessary. Command is interpreted by shell.\n" 146 | " -e Ensure exit (don't hang) even if children hang\n" 147 | " -P procs Set number of processes to spawn (default: 10, max: %i)\n" 148 | " -Q count Number of lines to buffer per proc (default: %i)\n" 149 | " -R usec Wait period if consumers become bottleneck (increase to limit CPU lost to polling) (default: %i)\n" 150 | "\n" 151 | "for each child process that xpipes creates, it sets the environment variable XPIPES_INDEX to a unique value.\n" 152 | "this value can be used inside of command and will be interpretted by the shell.\n" 153 | "\n", MAX_NUM_PROCS, list_elements_per_proc, consumer_bottleneck_poll_interval); 154 | } 155 | 156 | 157 | //get length of list 158 | int list_length(struct list_metadata_t *list_p) 159 | { 160 | return list_p->count; 161 | } 162 | 163 | //remove something from head of list 164 | struct list_element_t *list_remove(struct list_metadata_t *list_p) 165 | { 166 | struct list_element_t *temp_element = NULL; 167 | if (list_p->count < 1) 168 | { 169 | return NULL; 170 | } else 171 | { 172 | temp_element = list_p->head; 173 | list_p->head = temp_element->next; 174 | list_p->count--; 175 | temp_element->next = NULL; 176 | return temp_element; 177 | } 178 | 179 | } 180 | 181 | 182 | //add something to tail of list 183 | void list_add(struct list_metadata_t *list_p, struct list_element_t *element_p) 184 | { 185 | struct list_element_t *temp_element = list_p->head; 186 | if (list_p->count > 0) 187 | { 188 | while ( temp_element->next != NULL ) 189 | { 190 | temp_element = temp_element->next; 191 | } 192 | temp_element->next = element_p; 193 | element_p->next = NULL; 194 | list_p->count++; 195 | 196 | } else 197 | { 198 | list_p->head = element_p; 199 | list_p->count = 1; 200 | } 201 | } 202 | 203 | 204 | 205 | //function for writer threads. 206 | void *writer_thread(void *arg) 207 | { 208 | 209 | FILE *output_fh; 210 | int proc_id; 211 | struct list_element_t *element; 212 | int tmp; 213 | 214 | char proc_id_str[MAX_NUM_PROCS_DIGITS+1]; 215 | proc_id_str[0] = '\0'; 216 | 217 | grablock(); 218 | proc_id = num_procs_initialized; 219 | num_procs_initialized++; 220 | 221 | 222 | //Set ENV 223 | if (snprintf(proc_id_str, MAX_NUM_PROCS_DIGITS, "%i", proc_id) > 0) 224 | { 225 | setenv( "XPIPES_INDEX", proc_id_str, 1); 226 | } 227 | 228 | 229 | //If you want to do cpu affinity locking, priorities, etc, do it here! 230 | 231 | output_fh = popen(command, popen_type); 232 | 233 | if (output_fh == NULL) 234 | { 235 | fprintf(stderr,"popen failed!\n"); 236 | exit(3); 237 | } 238 | 239 | //make out line buffered 240 | setvbuf(output_fh, NULL, _IOLBF, 0); 241 | 242 | //fprintf(stderr,"Initialzied proc %i \n",num_procs_initialized); 243 | 244 | releaselock(); 245 | 246 | 247 | while (input_done == 0) 248 | { 249 | 250 | 251 | wait_data_ready(); 252 | 253 | grablock(); 254 | if (list_length(&ready_list) > 0) 255 | { 256 | element = list_remove(&ready_list); 257 | releaselock(); 258 | 259 | if (element != NULL) 260 | { 261 | tmp = fprintf(output_fh, "%s", element->buffer); 262 | 263 | if (tmp < strlen(element->buffer)) 264 | { 265 | fprintf(stderr,"Error printing whole string to pipe\n"); 266 | } 267 | if (tmp < 0) 268 | { 269 | fprintf(stderr,"Pipe %i broken\n", proc_id); 270 | pclose(output_fh); 271 | pthread_exit(NULL); 272 | } 273 | 274 | element->buffer[0] = '\0'; 275 | grablock(); 276 | list_add(&free_list, element); 277 | releaselock(); 278 | 279 | } 280 | 281 | } else 282 | { 283 | releaselock(); 284 | } 285 | 286 | 287 | } 288 | 289 | //flush any data in pipe 290 | 291 | 292 | if (ensure_exit == 1) 293 | { 294 | //give consumers fair chance to consume data, but don't hang infinately 295 | usleep(1000); 296 | } else 297 | { 298 | //this can make us hang 299 | fflush(output_fh); 300 | } 301 | 302 | //do pclose 303 | pclose(output_fh); 304 | pthread_exit(NULL); 305 | } 306 | 307 | 308 | //opens file, mallocs buffer, and reads file into buffer--returns number of bytes read. 309 | //if returns 0, buffer is not alloc'd, otherwise it is and must be freed later 310 | //buffer is 1 byte longer than file so that buffer can be null terminated. Null char is not included in byte count returned 311 | 312 | int read_file_into_buffer(char *filter_file, char **filter_buffer) 313 | { 314 | 315 | FILE *filter_file_fp; 316 | int filter_file_len; 317 | 318 | //Open file 319 | filter_file_fp = fopen(filter_file, "r"); 320 | if (!filter_file_fp) 321 | { 322 | return 0; 323 | } 324 | 325 | 326 | //Get file length 327 | fseek(filter_file_fp, 0, SEEK_END); 328 | filter_file_len=ftell(filter_file_fp); 329 | fseek(filter_file_fp, 0, SEEK_SET); 330 | 331 | //Allocate memory 332 | *filter_buffer=(char *)calloc(1,(filter_file_len+1)); 333 | if (!*filter_buffer) 334 | { 335 | return 0; 336 | } 337 | 338 | 339 | //Read file contents into buffer 340 | if (fread(*filter_buffer, filter_file_len, 1, filter_file_fp) != 1) 341 | { 342 | free(*filter_buffer); 343 | return 0; 344 | } 345 | 346 | fclose(filter_file_fp); 347 | 348 | //Null terminate 349 | (*filter_buffer)[filter_file_len] = '\0'; 350 | 351 | 352 | return filter_file_len; 353 | } 354 | 355 | 356 | 357 | int main (int argc, char **argv) 358 | { 359 | 360 | int i; 361 | int tmp; 362 | struct list_element_t *temp_element_p; 363 | int opt; 364 | process_name = argv[0]; 365 | 366 | pthread_mutex_init(&list_mtx, NULL); 367 | pthread_mutex_init(&data_ready_mtx, NULL); 368 | pthread_cond_init(&data_ready_cv, NULL); 369 | 370 | pthread_t thread_refs[MAX_NUM_PROCS]; 371 | 372 | 373 | 374 | 375 | //get command line options 376 | while ((opt = getopt(argc, argv, "hP:c:f:eQ:R:")) != -1) 377 | { 378 | switch (opt) 379 | { 380 | case 'c': 381 | command = optarg; 382 | break; 383 | case 'e': 384 | ensure_exit = 1; 385 | break; 386 | case 'f': 387 | if ( read_file_into_buffer(optarg,&command) == 0) 388 | { 389 | exit(2); 390 | } 391 | break; 392 | case 'P': 393 | num_procs = atoi(optarg); 394 | if (num_procs > MAX_NUM_PROCS) num_procs = MAX_NUM_PROCS; 395 | break; 396 | case 'R': 397 | consumer_bottleneck_poll_interval = atoi(optarg); 398 | break; 399 | 400 | case 'Q': 401 | list_elements_per_proc = atoi(optarg); 402 | break; 403 | 404 | default: 405 | print_usage(); 406 | fprintf(stderr,"Invalid option %c\n", opt); 407 | exit(2); 408 | break; 409 | 410 | } 411 | } 412 | 413 | 414 | //too many args or too few 415 | if (argc > optind || argc < 2) 416 | { 417 | print_usage(); 418 | fprintf(stderr,"Invalid command line options\n"); 419 | } 420 | 421 | //make sure there is a command to run 422 | if ( command == NULL ) 423 | { 424 | fprintf(stderr, "You must specify a command to run\n"); 425 | print_usage(); 426 | exit(2); 427 | } 428 | 429 | if (consumer_bottleneck_poll_interval < 0) 430 | { 431 | print_usage(); 432 | fprintf(stderr,"Invalid Wait Period\n"); 433 | exit(2); 434 | } 435 | 436 | if (list_elements_per_proc < 1) 437 | { 438 | print_usage(); 439 | fprintf(stderr,"Invalid line buffer size\n"); 440 | exit(2); 441 | } 442 | 443 | if (num_procs < 1) 444 | { 445 | print_usage(); 446 | fprintf(stderr,"Invalid number of processes\n"); 447 | exit(2); 448 | } 449 | 450 | 451 | num_list_elements = num_procs * list_elements_per_proc; 452 | 453 | //intialize the list elements and lists 454 | for (i = 0; i < num_list_elements; i++) 455 | { 456 | //malloc each element and set link 457 | temp_element_p = (struct list_element_t *) malloc(sizeof(struct list_element_t)); 458 | if (temp_element_p == NULL) 459 | { 460 | fprintf(stderr, "Malloc failed.\n"); 461 | exit(2); 462 | } 463 | list_add(&free_list, temp_element_p); 464 | } 465 | 466 | //now start up the threads. 467 | 468 | 469 | for (i=0; i < num_procs; i++) 470 | { 471 | tmp = pthread_create(&thread_refs[i], NULL, writer_thread, NULL); 472 | if (tmp != 0 ) 473 | { 474 | fprintf(stderr, "pthread create failed for proc %i with return value of: %i.\n", i, tmp); 475 | exit(2); 476 | } 477 | } 478 | 479 | 480 | 481 | 482 | 483 | grablock(); 484 | temp_element_p = list_remove(&free_list); 485 | releaselock(); 486 | 487 | //loop through each line of input 488 | while(fgets(temp_element_p->buffer, PATH_MAX, stdin) != NULL) 489 | { 490 | grablock(); 491 | list_add(&ready_list, temp_element_p); 492 | temp_element_p = list_remove(&free_list); 493 | releaselock(); 494 | 495 | signal_data_ready(); 496 | 497 | //if we are backed up, signal again 498 | if (list_length(&free_list) < (num_list_elements/2)) 499 | { 500 | broadcast_data_ready(); 501 | } 502 | 503 | //wait if we don't have any free list elements available, keep signalling all the while 504 | while (temp_element_p == NULL) 505 | { 506 | usleep(consumer_bottleneck_poll_interval); 507 | broadcast_data_ready(); 508 | grablock(); 509 | temp_element_p = list_remove(&free_list); 510 | releaselock(); 511 | } 512 | 513 | } 514 | 515 | //wait for ready_list to empty 516 | while(list_length(&ready_list) > 0) 517 | { 518 | signal_data_ready(); 519 | } 520 | 521 | input_done = 1; 522 | 523 | 524 | //make sure no one is stuck on condition variable 525 | broadcast_data_ready(); 526 | 527 | 528 | //fprintf(stderr, "Done processing input, finishing up\n"); 529 | 530 | //join all the writer threads 531 | for (i = 0; i < num_procs; i++) 532 | { 533 | pthread_join(thread_refs[i], NULL); 534 | } 535 | 536 | exit(0); 537 | 538 | } 539 | 540 | 541 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | {description} 294 | Copyright (C) {year} {fullname} 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | {signature of Ty Coon}, 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | 341 | -------------------------------------------------------------------------------- /vortex/README: -------------------------------------------------------------------------------- 1 | Copyright 2008-2011 Lockheed Martin Corporation 2 | 3 | The Vortex program is open source software: you can copy it, redistribute it and/or modify 4 | it under the terms of the GNU General Public License version 2.0 as published by 5 | the Free Software Foundation. The Vortex Program and any derivatives of the Vortex program 6 | must be licensed under GPL version 2.0 and may not be licensed under GPL version 3.0. 7 | 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY OF ANY KIND, including without limitation the implied warranties of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details at http://www.gnu.org/licenses/gpl-2.0.html. 12 | 13 | The term "Vortex" should be taken to also include any portions or derived works of Vortex. 14 | 15 | You are highly encouraged to send your changes to the Vortex program to 16 | opensource.tools.security@lmco.com for possible incorporation into the main distribution. 17 | By sending these changes to Lockheed Martin, you are granting to Lockheed Martin 18 | Corporation the unlimited, perpetual, non-exclusive right to reuse, modify, 19 | and/or relicense the code on a royalty-free basis. 20 | 21 | The libraries to which Vortex links are distributed under the terms of their own licenses. 22 | Please see those libraries for their applicable licenses. 23 | 24 | Applications created to analyze or process data that is outputted from 25 | Vortex are viewed as separate works and not derivative. 26 | 27 | 28 | Introduction: 29 | 30 | Vortex is an application for intrusion detection and network surveillance. It relies heavily on libnids by Rafal Wojtczuk (see http://libnids.sourceforge.net/). 31 | 32 | Vortex is designed to facilitate agile creation of advanced network intrusion detection or network surveillance systems. It performs the hard work of packet capture, filtering, and reassembly then provides that data to external programs for analysis. Stream data is stored in files and stream meta data is output via STDOUT (meta data is encoded in file name). Vortex provides a platform for highly flexible, multithreaded, near real time deep analysis of network payload data. 33 | 34 | Changes: 35 | 36 | See RPM changelog for changes (end of vortex.spec) 37 | 38 | Motivation: 39 | 40 | Vortex is designed to compliment current NIDS such as Snort (http://www.snort.org) or Bro (http://bro-ids.org) by facilitating analysis not possible in conventional NIDS. Some examples of things that can be done in conjunction with vortex are: 41 | 1. Leverage existing code and/or scripting languages 42 | 2. High latency analysis 43 | 3. Multithreaded analysis 44 | 45 | Since it can be used for offline analysis it can also compliment tools such as wireshark (http://wireshark.org) or tcpflow (http://www.circlemud.org/~jelson/software/tcpflow). Some situations where vortex may be of benefit include: 46 | 1. When wireshark doesn't have a (complete) decoder for the protocol you are interested in analyzing. 47 | 2. When the output of vortex, including extended metadata, is convenient for your analysis. 48 | 3. When the performance and accuracy is of vortex is preferable over other tools. 49 | 50 | If not clear at this point, vortex is designed to facilitate analysis that is not very common. If you trying to do something that can be done in any of the aforementioned tools, you'll probably be better off and happier using them. If, however, you are trying to do something that conventional tools can't do feasibly, then hopefully you find vortex useful. 51 | 52 | It is also hoped that use of vortex will drive research and development of NIDS, such that some day, some other tool will provide a better solution. Until that occurs, it is hoped that vortex provides a useful and common platform for both research and operations. 53 | 54 | Usage: 55 | 56 | Vortex can be used as a user program much like tcptrace et al. Vortex can also be used as daemon. Example init scripts and config file are provided. 57 | 58 | Vortex is used by specifying an input (pcap file or interface), a directory in which to store stream data, amount of data to collect in each direction, filters, and numerous other options. The analyzer program reads the stream meta data (file names), open and analyzes the files, and optionally purges the files after they are analyzed. If large numbers of streams are to be analyzed simultaneously, the connection hash table (-s) may need to be increased. See TCP_LIMIT count in error logs (-E). 59 | 60 | Example: 61 | 62 | Imagine that you want to apply the spamsum algorithm to all web requests. You could do something like the following: 63 | 64 | vortex -i eth0 -t /tmp/vortex_demo/ | xargs -l ssdeep | logger 65 | 66 | which would yield logs such as: 67 | 68 | Mar 23 18:54:22 hostname logger: 3:ixx1PQcFI5LM38iIjMFykMFPDD7IM/R/CZJMT9yn:ixz7Z3EPLIM/7By,"/tmp/vortex_demo/127.0.0.1:42742s127.0.0.1:80" 69 | 70 | Note that this example neglects many details. See section "Analyzer Development" below. 71 | 72 | Compiling: 73 | 74 | You will need libnids, libpcap, and optionally libbsf to build and use vortex. Compile instructions can be found at the begginging of the source files. 75 | 76 | See http://libnids.sourceforge.net for libnids. You'll want at least v1.23. Satisfying this dependency will likely be the hardest part of bulding/installing vortex. 77 | 78 | If you are using Red Hat or CentOS and want to use RPMS to install, you might consider the RPM for libnids offered by EPEL (http://fedoraproject.org/wiki/EPEL). 79 | 80 | 81 | Filtering: 82 | 83 | Vortex supports tcpdump style filter expressions which are used by the BPF filtering mechanism. 84 | Vortex also can use libBSF which is similar to BPF but with stream semantics instead of packet semantics. 85 | Vortex also supports simple sampling or polling. 86 | 87 | BPF filtering 88 | 89 | It is highly recommended that you use a BPF to limit traffic seen by vortex to the extent possible. In the case of IP fragmented streams (which vortex supports) is possible to loose data with a naive BPF. For example, if a TCP packet is fragmented, only the first fragment will contain the tcp header and so only the first packet will match a filter such as "tcp port 80". See the LibNIDS documentation for more details and workarounds. 90 | 91 | libBSF 92 | 93 | LibBSF implements the tcpdump filter syntax and the BPF virtual machine with stream semantics. The tcpdump-like expression is very similar to tcpdump filter language. See man tcpdump(8). Syntax is the same. The allowed primitives are close to the subset of primitives relating to streams. 94 | Allowable primitives are: clt, svr, host, net, mask, port, portrange 95 | "clt" indicates the stream client, "svr" indicates the stream server, otherwise the primitives have the same meaning 96 | Don't use primitives like ether, ip, tcp, udp, len, etc. The results will not be coherent even if no errors are thrown. If you are tempted to use them, you are probably trying to use the wrong tool for your needs. 97 | 98 | Metadata: 99 | 100 | Metadata for each stream is encoded in the filename. 101 | 102 | Short metadata is as follows: 103 | {client_ip}:{client_port}{direction}{server_ip}:{server_port} 104 | ex. 105 | 10.1.1.1:1954s172.16.1.1:80 106 | 107 | The direction, either "s" or "c" represent the stream from client to server and from server to client respectively 108 | 109 | Extended metatdata is as follows: 110 | 111 | {proto}-{connection_serial_number}-{connection_start_time}-{connection_end_time}-{connection_end_reason}-{connection_size}-{client_ip}:{client_port}{direction}{server_ip}:{server_port} 112 | ex. 113 | 114 | tcp-1-1229100756-1229100756-c-390-10.1.1.1:1954s172.16.1.1:80 115 | 116 | The serial number is simply an increasing stream identifier. All times are in unix timestamp form. connection_end_time represents the last time data was sent over the connection, not when the stream was dumped by vortex (fstat the file if you want that). connection_size indicates the size captured for the whole connection which is the sum of both simplex flows (if you want individual stream sizes, use fstat). connection_end_reason is one of the following values indicating the reason the flow was dumped: 117 | 118 | c => Well formed TCP close 119 | r => Reset 120 | t => Time out 121 | e => Stream ended prematurely becuase vortex is exiting 122 | l => Stream size limit exceeded 123 | i => Connection exceeded idle limit 124 | 125 | Performance: 126 | 127 | Vortex has been used to capture and analyze large amounts of data with very low packet loss. Vortex was designed with multithreaded analysis in mind, allowing for very expensive or high-latency analysis to be performed in near real time on commodity hardware. 128 | 129 | Multithreaded Analysis 130 | 131 | One of the major design goals of vortex was to enable multithreaded analysis in support of high latency and/or computationally expensive analysis. Within reason, vortex sufficiently removes the real-time contraints of packet capture and provided reassembled stream data to external analysis applications. 132 | 133 | There are numerous ways to implement multithreading using vortex. 134 | 135 | One approach that works well with standard CLI utilities is the multithreading that can be implemented with xargs (see -L and -P options). 136 | 137 | Similarly, if you have an analyzer that accepts the input file list in STDIN instead of as arguments (as with xargs) then you should consider using the included xpipes program. xpipes was written specifically to facilitate multithread analysis on top of vortex, especially with single threaded analyzers. 138 | 139 | xpipes can be used as follows to create a multithreaded analyzer from a single threaded one: 140 | 141 | single threaded: 142 | 143 | vortex {vortex arguments} | analyzer {analyzer arguments} 144 | 145 | multithread: 146 | 147 | vortex {vortex arguments} | xpipes -c 'analyzer {analzyer arguments}' 148 | 149 | or 150 | 151 | echo 'analyzer {analzyer arguments}' > analyzer.cmd 152 | vortex {vortex arguments} | xpipes -f analyzer.cmd 153 | 154 | 155 | RAMdisk 156 | 157 | It is recommended that if vortex is used to analyze large amounts of data, especially if the data is deleted after it is analyzed, that temporary stream files be written to a file system in RAM instead of on hard disk. Ex. tmpfs or /dev/shm on most linux distros. 158 | 159 | Packet Capture: 160 | 161 | The following setting changes may help if you are experiencing packet loss between the kernel and vortex (PCAP_DROP in performance stats). 162 | 163 | #! /bin/bash 164 | # 165 | #Red Hat Defaults: 166 | # 167 | cat /proc/sys/net/core/rmem_default 168 | cat /proc/sys/net/core/rmem_max 169 | cat /proc/sys/net/core/netdev_max_backlog 170 | # 171 | #New settings 172 | # 173 | echo 67108864 > /proc/sys/net/core/rmem_default 174 | echo 67108864 > /proc/sys/net/core/rmem_max 175 | echo 50000 > /proc/sys/net/core/netdev_max_backlog 176 | 177 | 178 | 179 | #for /etc/sysctl.conf to make the changes persistant 180 | 181 | #Set some capture pertinant settings 182 | #defaults 183 | #net.core.rmem_default = 109568 184 | #net.core.rmem_max = 131071 185 | #net.core.netdev_max_backlog = 1000 186 | # 187 | #net.core.rmem_default = 67108864 188 | #net.core.rmem_max = 67108864 189 | #net.core.netdev_max_backlog = 50000 190 | 191 | The following may help if you are experiencing packet loss between the NIC and kernel (ifconfig RX dropped): 192 | 193 | To check the current setting and determine the maximum, use the ethtool -g option: 194 | 195 | /sbin/ethtool -g eth2 196 | # Ring parameters for eth2: 197 | # Pre-set maximums: 198 | # RX: 1020 199 | # RX Mini: 0 200 | # RX Jumbo: 0 201 | # TX: 255 202 | # Current hardware settings: 203 | # RX: 255 204 | # RX Mini: 0 205 | # RX Jumbo: 0 206 | # TX: 255 207 | 208 | 209 | #To change the rx ring size, use the ethtool -G option: 210 | 211 | /sbin/ethtool -G eth2 rx 1020 212 | 213 | #To make this change permanent put the command(s) in the /sbin/ifup-local script. It can be created as follows: 214 | 215 | touch /sbin/ifup-local 216 | chmod 744 /sbin/ifup-local 217 | echo "/sbin/ethtool -G eth2 rx 1020" >> /sbin/ifup-local 218 | 219 | #The max rx ring size is 1020 bnx2 and 4096 for e1000e. 220 | 221 | 222 | Errors: 223 | 224 | The errors reported by vortex are explained in more detail as follows: 225 | 226 | TOTAL: total errors since program start 227 | IP_SIZE: packets with invalid size (reflected from libNIDS) 228 | IP_FRAG: problems with IP frag (reflected from libNIDS) 229 | IP_HDR: problems with IP header (reflected from libNIDS) 230 | IP_SRCRT: packets attempting source routing (reflected from libNIDS) 231 | TCP_LIMIT: maximum number of simultaneous TCP streams to follow exceeded (reflected from libNIDS) 232 | TCP_HDR: problems with TCP header. If this value is very high, consider disabling TCP checksums (-k) (reflected from libNIDS) 233 | TCP_QUE: problems with the TCP received queue. This is usually caused by packet loss. (reflected from libNIDS) 234 | TCP_FLAGS: problems with hte TCP header flags (reflected from libNIDS) 235 | UDP_ALL: various problems with UDP (reflected from libNIDS) 236 | SCAN_ALL: various detections such as port scans. These detection mechanisms are normally disabled. (reflected from libNIDS) 237 | VTX_IO: Errors occuring writing data stream out 238 | VTX_MEM: Errors occured allocating memory. Streams may either be dropped or may contain "holes". 239 | VTX_RING: internal lockless stream ring queue was overrun and streams were dropped. Try increasing ring size (-Q) and/or decreasing poll interval (-R) 240 | OTHER: Other miscelaneous errors 241 | 242 | Performance Stats: 243 | 244 | The stats reported by vortex are explained in more detail as follows: 245 | 246 | PCAP_RECV: number of packets processed by vortex (reflected from libpcap). Now 64-bit wrapper around the 32-bit value returned by PCAP. Collection and reporting thread must be with low enough period to prevent aliasing. 247 | PCAP_DROP: number of packets dropped by vortex (reflected from libpcap). Now 64-bit wrapper around the 32-bit value returned by PCAP. Collection and reporting thread must be with low enough period to prevent aliasing. 248 | VTX_BYTES: number of bytes of stream data collected by vortex (doesn't include headers, filtered, or other non-collected data) 249 | VTX_EST: number of streams established 250 | VTX_WAIT: number of streams finished but not yet dumped 251 | VTX_CLOSE_TOT: number of streams for which we are finished collection 252 | VTX_CLOSE: number of streams finished because of tcp close 253 | VTX_LIMIT: number of streams finished because of size limits 254 | VTX_POLL: number of streams finished (skipped) due to polling 255 | VTX_TIMOUT: number of streams finished due to TCP timeout 256 | VTX_IDLE: number of streams finished due to inactivity (tunable -K) 257 | VTX_RST: number of streams finished due to TCP reset 258 | VTX_EXIT: number of stream finished (possibly pre-maturely) due to vortex exiting 259 | VTX_BSF: number of streams finished (skipped) due to BSF filtering 260 | 261 | Analyzer Development: 262 | 263 | While vortex is designed to facilitate use of existing code, it is often beneficial to create analyzers which may well be simply wrappers for other existing code. In some situations, it may be useful to create an analyzer from scratch specifically for use with vortex. 264 | 265 | It is perfectly acceptable to use scripting or interpretted langauages to implement vortex analyzers. In general, the analyzer will: 266 | 1. Read file names (meta data) from STDIN 267 | 2. Do some processing on the file 268 | 3. Purge the file 269 | 270 | The following is an example of a possible shell script based analyzer which runs cracklib on FTP traffic, identifying users with weak passwords: 271 | 272 | 273 | vortex -i eth0 -s 1000000 -C 0 -S 1000 -l -t /dev/shm/ -f "tcp port 21" -e -K 300 | while read file 274 | #Loop on each stream file. file contains metadata that looks something like "/dev/shm/tcp-38-1261503711-1261503711-c-174-172.16.17.18:3787s10.20.30.40:21" 275 | do 276 | #extract credentials from stream 277 | CREDS=`cat $file | tr '\r\n' ' ' | sed -r 's/.*USER (\S+) PASS (\S+) .+$/\1 \2/g'` 278 | #CREDS is something like "username password" 279 | 280 | RESULT=`echo "$CREDS" | cut -f 1 -d " " --complement | /usr/sbin/cracklib-check` 281 | #RESULTS is either "password: warning message" or "password: OK" 282 | 283 | #If password isn't OK, the log then username, warning message, and connection info 284 | if ! echo $RESULT | grep "OK$" > /dev/null 285 | then 286 | USER=`echo $CREDS | cut -f 1 -d " "` 287 | MESSAGE=`echo $RESULT | awk -F: '{ print $NF }'` 288 | METADATA=`basename $file` 289 | 290 | echo "Weak password ($MESSAGE ) for $USER in connection $METADATA" | logger -t "ftp_pass_check" -s 291 | fi 292 | 293 | #Purge the file 294 | rm $file 295 | done 296 | 297 | 298 | The output would be something like the following: 299 | 300 | ftp_pass_check: Weak password ( it is based on a dictionary word ) for alice in connection tcp-38-1261503711-1261503711-c-174-172.16.17.18:3787s10.20.30.40:21 301 | ftp_pass_check: Weak password ( it is too short ) for bob in connection tcp-40-1261503713-1261503714-c-192.168.254.254:54738s172.16.4.82:21 302 | 303 | Note the command line options. These options are the more common options you should consider using for live analysis. In addition to these, it is recommended to set -H, -E, -R, and -L accordingly. 304 | 305 | If you are performing analysis that has very high latency, you may need to implement threading or parallel processing. If you are using tmpfs (/dev/shm) for your temporary storage area, you might use mmap() instead of read(), if available, to load data into a buffer for analysis. 306 | 307 | 308 | Limitations: 309 | 310 | 311 | Vortex has very little support for network encapsulation other than IPv4 over Ethernet. You will likely have problems with IPv6, GRE tunnels, IPSEC, etc. 312 | 313 | Vortex doesn't dump streams until collection for that stream has finished (either a limit was reached or stream was closed). This is by design but can be troublematic for streams that are kept open for a very long time (which are rare in practice). 314 | 315 | Vortex only supports TCP. 316 | 317 | 318 | Community Contributions: 319 | (The following are provided for information purposes only.) 320 | 321 | Securityfu Blog: Vortex IDS - Get Super Snagadocious on Ubuntu, http://securityfu.blogspot.com/2010/02/vortex-ids-get-super-snagadocious-on.html 322 | - Introduction to vortex on Ubuntu 323 | SmuSec Blog: Vortex Howto Series http://smusec.blogspot.com/search/label/vortex%20howto 324 | - Series of posts on how to use vortex for various tasks 325 | Security Onion LiveCD: http://securityonion.blogspot.com/, https://sourceforge.net/projects/security-onion/ 326 | - Ubuntu based LiveCD contain numerous netwerk security/IDS tools 327 | StreamDB: Network stream database framework, http://code.google.com/p/streamdb/ 328 | - Open source project that stores and retrieves streams recontructed by vortex 329 | Ruminate IDS: Modular System for Network Payload Analysis, http://ruminate-ids.org/ 330 | - IDS based on vortex focussing on analysis of payload object such as documents 331 | FreeBSD patch: https://sourceforge.net/news/?group_id=255425&id=298842 332 | - Patch disabling some less portable features of vortex, making compilation easier on BSDs 333 | 334 | Note: the Smusec blog and Ruminate IDS are independent works of Charles Smutz, one of the authors of vortex. 335 | 336 | Newer community contributions can be found on the vortex news page: https://sourceforge.net/news/?group_id=255425 337 | 338 | 339 | Improvements: 340 | 341 | There are many improvements to be made to vortex. If you make useful improvements, please provide them to the authors so we can consider integrating them. Even improvements to documentation, packaging, etc are both needed and desired. 342 | 343 | Support and Feature requests: 344 | 345 | While we can provide no gaurantees for support or improvements, if you are using vortex and have any feedback that will help us drive improvements, we'd love to hear from you. You can post on the sourceforge forum or contact us directly: opensource.tools.security@lmco.com 346 | 347 | 348 | 349 | -------------------------------------------------------------------------------- /vortex/LICENSE: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2008-2011 Lockheed Martin Corporation 4 | * 5 | * The Vortex program is open source software: you can copy it, redistribute it and/or modify 6 | * it under the terms of the GNU General Public License version 2.0 as published by 7 | * the Free Software Foundation. The Vortex Program and any derivatives of the Vortex program 8 | * must be licensed under GPL version 2.0 and may not be licensed under GPL version 3.0. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY OF ANY KIND, including without limitation the implied warranties of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details at http://www.gnu.org/licenses/gpl-2.0.html. 14 | * 15 | * The term "Vortex" should be taken to also include any portions or derived works of Vortex. 16 | * 17 | * You are highly encouraged to send your changes to the Vortex program to 18 | * opensource.tools.security@lmco.com for possible incorporation into the main distribution. 19 | * By sending these changes to Lockheed Martin, you are granting to Lockheed Martin 20 | * Corporation the unlimited, perpetual, non-exclusive right to reuse, modify, 21 | * and/or relicense the code on a royalty-free basis. 22 | * 23 | * The libraries to which Vortex links are distributed under the terms of their own licenses. 24 | * Please see those libraries for their applicable licenses. 25 | * 26 | * 27 | * Applications created to analyze or process data that is outputted from 28 | * Vortex are viewed as separate works and not derivative. 29 | * 30 | */ 31 | 32 | GNU GENERAL PUBLIC LICENSE 33 | Version 2, June 1991 34 | 35 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 36 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 37 | Everyone is permitted to copy and distribute verbatim copies 38 | of this license document, but changing it is not allowed. 39 | 40 | Preamble 41 | 42 | The licenses for most software are designed to take away your 43 | freedom to share and change it. By contrast, the GNU General Public 44 | License is intended to guarantee your freedom to share and change free 45 | software--to make sure the software is free for all its users. This 46 | General Public License applies to most of the Free Software 47 | Foundation's software and to any other program whose authors commit to 48 | using it. (Some other Free Software Foundation software is covered by 49 | the GNU Lesser General Public License instead.) You can apply it to 50 | your programs, too. 51 | 52 | When we speak of free software, we are referring to freedom, not 53 | price. Our General Public Licenses are designed to make sure that you 54 | have the freedom to distribute copies of free software (and charge for 55 | this service if you wish), that you receive source code or can get it 56 | if you want it, that you can change the software or use pieces of it 57 | in new free programs; and that you know you can do these things. 58 | 59 | To protect your rights, we need to make restrictions that forbid 60 | anyone to deny you these rights or to ask you to surrender the rights. 61 | These restrictions translate to certain responsibilities for you if you 62 | distribute copies of the software, or if you modify it. 63 | 64 | For example, if you distribute copies of such a program, whether 65 | gratis or for a fee, you must give the recipients all the rights that 66 | you have. You must make sure that they, too, receive or can get the 67 | source code. And you must show them these terms so they know their 68 | rights. 69 | 70 | We protect your rights with two steps: (1) copyright the software, and 71 | (2) offer you this license which gives you legal permission to copy, 72 | distribute and/or modify the software. 73 | 74 | Also, for each author's protection and ours, we want to make certain 75 | that everyone understands that there is no warranty for this free 76 | software. If the software is modified by someone else and passed on, we 77 | want its recipients to know that what they have is not the original, so 78 | that any problems introduced by others will not reflect on the original 79 | authors' reputations. 80 | 81 | Finally, any free program is threatened constantly by software 82 | patents. We wish to avoid the danger that redistributors of a free 83 | program will individually obtain patent licenses, in effect making the 84 | program proprietary. To prevent this, we have made it clear that any 85 | patent must be licensed for everyone's free use or not licensed at all. 86 | 87 | The precise terms and conditions for copying, distribution and 88 | modification follow. 89 | 90 | GNU GENERAL PUBLIC LICENSE 91 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 92 | 93 | 0. This License applies to any program or other work which contains 94 | a notice placed by the copyright holder saying it may be distributed 95 | under the terms of this General Public License. The "Program", below, 96 | refers to any such program or work, and a "work based on the Program" 97 | means either the Program or any derivative work under copyright law: 98 | that is to say, a work containing the Program or a portion of it, 99 | either verbatim or with modifications and/or translated into another 100 | language. (Hereinafter, translation is included without limitation in 101 | the term "modification".) Each licensee is addressed as "you". 102 | 103 | Activities other than copying, distribution and modification are not 104 | covered by this License; they are outside its scope. The act of 105 | running the Program is not restricted, and the output from the Program 106 | is covered only if its contents constitute a work based on the 107 | Program (independent of having been made by running the Program). 108 | Whether that is true depends on what the Program does. 109 | 110 | 1. You may copy and distribute verbatim copies of the Program's 111 | source code as you receive it, in any medium, provided that you 112 | conspicuously and appropriately publish on each copy an appropriate 113 | copyright notice and disclaimer of warranty; keep intact all the 114 | notices that refer to this License and to the absence of any warranty; 115 | and give any other recipients of the Program a copy of this License 116 | along with the Program. 117 | 118 | You may charge a fee for the physical act of transferring a copy, and 119 | you may at your option offer warranty protection in exchange for a fee. 120 | 121 | 2. You may modify your copy or copies of the Program or any portion 122 | of it, thus forming a work based on the Program, and copy and 123 | distribute such modifications or work under the terms of Section 1 124 | above, provided that you also meet all of these conditions: 125 | 126 | a) You must cause the modified files to carry prominent notices 127 | stating that you changed the files and the date of any change. 128 | 129 | b) You must cause any work that you distribute or publish, that in 130 | whole or in part contains or is derived from the Program or any 131 | part thereof, to be licensed as a whole at no charge to all third 132 | parties under the terms of this License. 133 | 134 | c) If the modified program normally reads commands interactively 135 | when run, you must cause it, when started running for such 136 | interactive use in the most ordinary way, to print or display an 137 | announcement including an appropriate copyright notice and a 138 | notice that there is no warranty (or else, saying that you provide 139 | a warranty) and that users may redistribute the program under 140 | these conditions, and telling the user how to view a copy of this 141 | License. (Exception: if the Program itself is interactive but 142 | does not normally print such an announcement, your work based on 143 | the Program is not required to print an announcement.) 144 | 145 | These requirements apply to the modified work as a whole. If 146 | identifiable sections of that work are not derived from the Program, 147 | and can be reasonably considered independent and separate works in 148 | themselves, then this License, and its terms, do not apply to those 149 | sections when you distribute them as separate works. But when you 150 | distribute the same sections as part of a whole which is a work based 151 | on the Program, the distribution of the whole must be on the terms of 152 | this License, whose permissions for other licensees extend to the 153 | entire whole, and thus to each and every part regardless of who wrote it. 154 | 155 | Thus, it is not the intent of this section to claim rights or contest 156 | your rights to work written entirely by you; rather, the intent is to 157 | exercise the right to control the distribution of derivative or 158 | collective works based on the Program. 159 | 160 | In addition, mere aggregation of another work not based on the Program 161 | with the Program (or with a work based on the Program) on a volume of 162 | a storage or distribution medium does not bring the other work under 163 | the scope of this License. 164 | 165 | 3. You may copy and distribute the Program (or a work based on it, 166 | under Section 2) in object code or executable form under the terms of 167 | Sections 1 and 2 above provided that you also do one of the following: 168 | 169 | a) Accompany it with the complete corresponding machine-readable 170 | source code, which must be distributed under the terms of Sections 171 | 1 and 2 above on a medium customarily used for software interchange; or, 172 | 173 | b) Accompany it with a written offer, valid for at least three 174 | years, to give any third party, for a charge no more than your 175 | cost of physically performing source distribution, a complete 176 | machine-readable copy of the corresponding source code, to be 177 | distributed under the terms of Sections 1 and 2 above on a medium 178 | customarily used for software interchange; or, 179 | 180 | c) Accompany it with the information you received as to the offer 181 | to distribute corresponding source code. (This alternative is 182 | allowed only for noncommercial distribution and only if you 183 | received the program in object code or executable form with such 184 | an offer, in accord with Subsection b above.) 185 | 186 | The source code for a work means the preferred form of the work for 187 | making modifications to it. For an executable work, complete source 188 | code means all the source code for all modules it contains, plus any 189 | associated interface definition files, plus the scripts used to 190 | control compilation and installation of the executable. However, as a 191 | special exception, the source code distributed need not include 192 | anything that is normally distributed (in either source or binary 193 | form) with the major components (compiler, kernel, and so on) of the 194 | operating system on which the executable runs, unless that component 195 | itself accompanies the executable. 196 | 197 | If distribution of executable or object code is made by offering 198 | access to copy from a designated place, then offering equivalent 199 | access to copy the source code from the same place counts as 200 | distribution of the source code, even though third parties are not 201 | compelled to copy the source along with the object code. 202 | 203 | 4. You may not copy, modify, sublicense, or distribute the Program 204 | except as expressly provided under this License. Any attempt 205 | otherwise to copy, modify, sublicense or distribute the Program is 206 | void, and will automatically terminate your rights under this License. 207 | However, parties who have received copies, or rights, from you under 208 | this License will not have their licenses terminated so long as such 209 | parties remain in full compliance. 210 | 211 | 5. You are not required to accept this License, since you have not 212 | signed it. However, nothing else grants you permission to modify or 213 | distribute the Program or its derivative works. These actions are 214 | prohibited by law if you do not accept this License. Therefore, by 215 | modifying or distributing the Program (or any work based on the 216 | Program), you indicate your acceptance of this License to do so, and 217 | all its terms and conditions for copying, distributing or modifying 218 | the Program or works based on it. 219 | 220 | 6. Each time you redistribute the Program (or any work based on the 221 | Program), the recipient automatically receives a license from the 222 | original licensor to copy, distribute or modify the Program subject to 223 | these terms and conditions. You may not impose any further 224 | restrictions on the recipients' exercise of the rights granted herein. 225 | You are not responsible for enforcing compliance by third parties to 226 | this License. 227 | 228 | 7. If, as a consequence of a court judgment or allegation of patent 229 | infringement or for any other reason (not limited to patent issues), 230 | conditions are imposed on you (whether by court order, agreement or 231 | otherwise) that contradict the conditions of this License, they do not 232 | excuse you from the conditions of this License. If you cannot 233 | distribute so as to satisfy simultaneously your obligations under this 234 | License and any other pertinent obligations, then as a consequence you 235 | may not distribute the Program at all. For example, if a patent 236 | license would not permit royalty-free redistribution of the Program by 237 | all those who receive copies directly or indirectly through you, then 238 | the only way you could satisfy both it and this License would be to 239 | refrain entirely from distribution of the Program. 240 | 241 | If any portion of this section is held invalid or unenforceable under 242 | any particular circumstance, the balance of the section is intended to 243 | apply and the section as a whole is intended to apply in other 244 | circumstances. 245 | 246 | It is not the purpose of this section to induce you to infringe any 247 | patents or other property right claims or to contest validity of any 248 | such claims; this section has the sole purpose of protecting the 249 | integrity of the free software distribution system, which is 250 | implemented by public license practices. Many people have made 251 | generous contributions to the wide range of software distributed 252 | through that system in reliance on consistent application of that 253 | system; it is up to the author/donor to decide if he or she is willing 254 | to distribute software through any other system and a licensee cannot 255 | impose that choice. 256 | 257 | This section is intended to make thoroughly clear what is believed to 258 | be a consequence of the rest of this License. 259 | 260 | 8. If the distribution and/or use of the Program is restricted in 261 | certain countries either by patents or by copyrighted interfaces, the 262 | original copyright holder who places the Program under this License 263 | may add an explicit geographical distribution limitation excluding 264 | those countries, so that distribution is permitted only in or among 265 | countries not thus excluded. In such case, this License incorporates 266 | the limitation as if written in the body of this License. 267 | 268 | 9. The Free Software Foundation may publish revised and/or new versions 269 | of the General Public License from time to time. Such new versions will 270 | be similar in spirit to the present version, but may differ in detail to 271 | address new problems or concerns. 272 | 273 | Each version is given a distinguishing version number. If the Program 274 | specifies a version number of this License which applies to it and "any 275 | later version", you have the option of following the terms and conditions 276 | either of that version or of any later version published by the Free 277 | Software Foundation. If the Program does not specify a version number of 278 | this License, you may choose any version ever published by the Free Software 279 | Foundation. 280 | 281 | 10. If you wish to incorporate parts of the Program into other free 282 | programs whose distribution conditions are different, write to the author 283 | to ask for permission. For software which is copyrighted by the Free 284 | Software Foundation, write to the Free Software Foundation; we sometimes 285 | make exceptions for this. Our decision will be guided by the two goals 286 | of preserving the free status of all derivatives of our free software and 287 | of promoting the sharing and reuse of software generally. 288 | 289 | NO WARRANTY 290 | 291 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 292 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 293 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 294 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 295 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 296 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 297 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 298 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 299 | REPAIR OR CORRECTION. 300 | 301 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 302 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 303 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 304 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 305 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 306 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 307 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 308 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 309 | POSSIBILITY OF SUCH DAMAGES. 310 | 311 | END OF TERMS AND CONDITIONS 312 | 313 | How to Apply These Terms to Your New Programs 314 | 315 | If you develop a new program, and you want it to be of the greatest 316 | possible use to the public, the best way to achieve this is to make it 317 | free software which everyone can redistribute and change under these terms. 318 | 319 | To do so, attach the following notices to the program. It is safest 320 | to attach them to the start of each source file to most effectively 321 | convey the exclusion of warranty; and each file should have at least 322 | the "copyright" line and a pointer to where the full notice is found. 323 | 324 | 325 | Copyright (C) 326 | 327 | This program is free software; you can redistribute it and/or modify 328 | it under the terms of the GNU General Public License as published by 329 | the Free Software Foundation; either version 2 of the License, or 330 | (at your option) any later version. 331 | 332 | This program is distributed in the hope that it will be useful, 333 | but WITHOUT ANY WARRANTY; without even the implied warranty of 334 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 335 | GNU General Public License for more details. 336 | 337 | You should have received a copy of the GNU General Public License along 338 | with this program; if not, write to the Free Software Foundation, Inc., 339 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 340 | 341 | Also add information on how to contact you by electronic and paper mail. 342 | 343 | If the program is interactive, make it output a short notice like this 344 | when it starts in an interactive mode: 345 | 346 | Gnomovision version 69, Copyright (C) year name of author 347 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 348 | This is free software, and you are welcome to redistribute it 349 | under certain conditions; type `show c' for details. 350 | 351 | The hypothetical commands `show w' and `show c' should show the appropriate 352 | parts of the General Public License. Of course, the commands you use may 353 | be called something other than `show w' and `show c'; they could even be 354 | mouse-clicks or menu items--whatever suits your program. 355 | 356 | You should also get your employer (if you work as a programmer) or your 357 | school, if any, to sign a "copyright disclaimer" for the program, if 358 | necessary. Here is a sample; alter the names: 359 | 360 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 361 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 362 | 363 | , 1 April 1989 364 | Ty Coon, President of Vice 365 | 366 | This General Public License does not permit incorporating your program into 367 | proprietary programs. If your program is a subroutine library, you may 368 | consider it more useful to permit linking proprietary applications with the 369 | library. If this is what you want to do, use the GNU Lesser General 370 | Public License instead of this License. 371 | -------------------------------------------------------------------------------- /vortex/vortex.c: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2008-2011 Lockheed Martin Corporation 4 | * 5 | * The Vortex program is open source software: you can copy it, redistribute it and/or modify 6 | * it under the terms of the GNU General Public License version 2.0 as published by 7 | * the Free Software Foundation. The Vortex Program and any derivatives of the Vortex program 8 | * must be licensed under GPL version 2.0 and may not be licensed under GPL version 3.0. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY OF ANY KIND, including without limitation the implied warranties of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details at http://www.gnu.org/licenses/gpl-2.0.html. 14 | * 15 | * The term "Vortex" should be taken to also include any portions or derived works of Vortex. 16 | * 17 | * You are highly encouraged to send your changes to the Vortex program to 18 | * opensource.tools.security@lmco.com for possible incorporation into the main distribution. 19 | * By sending these changes to Lockheed Martin, you are granting to Lockheed Martin 20 | * Corporation the unlimited, perpetual, non-exclusive right to reuse, modify, 21 | * and/or relicense the code on a royalty-free basis. 22 | * 23 | * The libraries to which Vortex links are distributed under the terms of their own licenses. 24 | * Please see those libraries for their applicable licenses. 25 | * 26 | */ 27 | /* 28 | * Vortex 29 | * a flexible program for tcp reassembly 30 | * compile: 31 | * (If you have libbsf installed as shared library) 32 | * gcc vortex.c -lnids -lpthread -lbsf -lpcap -Wall -DWITH_BSF -o vortex -O2 33 | * (If you don't have libsf) 34 | * gcc vortex.c -lnids -lpthread -lpcap -Wall -o vortex -O2 35 | */ 36 | 37 | #define _GNU_SOURCE 38 | 39 | #ifdef linux 40 | #include 41 | #endif 42 | 43 | #include 44 | #include 45 | #include 46 | #include 47 | #include 48 | #include 49 | #include 50 | #include 51 | #include 52 | #include 53 | #include 54 | #include 55 | #include 56 | #include 57 | #include 58 | #include 59 | #include 60 | #include "nids.h" 61 | #include 62 | 63 | #ifdef WITH_BSF 64 | #include 65 | #endif 66 | 67 | //don't ask me why this isn't in headers? 68 | #define gettid() syscall(__NR_gettid) 69 | #define my_sched_setaffinity(a,b,c) sched_setaffinity(a, b, c) 70 | 71 | //TODO LIST: 72 | 73 | //Long Term: 74 | //Deal with URG data? 75 | //Extend interface to UDP? 76 | //Major Performance Improvements? 77 | 78 | #define int_ntoa(x) inet_ntoa(*((struct in_addr *)&x)) 79 | #define VORTEX_DIRECTION_TO_SERVER 0 80 | #define VORTEX_DIRECTION_TO_CLIENT 1 81 | #define VORTEX_DIRECTION_TO_BOTH 2 82 | 83 | #define VORTEX_STILL_OPEN 0 84 | #define VORTEX_SIZE_LIMIT_REACHED 16 85 | #define VORTEX_IDLE 17 86 | 87 | #define REALLOC_INCREASE_PERCENT_MAX 10000 88 | 89 | //Define some Macros for error logging 90 | #define DEBUG(l,m...) { if (l <= debug_level) { LOG(LOG_DEBUG, m); } } 91 | #define WARN(m...) { LOG(LOG_WARNING, m); } 92 | #define ERROR(m...) { LOG(LOG_ERR, m); } 93 | #define DIE(m...) { LOG(LOG_CRIT, m); exit(1); } 94 | #define LOG(p,m...) { syslog(p, m); fprintf(stderr, m); fprintf(stderr, "\n"); } 95 | 96 | //max number of chars that can be included in metadata 97 | //this is used to ensure buffers are not overrun 98 | //45 is max for normal metadata 99 | //99 for extended output 100 | #define METADATA_MAX 200 101 | 102 | 103 | //global vars,Defaults 104 | //for getopt 105 | extern int optind; 106 | extern int opterr; 107 | extern int optopt; 108 | extern char *optarg; 109 | pthread_attr_t pthread_attrs; 110 | int collection = VORTEX_DIRECTION_TO_SERVER; 111 | char *temp_data_dir=""; 112 | 113 | //Limits for capture in both directions in bytes 114 | size_t to_server_data_size_limit = 104857600; //100 MB 115 | size_t to_client_data_size_limit = 104857600; //100 MB 116 | int debug_level = 0; 117 | 118 | unsigned long int connection_id = 0; 119 | unsigned long int closed_connections = 0; 120 | //connection closures by type--hopefully the sum of the following is equal to total 121 | unsigned long int closed_connections_close = 0; 122 | unsigned long int closed_connections_reset = 0; 123 | unsigned long int closed_connections_timeout = 0; 124 | unsigned long int closed_connections_idle = 0; 125 | unsigned long int closed_connections_exit = 0; 126 | 127 | unsigned long int closed_connections_limit = 0; 128 | unsigned long int closed_connections_poll = 0; 129 | unsigned long int closed_connections_bsf = 0; 130 | 131 | 132 | unsigned long int max_connections = 0; 133 | unsigned long int collected_bytes = 0; 134 | 135 | unsigned long int pcap_stats_total_recv = 0; 136 | unsigned long int pcap_stats_total_drop = 0; 137 | u_int pcap_stats_last_recv = 0; 138 | u_int pcap_stats_last_drop = 0; 139 | 140 | //realloc_copy 141 | //unsigned long int realloc_copy_count = 0; 142 | //unsigned long int realloc_copy_bytes = 0; 143 | 144 | 145 | 146 | 147 | int poll_rate = 1; 148 | //for BPF 149 | char *filter_file = NULL; 150 | char *filter_string = NULL; 151 | //for BSF 152 | char *bsf_filter_file = NULL; 153 | char *bsf_filter_string = NULL; 154 | int bsf_enabled = 0; 155 | char *user = NULL; 156 | char *process_name = NULL; 157 | 158 | //pcap options 159 | int snap_len = 1560; 160 | pcap_t *desc = NULL; 161 | char pcap_err_buf[PCAP_ERRBUF_SIZE]; 162 | 163 | //error logging 164 | struct error_stats errors; 165 | //name for syslog 166 | char *logging_name; 167 | unsigned int error_interval = 0; 168 | 169 | //pointer to libnids nids_syslog function 170 | void (*libnids_syslog)(int, int, struct ip *, void*)= NULL; 171 | 172 | //stats 173 | unsigned int stats_interval = 0; 174 | 175 | #ifdef WITH_BSF 176 | bsf_t *bsf_desc; 177 | #endif 178 | 179 | //stuff related to ring buffer 180 | //ring buffer is utilized to allow capture and reassembly thread to operate unihibited by stream writing thread. 181 | //the whole point of using a lockless ring buffer is that we will never have to lock the capture thread waiting for the stream writing thread. As such no locks are used. Additions to ring occur in one thread and removal occurs in the other. 182 | struct conn_param **conn_ring = NULL; 183 | int conn_ring_head = 0; 184 | int conn_ring_tail = 0; 185 | int conn_ring_size = 10000; 186 | unsigned int conn_ring_poll_interval = 10000; 187 | 188 | //adding notion of state of vortex program run. 0 => intialization, 1 => stream collection and processing, 2 => dump remaining streams, 3 => other cleanup (wait for error and stats threads to finish, etc) 189 | int program_state = 0; 190 | 191 | 192 | //thread priorities--yes even vortex has priorities 193 | int capture_prio = -15; 194 | int other_prio = 10; 195 | //cpu affinity--override CPU scheduling 196 | int capture_cpu = -1; 197 | int other_cpu = -1; 198 | int stats_thread_init = 0; 199 | int error_thread_init = 0; 200 | int output_thread_init = 0; 201 | //int cpu_locking = 0; 202 | int extended_output = 0; 203 | 204 | //By default open connections stay open for ever 205 | int tcp_max_idle = -1; 206 | struct conn_param *tcp_head; 207 | struct conn_param *tcp_tail; 208 | 209 | //timestampe from last packet we've processed 210 | unsigned int last_timestamp; 211 | 212 | int output_empty_streams = 0; 213 | 214 | //for time stomps 215 | int stomp_temp_file_timestamps = 0; 216 | int disable_temp_file_stomp = 0; 217 | 218 | //for realloc scaling 219 | //starting size of steam buffers 220 | //make this tunable??? 221 | size_t base_realloc_size = 4*1024; 222 | //percentage of growth that occurs when buffer needs to grow 223 | unsigned int realloc_increase_percent = 100; 224 | //multiplcation factor for buffer increase. 50% => 1.5, 100% => 2, 200% => 3 225 | float realloc_increase_factor = 2; 226 | //threshold above which increase isn't based on factor anymore, buffer is resized to SIZE_MAX then never increased again 227 | size_t realloc_increase_threshold = 0; 228 | 229 | 230 | 231 | //struct used for each connection we follow 232 | struct conn_param 233 | { 234 | struct tcp_stream *nids_stream; 235 | FILE *to_server_data_fp; 236 | FILE *to_client_data_fp; 237 | char *to_server_data_p; 238 | char *to_client_data_p; 239 | size_t to_server_data_size; 240 | unsigned int to_server_data_size_exceeded; 241 | size_t to_client_data_size; 242 | unsigned int to_client_data_size_exceeded; 243 | unsigned long int id; 244 | unsigned int start; 245 | unsigned int last_activity; 246 | struct tuple4 addr; 247 | char close_state; 248 | struct conn_param *prev_conn; 249 | struct conn_param *next_conn; 250 | size_t to_server_buffer_size; 251 | size_t to_client_buffer_size; 252 | 253 | }; 254 | 255 | 256 | //struct of errors 257 | struct error_stats 258 | { 259 | unsigned int total; 260 | unsigned int ip_size; 261 | unsigned int ip_frag; 262 | unsigned int ip_hdr; 263 | unsigned int ip_srcrt; 264 | unsigned int tcp_limit; 265 | unsigned int tcp_hdr; 266 | unsigned int tcp_queue; 267 | unsigned int tcp_flags; 268 | unsigned int udp_all; 269 | unsigned int scan_all; 270 | unsigned int vtx_ring; 271 | unsigned int vtx_io; 272 | unsigned int vtx_mem; 273 | unsigned int other; 274 | }; 275 | 276 | 277 | //List of every connection is maintained (to allow efficient timeouts) 278 | //The list is ordered by last activity (NIDS_DATA) for the connection, with head pointing to most stale connection 279 | 280 | //add connection to idle time list 281 | void add_conn_tcp(struct conn_param *item) 282 | { 283 | DEBUG(180, "Idle list add: %lu", item->id); 284 | 285 | if (tcp_head == NULL) 286 | { 287 | tcp_head = item; 288 | item->prev_conn = NULL; 289 | } else 290 | { 291 | item->prev_conn = tcp_tail; 292 | tcp_tail->next_conn = item; 293 | } 294 | item->next_conn = NULL; 295 | tcp_tail = item; 296 | } 297 | 298 | //remove connection from idle time list 299 | void rem_conn_tcp(struct conn_param *item) 300 | { 301 | DEBUG(180, "Idle list rem: %lu", item->id); 302 | 303 | if (item->prev_conn == NULL) 304 | { 305 | tcp_head = item->next_conn; 306 | } else 307 | { 308 | item->prev_conn->next_conn = item->next_conn; 309 | } 310 | 311 | if (item->next_conn == NULL) 312 | { 313 | tcp_tail = item->prev_conn; 314 | } else 315 | { 316 | item->next_conn->prev_conn = item->prev_conn; 317 | } 318 | } 319 | 320 | //bump connection to end of list 321 | void bump_conn_tcp(struct conn_param *item) 322 | { 323 | rem_conn_tcp(item); 324 | add_conn_tcp(item); 325 | } 326 | 327 | 328 | //return number of entries in ring 329 | //this should be safe to call from any thread with the caveat that it is only so accurate... 330 | int ring_entries() 331 | { 332 | int conn_ring_head_cache = conn_ring_head; 333 | int conn_ring_tail_cache = conn_ring_tail; 334 | 335 | if (conn_ring_head_cache >= conn_ring_tail_cache) 336 | { 337 | return (conn_ring_head_cache - conn_ring_tail_cache); 338 | } else 339 | { 340 | return (conn_ring_size - (conn_ring_tail_cache - conn_ring_head_cache)); 341 | } 342 | } 343 | 344 | //returns 1 if ring is full, 0 otherwise 345 | //should only be called by inserting thread 346 | int ring_full() 347 | { 348 | if (conn_ring_tail == 0) 349 | { 350 | if (conn_ring_head == (conn_ring_size - 1)) 351 | { 352 | return 1; 353 | } else 354 | { 355 | return 0; 356 | } 357 | } else 358 | { 359 | if (conn_ring_head == (conn_ring_tail - 1)) 360 | { 361 | return 1; 362 | } else 363 | { 364 | return 0; 365 | } 366 | } 367 | } 368 | 369 | 370 | //return 1 if ring is empty (0 entries), 0 if it is not empty 371 | int ring_empty() 372 | { 373 | if (conn_ring_head == conn_ring_tail) 374 | { 375 | return 1; 376 | } else 377 | { 378 | return 0; 379 | } 380 | } 381 | //adds and entry to the ring. ring had better be intialized and not full when this is called. Obviously, only called by capture thread. 382 | void ring_add(struct conn_param *a_conn) 383 | { 384 | conn_ring[conn_ring_head] = a_conn; 385 | conn_ring_head++; 386 | if (conn_ring_head >= conn_ring_size) 387 | { 388 | conn_ring_head = 0; 389 | } 390 | } 391 | 392 | //remove conn from ring 393 | struct conn_param *ring_remove() 394 | { 395 | struct conn_param *a_conn = NULL; 396 | a_conn = conn_ring[conn_ring_tail]; 397 | conn_ring_tail++; 398 | if (conn_ring_tail >= conn_ring_size) 399 | { 400 | conn_ring_tail = 0; 401 | } 402 | return a_conn; 403 | } 404 | 405 | 406 | /* 407 | So this funkiness deserves a little extra documentation: 408 | What we do here is override the nids_syslog function in libnids.c with this one (vortex_nids_syslog). That is fairly striaghtforward--libnids has provisions for it built into the api. However, if debugging is enabled, we want to call the nids_syslog function inside of this one. Herein lies funkiness. We grab the pointer to the nids_syslog function from the default nids_params struct (nids_params.syslog), save it off (as libnids_syslog), and then override it (with vortex_nids_syslog). If we do want to call libnids nids_syslog we now call libnids_syslog. 409 | */ 410 | //This function overrides the libnids provided nids_syslog function. We keep counters (to be printed in a separate thread periodically) instead of printing a message for each error (unless debugging is enabled, in which we do both). 411 | //Note that vortex errors don't call this function, they increment the counts and call DEBUG directly. 412 | void vortex_nids_syslog(int type, int err_num, struct ip *iph, void *data) 413 | { 414 | errors.total++; 415 | 416 | //if debug_level is above threshold, call libnids implemented nids_syslog funcation also 417 | if ((debug_level >= 10) && (libnids_syslog != NULL)) 418 | { 419 | libnids_syslog(type, err_num, iph, data); 420 | } 421 | 422 | switch (type) { 423 | case NIDS_WARN_IP: 424 | if (err_num == NIDS_WARN_IP_OVERSIZED) errors.ip_size++; 425 | if (err_num == NIDS_WARN_IP_INVLIST) errors.ip_frag++; 426 | if (err_num == NIDS_WARN_IP_OVERLAP) errors.ip_frag++; 427 | if (err_num == NIDS_WARN_IP_HDR) errors.ip_hdr++; 428 | if (err_num == NIDS_WARN_IP_SRR) errors.ip_srcrt++; 429 | break; 430 | 431 | case NIDS_WARN_TCP: 432 | if (err_num == NIDS_WARN_TCP_TOOMUCH) errors.tcp_limit++; 433 | if (err_num == NIDS_WARN_TCP_HDR) errors.tcp_hdr++; 434 | if (err_num == NIDS_WARN_TCP_BIGQUEUE) errors.tcp_queue++; 435 | if (err_num == NIDS_WARN_TCP_BADFLAGS) errors.tcp_flags++; 436 | break; 437 | 438 | case NIDS_WARN_UDP: 439 | errors.udp_all++; 440 | break; 441 | case NIDS_WARN_SCAN: 442 | errors.scan_all++; 443 | break; 444 | default: 445 | errors.other++; 446 | break; 447 | } 448 | 449 | 450 | 451 | } 452 | 453 | //Available switches 454 | //a b . . . . . . . j . . . . . . . . . . . . . . y z 455 | //A B . . . . . . I J . . . . . . . . . . U V W X Y Z 456 | 457 | void print_usage() 458 | { 459 | fprintf(stderr,"Usage: %s [ -lpmheId ] [ -c count ] [ -i device ] [ -r file ] [ -u user ] [ -S bytes ] [ -C bytes ] [ -t dir ] [ -s count ] [ -H count ] [ -q limit ] [ -D level ] [-F file | -f filter ] [-M MTU (snaplen)] [-P poll rate] [ -TEK time ] [ -Q size ] [ -R usecs ] [ -Nn prio ] [ -Oo cpu ] [ -L name ] [ -x percent ]", process_name); 460 | 461 | #ifdef WITH_BSF 462 | fprintf(stderr,"[-G file | -g filter ]"); 463 | #endif 464 | 465 | fprintf(stderr,"\n\n"); 466 | 467 | 468 | fprintf(stderr, 469 | " -h print this help message and exit\n" 470 | " -c count set number to connections to follow\n" 471 | " -i device listen on device\n" 472 | " -r file read capture from pcap file\n" 473 | " -l set output to line buffering\n" 474 | " -p don't put interface(s) in promiscuous mode\n" 475 | " -u user after initialization, setuid to user\n" 476 | " -S bytes number of bytes to collect from client to server Default: 104857600 (100MB)\n" 477 | " -C bytes number of bytes to collect from server to client Default: 104857600 (100MB)\n" 478 | " -t dir directory for storage of stream data (defaut: currend working dir)\n" 479 | " -s count Size of connection hash table--Maximum number of streams to follow simultaneously = 3/4 * count. Default: 1048576\n \t\t\t\t This affects memory consumption significantly. If you have problems with TCP_LIMIT, increase this value. (See n_tcp_streams in libNIDS)\n" 480 | " -H count size of IP defrag has table. Default: 65536 (See n_hosts in libNIDS)\n" 481 | " -m enable libNIDS multiprocess mode DEPRICATED--don't use this (See multiproc in libNIDS)\n" 482 | " -q limit set libNIDS packetqueue limit. DEPRICATED--only applies in multiproc mode\n" 483 | " -D level set debug level Default: 0\n" 484 | " -f filter tcpdump-style capture filter expression (don't forget quotes/shell escapes)\n" 485 | " -F file file containing packet filter expression\n" 486 | " -M MTU MTU or snaplen--maximum packet size to capture. default: 1560\n" 487 | " -w enable libNIDS TCP/IP stack workaround mode (See TCP_workarounds in libNIDS)\n" 488 | " -k disable libNIDS TCP/IP checksum processing (See TCP_checksums in libNIDS)\n" 489 | " -P rate Only reassemble and collect every poll rate connections. default: 1\n" 490 | " -T time Report Performance Statistics every time seconds (approx) default: 0\n" 491 | " -E time Report Error counts every time seconds (approx) default: 0\n" 492 | " -L name Logging name for syslog. Default: vortex\n" 493 | " -Q size Size of output ring queue. Sets limit for number of finished streams waiting to be written. Default: 10000\n" 494 | " -R usec Wait period in us (inverse of poll rate) for stream output thread in microseconds. Default: 10000\n" 495 | " -n prio Priority (niceness) for capture thread. Can be from -20 to 19 on most systems. Default: -15\n" 496 | " -N prio Priority (niceness) for other threads. Can be from -20 to 19 on most systems. Default: 10\n" 497 | " -o cpu CPU to bind capture thread to. Negative to disable. Default: -1\n" 498 | " -O cpu CPU to bind other threads to. Negative to disable. Default: -1\n" 499 | //" -I Lock threads to specific cores. (see o and O above). Default is to not lock so specific cores (Expiramental--still not working properly!). \n" 500 | " -e enable extended output (more metadata in file name). \n" 501 | " -K TCP Idle connection timeout in seconds Default: -1 (disabled). This timeout ignores empty keepalives. \n" 502 | " -v Output empty streams (create files with 0 bytes).\n" 503 | " -x percent Grow stream buffers by percent when needed. Must be 0-10000. 0 is minimum required to hold data. Default: 100.\n" 504 | " -d Disable setting timestamp of output files to that from pcap file when replaying packets.\n"); 505 | 506 | #ifdef WITH_BSF 507 | fprintf(stderr, 508 | " -g filter BSF stream filter expression (don't forget quotes/shell escapes)\n" 509 | " -G file file containing stream filter expression\n" 510 | ); 511 | #endif 512 | 513 | fprintf(stderr," \n\n"); 514 | 515 | } 516 | 517 | void flow_name(int conn_proto, unsigned long int conn_id, unsigned int conn_start, unsigned int conn_end, char conn_close_state, struct tuple4 addr, uint direction, unsigned int s_size, unsigned int c_size, char *buf) 518 | { 519 | //METADATA_MAX must be updated if this is updated! 520 | buf[0]='\0'; 521 | char close_state = 'c'; 522 | if (extended_output) 523 | { 524 | if (conn_proto == 17) 525 | { 526 | sprintf (buf + strlen (buf), "udp-"); 527 | } else 528 | { 529 | sprintf (buf + strlen (buf), "tcp-"); 530 | } 531 | 532 | unsigned int flow_size = c_size + s_size; 533 | 534 | 535 | 536 | 537 | switch (conn_close_state) 538 | { 539 | 540 | case NIDS_CLOSE: 541 | close_state='c'; 542 | break; 543 | case NIDS_RESET: 544 | close_state='r'; 545 | break; 546 | case NIDS_TIMED_OUT: 547 | close_state='t'; 548 | break; 549 | case NIDS_EXITING: 550 | close_state='e'; 551 | break; 552 | case VORTEX_SIZE_LIMIT_REACHED: 553 | close_state='l'; 554 | break; 555 | case VORTEX_IDLE: 556 | close_state='i'; 557 | break; 558 | } 559 | 560 | sprintf(buf + strlen(buf), "%li-%u-%u-%c-%u-", conn_id, conn_start, conn_end, close_state, flow_size); 561 | 562 | 563 | } 564 | 565 | if (direction == VORTEX_DIRECTION_TO_SERVER) 566 | { 567 | strcat (buf, int_ntoa (addr.saddr)); 568 | sprintf (buf + strlen (buf), ":%is", addr.source); 569 | strcat (buf, int_ntoa (addr.daddr)); 570 | sprintf (buf + strlen (buf), ":%i", addr.dest); 571 | } 572 | else if (direction == VORTEX_DIRECTION_TO_CLIENT) 573 | { 574 | 575 | strcat (buf, int_ntoa (addr.saddr)); 576 | sprintf (buf + strlen (buf), ":%ic", addr.source); 577 | strcat (buf, int_ntoa (addr.daddr)); 578 | sprintf (buf + strlen (buf), ":%i", addr.dest); 579 | } 580 | else 581 | { 582 | strcat (buf, int_ntoa (addr.saddr)); 583 | sprintf (buf + strlen (buf), ":%i-", addr.source); 584 | strcat (buf, int_ntoa (addr.daddr)); 585 | sprintf (buf + strlen (buf), ":%i", addr.dest); 586 | } 587 | return; 588 | } 589 | 590 | void data_filename(int conn_proto, unsigned long int conn_id, unsigned int conn_start, unsigned int conn_end, char conn_close_state, struct tuple4 addr, uint direction, unsigned int s_size, unsigned int c_size, char *buf) 591 | { 592 | //we use a lot of "unsafe" functions here and in above. Rationale is that metadata len has hard max (see METADATA max) that can be determined in advance (max not dependent on input). Temp dir was checked to ensure it is short enough such that buffer of PATH_MAX will not be overrun, even when dir, metadata, and / separater are combined. 593 | int dirstr_len; 594 | char flow_str[METADATA_MAX]; //for flow identifier 595 | strcpy (buf, temp_data_dir); 596 | dirstr_len = strlen(buf); 597 | if ((dirstr_len != 0) && (buf[dirstr_len-1] != '/') ) 598 | { 599 | strcat(buf + dirstr_len, "/"); 600 | } 601 | flow_name(conn_proto, conn_id, conn_start, conn_end, conn_close_state, addr, direction, s_size, c_size, flow_str); 602 | strcat(buf,flow_str); 603 | return; 604 | } 605 | 606 | //opens file, mallocs buffer, and reads file into buffer--returns number of bytes read. 607 | //if returns 0, buffer is not alloc'd, otherwise it is and must be freed later 608 | //buffer is 1 byte longer than file so that buffer can be null terminated. Null char is not included in byte count returned 609 | int read_file_into_buffer(char *filter_file, char **filter_buffer) 610 | { 611 | 612 | FILE *filter_file_fp; 613 | int filter_file_len; 614 | 615 | //Open file 616 | filter_file_fp = fopen(filter_file, "r"); 617 | if (!filter_file_fp) 618 | { 619 | WARN("Couldn't open file: %s",filter_file); 620 | return 0; 621 | } 622 | 623 | DEBUG(200,"Opened file"); 624 | //Get file length 625 | fseek(filter_file_fp, 0, SEEK_END); 626 | filter_file_len=ftell(filter_file_fp); 627 | fseek(filter_file_fp, 0, SEEK_SET); 628 | 629 | DEBUG(300,"File %s contains %i bytes", filter_file, filter_file_len); 630 | 631 | //Allocate memory 632 | *filter_buffer=(char *)calloc(1,(filter_file_len+1)); 633 | if (!*filter_buffer) 634 | { 635 | WARN("Couldn't malloc buffer for file: %s", filter_file); 636 | return 0; 637 | } 638 | 639 | 640 | //Read file contents into buffer 641 | if (fread(*filter_buffer, filter_file_len, 1, filter_file_fp) != 1) 642 | { 643 | 644 | WARN("Couldn't read file: %s",filter_file); 645 | free(*filter_buffer); 646 | return 0; 647 | } 648 | 649 | fclose(filter_file_fp); 650 | 651 | //Null terminate 652 | (*filter_buffer)[filter_file_len] = '\0'; 653 | 654 | 655 | return filter_file_len; 656 | } 657 | 658 | 659 | 660 | 661 | // Disable checksum feature of libnids 662 | 663 | void disable_chksum_ctl() 664 | { 665 | static struct nids_chksum_ctl ctl; 666 | 667 | ctl.netaddr = inet_addr("0.0.0.0"); 668 | ctl.mask = inet_addr("0.0.0.0"); 669 | ctl.action = NIDS_DONT_CHKSUM; 670 | nids_register_chksum_ctl(&ctl, 1); 671 | } 672 | 673 | 674 | //function used to dump error counts 675 | 676 | void print_errors() 677 | { 678 | DEBUG(0,"VORTEX_ERRORS TOTAL: %u IP_SIZE: %u IP_FRAG: %u IP_HDR: %u IP_SRCRT: %u TCP_LIMIT: %u TCP_HDR: %u TCP_QUE: %u TCP_FLAGS: %u UDP_ALL: %u SCAN_ALL: %u VTX_RING: %u VTX_IO: %u VTX_MEM: %u OTHER: %u", errors.total, errors.ip_size, errors.ip_frag, errors.ip_hdr, errors.ip_srcrt, errors.tcp_limit, errors.tcp_hdr, errors.tcp_queue, errors.tcp_flags, errors.udp_all, errors.scan_all, errors.vtx_ring, errors.vtx_io, errors.vtx_mem, errors.other); 679 | } 680 | 681 | //Thread for periodically dumping error counts 682 | //Since this thread only read, no synchronization is required 683 | void *errors_thread(void *arg) 684 | { 685 | //set priority of this thread--likely not portable 686 | if (setpriority(PRIO_PROCESS, gettid(), other_prio) != 0) 687 | { 688 | WARN("Couldn't set error thread priority!"); 689 | } 690 | //ok, now lock to a specific processor 691 | if (other_cpu >= 0) 692 | { 693 | cpu_set_t csmask; 694 | CPU_ZERO(&csmask); 695 | CPU_SET(other_cpu, &csmask); 696 | if (my_sched_setaffinity(gettid(), sizeof(cpu_set_t), &csmask) != 0) 697 | { 698 | WARN("Couldn't set processor affinity for error thread"); 699 | } 700 | } 701 | 702 | error_thread_init = 1; 703 | 704 | 705 | while ((error_interval > 0) && (program_state < 3)) 706 | { 707 | print_errors(); 708 | sleep(error_interval); 709 | } 710 | pthread_exit(NULL); 711 | } 712 | 713 | //function that actually does the output of stats 714 | 715 | void print_stats() 716 | { 717 | struct pcap_stat pcap_dev_stats; 718 | pcap_dev_stats.ps_recv = 0; 719 | pcap_dev_stats.ps_drop = 0; 720 | 721 | 722 | if (nids_params.pcap_desc != NULL) 723 | { 724 | if (pcap_stats(nids_params.pcap_desc, &pcap_dev_stats ) != 0) 725 | { 726 | //didn't get anything back 727 | WARN("PCAP_STATS failed!"); 728 | pcap_dev_stats.ps_recv = 0; 729 | pcap_dev_stats.ps_drop = 0; 730 | } else 731 | { 732 | //got something back, now increment long counter appropriately 733 | if (pcap_dev_stats.ps_recv < pcap_stats_last_recv) 734 | { 735 | //short counter rolled over 736 | pcap_stats_total_recv += (UINT_MAX - pcap_stats_last_recv) + pcap_dev_stats.ps_recv; 737 | } else 738 | { 739 | //simple case, simple increment 740 | pcap_stats_total_recv += (pcap_dev_stats.ps_recv - pcap_stats_last_recv); 741 | } 742 | if (pcap_dev_stats.ps_drop < pcap_stats_last_drop) 743 | { 744 | //short counter rolled over 745 | pcap_stats_total_drop += (UINT_MAX - pcap_stats_last_drop) + pcap_dev_stats.ps_drop; 746 | } else 747 | { 748 | //simple case, simple increment 749 | pcap_stats_total_drop += (pcap_dev_stats.ps_drop - pcap_stats_last_drop); 750 | } 751 | 752 | //set "last" 32bit counters 753 | pcap_stats_last_recv = pcap_dev_stats.ps_recv; 754 | pcap_stats_last_drop = pcap_dev_stats.ps_drop; 755 | 756 | } 757 | } 758 | DEBUG(0,"VORTEX_STATS PCAP_RECV: %lu PCAP_DROP: %lu VTX_BYTES: %lu VTX_EST: %lu VTX_WAIT: %i VTX_CLOSE_TOT: %lu VTX_CLOSE: %lu VTX_LIMIT: %lu VTX_POLL: %lu VTX_TIMOUT: %lu VTX_IDLE: %lu VTX_RST: %lu VTX_EXIT: %lu VTX_BSF: %lu",pcap_stats_total_recv, pcap_stats_total_drop, collected_bytes, connection_id, ring_entries(), closed_connections, closed_connections_close, closed_connections_limit, closed_connections_poll, closed_connections_timeout, closed_connections_idle, closed_connections_reset, closed_connections_exit, closed_connections_bsf ); 759 | } 760 | 761 | //Thread for periodically dumping stats 762 | //since this thread only reads, no synchronization is required 763 | void *stats_thread(void *arg) 764 | { 765 | 766 | 767 | //set priority of this thread--likely not portable 768 | if (setpriority(PRIO_PROCESS, gettid(), other_prio) != 0) 769 | { 770 | WARN("Couldn't set stats thread priority!"); 771 | } 772 | 773 | //ok, now lock to a specific processor 774 | if (other_cpu >= 0) 775 | { 776 | cpu_set_t csmask; 777 | CPU_ZERO(&csmask); 778 | CPU_SET(other_cpu, &csmask); 779 | if (my_sched_setaffinity(gettid(), sizeof(cpu_set_t), &csmask) != 0) 780 | { 781 | WARN("Couldn't set processor affinity for stats thread"); 782 | } 783 | } 784 | stats_thread_init = 1; 785 | 786 | while ( (stats_interval > 0) && (program_state < 3) ) 787 | { 788 | print_stats(); 789 | sleep(stats_interval); 790 | } 791 | pthread_exit(NULL); 792 | } 793 | 794 | 795 | //Free the malloc'd buffers associated with a connection 796 | void free_connection(struct conn_param *a_conn) 797 | { 798 | if (a_conn->to_server_data_p != NULL) 799 | { 800 | free(a_conn->to_server_data_p); 801 | } 802 | if (a_conn->to_client_data_p != NULL) 803 | { 804 | free(a_conn->to_client_data_p); 805 | } 806 | free(a_conn); 807 | } 808 | 809 | //Used to take all the connection data, provide it to consumers, free, and close 810 | void dump_stream(struct conn_param *a_conn) 811 | { 812 | //taken from linux/limits.h 813 | char temp_filename[PATH_MAX]; 814 | 815 | //these vars only used for debugging 816 | struct timeval system_time; 817 | char server_ip_string[INET6_ADDRSTRLEN+1]; 818 | char client_ip_string[INET6_ADDRSTRLEN+1]; 819 | // 820 | 821 | //these values used from timestomping 822 | struct timeval stomp_times[2]; 823 | 824 | if (stomp_temp_file_timestamps) 825 | { 826 | stomp_times[0].tv_sec = last_timestamp; 827 | stomp_times[0].tv_usec = 0; 828 | stomp_times[1].tv_sec = last_timestamp; 829 | stomp_times[1].tv_usec = 0; 830 | } 831 | 832 | DEBUG(400, "Closing Flow: %lu\n", a_conn->id); 833 | 834 | //Output stream if we are collecting it and if there is data or we want to output empty streams 835 | if ((to_server_data_size_limit > 0) && ( (a_conn->to_server_data_p != NULL ) || ( output_empty_streams == 1 ) ) ) 836 | { 837 | DEBUG(400, "Dumping Server Data: %zu bytes", a_conn->to_server_data_size); 838 | data_filename(6, a_conn->id, a_conn->start, a_conn->last_activity, a_conn->close_state, a_conn->addr,VORTEX_DIRECTION_TO_SERVER, a_conn->to_server_data_size, a_conn->to_client_data_size, temp_filename); 839 | a_conn->to_server_data_fp = fopen(temp_filename, "w"); 840 | 841 | if (a_conn->to_server_data_fp != NULL) 842 | { 843 | DEBUG(500, "Server data file opened: %s", temp_filename); 844 | 845 | //Only write data if there is data to write 846 | if (a_conn->to_server_data_p != NULL) 847 | { 848 | if (fwrite(a_conn->to_server_data_p,a_conn->to_server_data_size, 1, a_conn->to_server_data_fp) == 1) 849 | { 850 | //write succesful; 851 | DEBUG(500,"%zu bytes written to %s",a_conn->to_server_data_size,temp_filename); 852 | } else 853 | { 854 | //write failed 855 | errors.vtx_io++; 856 | errors.total++; 857 | WARN("Couldn't write %zu bytes to server data file: %s, skipping", a_conn->to_server_data_size, temp_filename); 858 | } 859 | } 860 | fclose(a_conn->to_server_data_fp); 861 | 862 | if (stomp_temp_file_timestamps) 863 | { 864 | //don't care about return. If we fail we're not going to do anything about it 865 | utimes(temp_filename, stomp_times); 866 | } 867 | 868 | //only print if creating file was succesful 869 | //output filename to be read by consumer 870 | printf("%s\n",temp_filename); 871 | 872 | } else 873 | { 874 | errors.vtx_io++; 875 | errors.total++; 876 | ERROR("Couldn't open server data file: %s, skipping",temp_filename); 877 | } 878 | 879 | 880 | 881 | } 882 | 883 | if ((to_client_data_size_limit > 0) && ( (a_conn->to_client_data_p != NULL ) || ( output_empty_streams == 1 ) ) ) 884 | { 885 | DEBUG(400, "Dumping Client Data: %zu bytes", a_conn->to_client_data_size); 886 | data_filename(6, a_conn->id, a_conn->start, a_conn->last_activity, a_conn->close_state, a_conn->addr,VORTEX_DIRECTION_TO_CLIENT, a_conn->to_server_data_size, a_conn->to_client_data_size, temp_filename); 887 | a_conn->to_client_data_fp = fopen(temp_filename, "w"); 888 | 889 | if (a_conn->to_client_data_fp != NULL) 890 | { 891 | DEBUG(500, "Client data file opened: %s", temp_filename); 892 | 893 | //Only write data if there is data to write 894 | if (a_conn->to_client_data_p != NULL) 895 | { 896 | if (fwrite(a_conn->to_client_data_p,a_conn->to_client_data_size, 1, a_conn->to_client_data_fp) == 1) 897 | { 898 | //write succesful; 899 | DEBUG(500,"%zu bytes written to %s",a_conn->to_server_data_size,temp_filename); 900 | } else 901 | { 902 | //write failed 903 | errors.vtx_io++; 904 | errors.total++; 905 | WARN("Couldn't write %zu bytes to client data file: %s, skipping", a_conn->to_client_data_size, temp_filename); 906 | } 907 | } 908 | fclose(a_conn->to_client_data_fp); 909 | 910 | if (stomp_temp_file_timestamps) 911 | { 912 | //don't care about return. If we fail we're not going to do anything about it 913 | utimes(temp_filename, stomp_times); 914 | } 915 | 916 | //only print if creating file was succesful 917 | //output filename to be read by consumer 918 | printf("%s\n",temp_filename); 919 | 920 | } else 921 | { 922 | errors.vtx_io++; 923 | errors.total++; 924 | ERROR("Couldn't open client data file: %s, skipping",temp_filename); 925 | } 926 | 927 | 928 | } 929 | 930 | //do debuging for streams 931 | if (debug_level >= 112) 932 | { 933 | //get the server and client ips 934 | inet_ntop(AF_INET, (const void *)&a_conn->addr.saddr, client_ip_string, INET6_ADDRSTRLEN); 935 | inet_ntop(AF_INET, (const void *)&a_conn->addr.daddr, server_ip_string, INET6_ADDRSTRLEN); 936 | 937 | gettimeofday(&system_time, NULL); 938 | //don't do pacp time because may not be safe in this thread 939 | DEBUG(112, "Wrote connection: %lu, conn: %s:%i-%s:%i, system time: %lu.%06lu, pcap_time: 0.0", a_conn->id, client_ip_string, a_conn->addr.source, server_ip_string, a_conn->addr.dest, system_time.tv_sec, system_time.tv_usec); 940 | } 941 | 942 | free_connection(a_conn); 943 | } 944 | 945 | 946 | //Thread for writing stream data 947 | void *conn_writer(void *arg) 948 | { 949 | //set priority of this thread--likely not portable 950 | if (setpriority(PRIO_PROCESS, gettid(), other_prio) != 0) 951 | { 952 | WARN("Couldn't set output thread priority!"); 953 | } 954 | 955 | //ok, now lock to a specific processor 956 | if (other_cpu >= 0) 957 | { 958 | cpu_set_t csmask; 959 | CPU_ZERO(&csmask); 960 | CPU_SET(other_cpu, &csmask); 961 | if (my_sched_setaffinity(gettid(), sizeof(cpu_set_t), &csmask) != 0) 962 | { 963 | WARN("Couldn't set processor affinity for output thread"); 964 | } 965 | } 966 | output_thread_init = 1; 967 | 968 | while(program_state < 3) 969 | { 970 | usleep(conn_ring_poll_interval); 971 | while(!(ring_empty())) 972 | { 973 | dump_stream(ring_remove()); 974 | } 975 | if ((program_state == 2) && (ring_empty())) 976 | { 977 | //give ourselves time to ensure all connections have been put into ring 978 | //since there is (intentionally) no synchronization between capture thread and output thread, this is probably the best we can do. 979 | usleep(conn_ring_poll_interval*2); 980 | if (ring_empty()) 981 | { 982 | program_state = 3; 983 | } 984 | } 985 | } 986 | pthread_exit(NULL); 987 | } 988 | 989 | void close_connection(struct conn_param *a_conn, char close_reason) 990 | { 991 | 992 | //these vars only used for debugging 993 | struct timeval system_time; 994 | char server_ip_string[INET6_ADDRSTRLEN+1]; 995 | char client_ip_string[INET6_ADDRSTRLEN+1]; 996 | // 997 | 998 | //pthread_t thread_h; 999 | //int pthread_rc; 1000 | 1001 | //thread safety!!! we only reach this check in a single thread. If we ever get here in multiple threads, we need a mutex. 1002 | // This check prevents the connection from being closed more than once 1003 | if (a_conn->close_state >= NIDS_CLOSE) 1004 | { 1005 | return; 1006 | } 1007 | 1008 | 1009 | //do debuging for new streams 1010 | if (debug_level >= 110) 1011 | { 1012 | //get the server and client ips 1013 | inet_ntop(AF_INET, (const void *)&a_conn->addr.saddr, client_ip_string, INET6_ADDRSTRLEN); 1014 | inet_ntop(AF_INET, (const void *)&a_conn->addr.daddr, server_ip_string, INET6_ADDRSTRLEN); 1015 | 1016 | gettimeofday(&system_time, NULL); 1017 | DEBUG(110, "End connection: %lu, conn: %s:%i-%s:%i, system time: %lu.%06lu, pcap_time: %lu.%06lu", a_conn->id, client_ip_string, a_conn->addr.source, server_ip_string, a_conn->addr.dest, system_time.tv_sec, system_time.tv_usec, nids_last_pcap_header->ts.tv_sec, nids_last_pcap_header->ts.tv_usec); 1018 | } 1019 | 1020 | a_conn->close_state = close_reason; 1021 | rem_conn_tcp(a_conn); 1022 | 1023 | 1024 | //block here if we are reading from pcap, no reason to drop packets..... 1025 | if (nids_params.filename != NULL) 1026 | { 1027 | while(ring_full()) 1028 | { 1029 | usleep(conn_ring_poll_interval); 1030 | } 1031 | } 1032 | 1033 | //if there is room in ring, close connection normally. If not, close it brutually doing the frees. 1034 | if (ring_full()) 1035 | { 1036 | free_connection(a_conn); 1037 | errors.vtx_ring++; 1038 | errors.total++; 1039 | } else 1040 | { 1041 | ring_add(a_conn); 1042 | } 1043 | 1044 | 1045 | closed_connections++; 1046 | 1047 | switch (close_reason) 1048 | { 1049 | 1050 | case NIDS_CLOSE: 1051 | closed_connections_close++; 1052 | break; 1053 | case NIDS_RESET: 1054 | closed_connections_reset++; 1055 | break; 1056 | case NIDS_TIMED_OUT: 1057 | closed_connections_timeout++; 1058 | break; 1059 | case NIDS_EXITING: 1060 | closed_connections_exit++; 1061 | break; 1062 | case VORTEX_SIZE_LIMIT_REACHED: 1063 | closed_connections_limit++; 1064 | break; 1065 | case VORTEX_IDLE: 1066 | closed_connections_idle++; 1067 | break; 1068 | 1069 | } 1070 | 1071 | 1072 | 1073 | if ((max_connections > 0) && (closed_connections >= max_connections)) 1074 | { 1075 | WARN("Reached max connections, exiting...") 1076 | program_state = 2; 1077 | //nids_exit(); 1078 | //wait_for_other_threads(); 1079 | //TODO: make this ending a little less abrupt 1080 | 1081 | 1082 | DIE("Stopping after %lu connections", max_connections) 1083 | } 1084 | } 1085 | 1086 | 1087 | int filter_stream_bsf(struct tcp_stream *a_tcp) 1088 | { 1089 | 1090 | #ifdef WITH_BSF 1091 | if (bsf_filter(bsf_desc, a_tcp->addr.saddr, a_tcp->addr.source, a_tcp->addr.daddr, a_tcp->addr.dest) == BSF_RESULT_PASS) 1092 | { 1093 | return 1; 1094 | } else 1095 | { 1096 | return 0; 1097 | } 1098 | #else 1099 | return 1; 1100 | #endif 1101 | } 1102 | 1103 | 1104 | void tcp_callback (struct tcp_stream *a_tcp, struct conn_param **conn_param_ptr) 1105 | { 1106 | 1107 | 1108 | //these vars only used for debugging 1109 | struct timeval system_time; 1110 | char server_ip_string[INET6_ADDRSTRLEN+1]; 1111 | char client_ip_string[INET6_ADDRSTRLEN+1]; 1112 | // 1113 | 1114 | struct conn_param *a_conn; 1115 | 1116 | char *realloc_old_ptr = NULL; 1117 | size_t realloc_old_size = 0; 1118 | 1119 | 1120 | if (debug_level >= 150) 1121 | { 1122 | gettimeofday(&system_time, NULL); 1123 | if (a_tcp->nids_state == NIDS_JUST_EST) 1124 | { 1125 | DEBUG(150, "TCP callback. %s, state: %i, system time: %lu.%06lu, pcap_time: %lu.%06lu", "na", a_tcp->nids_state, system_time.tv_sec, system_time.tv_usec, nids_last_pcap_header->ts.tv_sec, nids_last_pcap_header->ts.tv_usec); 1126 | } else if (a_tcp->nids_state == NIDS_DATA) 1127 | { 1128 | DEBUG(151, "TCP callback: %lu, state: %i, system time: %lu.%06lu, pcap_time: %lu.%06lu", ((struct conn_param *) *conn_param_ptr)->id, a_tcp->nids_state, system_time.tv_sec, system_time.tv_usec, nids_last_pcap_header->ts.tv_sec, nids_last_pcap_header->ts.tv_usec); 1129 | } else 1130 | { 1131 | DEBUG(150, "TCP callback: %lu, state: %i, system time: %lu.%06lu, pcap_time: %lu.%06lu", ((struct conn_param *) *conn_param_ptr)->id, a_tcp->nids_state, system_time.tv_sec, system_time.tv_usec, nids_last_pcap_header->ts.tv_sec, nids_last_pcap_header->ts.tv_usec); 1132 | } 1133 | } 1134 | 1135 | 1136 | //short circuit if we are done 1137 | if (program_state > 1) 1138 | { 1139 | nids_exit(); 1140 | return; 1141 | } 1142 | 1143 | 1144 | if (a_tcp->nids_state != NIDS_EXITING) 1145 | { 1146 | last_timestamp = (unsigned int) nids_last_pcap_header->ts.tv_sec; 1147 | } 1148 | 1149 | //check for idle connections now 1150 | //we are intentionally only doing one per callback 1151 | if ((tcp_max_idle >= 0) && (tcp_head != NULL)) 1152 | { 1153 | //Make sure we don't clobber current connection 1154 | if (a_tcp != tcp_head->nids_stream) 1155 | { 1156 | if ((last_timestamp - tcp_head->last_activity) > tcp_max_idle) 1157 | { 1158 | nids_free_tcp_stream(tcp_head->nids_stream); 1159 | close_connection(tcp_head, VORTEX_IDLE); 1160 | } 1161 | } 1162 | } 1163 | 1164 | 1165 | 1166 | if (a_tcp->nids_state == NIDS_JUST_EST) 1167 | { 1168 | 1169 | connection_id++; 1170 | 1171 | //do debuging for new streams 1172 | if (debug_level >= 111) 1173 | { 1174 | //get the server and client ips 1175 | inet_ntop(AF_INET, (const void *)&a_tcp->addr.saddr, client_ip_string, INET6_ADDRSTRLEN); 1176 | inet_ntop(AF_INET, (const void *)&a_tcp->addr.daddr, server_ip_string, INET6_ADDRSTRLEN); 1177 | 1178 | gettimeofday(&system_time, NULL); 1179 | DEBUG(111, "New connection: %lu, conn: %s:%i-%s:%i, system time: %lu.%06lu, pcap_time: %lu.%06lu", connection_id, client_ip_string, a_tcp->addr.source, server_ip_string, a_tcp->addr.dest, system_time.tv_sec, system_time.tv_usec, nids_last_pcap_header->ts.tv_sec, nids_last_pcap_header->ts.tv_usec); 1180 | } 1181 | 1182 | 1183 | //Filtering First 1184 | 1185 | if (bsf_enabled) 1186 | { 1187 | if (filter_stream_bsf(a_tcp)) 1188 | { 1189 | DEBUG(450,"Stream PASSED BSF"); 1190 | 1191 | } else 1192 | { 1193 | DEBUG(450,"Stream FAILED BSF"); 1194 | closed_connections_bsf++; 1195 | closed_connections++; 1196 | return; 1197 | } 1198 | } 1199 | 1200 | 1201 | //Polling 1202 | if (connection_id % poll_rate == 0) 1203 | { 1204 | DEBUG(400,"Setting up new connection with id: %lu",connection_id); 1205 | 1206 | //Set up conn_param struct 1207 | a_conn=calloc(1,sizeof(struct conn_param)); 1208 | 1209 | if (a_conn == NULL) 1210 | { 1211 | inet_ntop(AF_INET, (const void *)&a_tcp->addr.saddr, client_ip_string, INET6_ADDRSTRLEN); 1212 | inet_ntop(AF_INET, (const void *)&a_tcp->addr.daddr, server_ip_string, INET6_ADDRSTRLEN); 1213 | 1214 | //return without setting up connection param and tell libNIDS to ignore this one--this should happen very infrequently 1215 | ERROR("Couldn't malloc connection parameter struct, skipping connection: conn: %s:%i-%s:%i\n", client_ip_string, a_tcp->addr.source, server_ip_string, a_tcp->addr.dest); 1216 | 1217 | //re-use conn id 1218 | connection_id--; 1219 | nids_free_tcp_stream(a_tcp); 1220 | errors.vtx_mem++; 1221 | errors.total++; 1222 | return; 1223 | } 1224 | 1225 | 1226 | 1227 | if ((collection == VORTEX_DIRECTION_TO_SERVER) || (collection == VORTEX_DIRECTION_TO_BOTH)) 1228 | { 1229 | a_tcp->server.collect++; // we want data received by client (to server) 1230 | } 1231 | if ((collection == VORTEX_DIRECTION_TO_CLIENT) || (collection == VORTEX_DIRECTION_TO_BOTH)) 1232 | { 1233 | a_tcp->client.collect++; // we want data received by server (to client) 1234 | } 1235 | 1236 | 1237 | 1238 | *conn_param_ptr=a_conn; 1239 | 1240 | //set id information 1241 | a_conn->id=connection_id; 1242 | memcpy (&a_conn->addr, &a_tcp->addr, sizeof(struct tuple4)); 1243 | 1244 | a_conn->nids_stream = a_tcp; 1245 | //add to timeout list 1246 | add_conn_tcp(a_conn); 1247 | 1248 | //DEBUG(400, "Flow %d Established: %s\n", a_conn->id, connection_string); 1249 | 1250 | //Set the data sizes to 0 (collecting or not) 1251 | a_conn->to_server_data_size = 0; 1252 | a_conn->to_server_data_size_exceeded = 0; 1253 | a_conn->to_client_data_size = 0; 1254 | a_conn->to_client_data_size_exceeded = 0; 1255 | //set the data pointers to null (collecting or not) 1256 | a_conn->to_server_data_p = NULL; 1257 | a_conn->to_client_data_p = NULL; 1258 | a_conn->close_state = 0; 1259 | //TODO: change this to time 1260 | a_conn->start = last_timestamp; 1261 | a_conn->last_activity = last_timestamp; 1262 | 1263 | // 1264 | a_conn->to_server_buffer_size = 0; 1265 | a_conn->to_client_buffer_size = 0; 1266 | 1267 | } else 1268 | { 1269 | //This is not a polled connection, close it! 1270 | closed_connections_poll++; 1271 | closed_connections++; 1272 | } //end of polling conditional 1273 | return; 1274 | }//End of NIDS_JUST_EST 1275 | else 1276 | { 1277 | //Not Just Established 1278 | //Get our conn_param struct 1279 | a_conn = *conn_param_ptr; 1280 | //return if we don't have a conn_param for this connection (probably exluded due to polling) 1281 | if (a_conn == NULL) 1282 | { 1283 | return; 1284 | } 1285 | } 1286 | 1287 | //CLOSE, RESET, TIMED_OUT, EXITING 1288 | if ( (a_tcp->nids_state == NIDS_CLOSE) || (a_tcp->nids_state == NIDS_RESET) || (a_tcp->nids_state == NIDS_TIMED_OUT) || (a_tcp->nids_state == NIDS_EXITING) ) 1289 | { 1290 | 1291 | 1292 | //fprintf (stderr, "Connection %d Closed\n", a_conn->id); 1293 | 1294 | close_connection(a_conn, a_tcp->nids_state); 1295 | 1296 | return; 1297 | } 1298 | 1299 | if (a_tcp->nids_state == NIDS_DATA) 1300 | { 1301 | a_conn->last_activity = last_timestamp; 1302 | bump_conn_tcp(a_conn); 1303 | if ((a_tcp->client.count_new) && ((collection == VORTEX_DIRECTION_TO_CLIENT) || (collection == VORTEX_DIRECTION_TO_BOTH))) //probably ought to check if we are colleting also 1304 | { 1305 | 1306 | //see if buffer needs to be realloc'd 1307 | if ( a_tcp->client.count_new+a_conn->to_client_data_size > a_conn->to_client_buffer_size ) 1308 | { 1309 | realloc_old_ptr = a_conn->to_client_data_p; 1310 | realloc_old_size = a_conn->to_client_buffer_size; 1311 | if (realloc_increase_percent == 0 ) 1312 | { 1313 | a_conn->to_client_buffer_size = a_conn->to_client_data_size + a_tcp->client.count_new; 1314 | } 1315 | if ( a_conn->to_client_buffer_size == 0 ) 1316 | { 1317 | a_conn->to_client_buffer_size = base_realloc_size; 1318 | } 1319 | while ( a_tcp->client.count_new+a_conn->to_client_data_size > a_conn->to_client_buffer_size ) 1320 | { 1321 | if ( a_conn->to_client_buffer_size >= realloc_increase_threshold ) 1322 | { 1323 | a_conn->to_client_buffer_size = SIZE_MAX; 1324 | } else 1325 | { 1326 | a_conn->to_client_buffer_size *= realloc_increase_factor; 1327 | } 1328 | } 1329 | a_conn->to_client_data_p = realloc(a_conn->to_client_data_p,a_conn->to_client_buffer_size); 1330 | 1331 | //not used 1332 | //if (realloc_old_ptr != a_conn->to_client_data_p) 1333 | //{ 1334 | // realloc_copy_count++; 1335 | // realloc_copy_bytes += a_conn->to_client_data_size; 1336 | //} 1337 | } 1338 | 1339 | 1340 | //copy client data to connection buffer 1341 | if (a_conn->to_client_data_p == NULL) 1342 | { 1343 | errors.vtx_mem++; 1344 | errors.total++; 1345 | ERROR("Couldn't realloc client data buffer (leaving hole in data) in conn: %lu\n", a_conn->id); 1346 | a_conn->to_client_data_p = realloc_old_ptr; 1347 | a_conn->to_client_buffer_size = realloc_old_size; 1348 | //this stream has a "hole" in it due to realloc failure but currently is not marked as such in any way--these errors should be very rare 1349 | } else 1350 | { 1351 | //Copy the data and set the size accordingly 1352 | memcpy((a_conn->to_client_data_p + a_conn->to_client_data_size), a_tcp->client.data, a_tcp->client.count_new); 1353 | a_conn->to_client_data_size = a_tcp->client.count_new + a_conn->to_client_data_size; 1354 | collected_bytes += (unsigned long int)a_tcp->client.count_new; 1355 | } 1356 | 1357 | 1358 | 1359 | //check if connection collection size has been exceeded 1360 | if (a_conn->to_client_data_size >= to_client_data_size_limit ) 1361 | { 1362 | a_conn->to_client_data_size_exceeded += (a_conn->to_client_data_size - to_client_data_size_limit); 1363 | DEBUG(500, "Client max collection size exceeded for connection %lu by %d bytes\n", a_conn->id, a_conn->to_client_data_size_exceeded ); 1364 | //disable further data collection 1365 | a_tcp->client.collect--; 1366 | 1367 | //close this connection if we have exceeded () client and server limits (libnids still tracks it, but we don't) 1368 | if ((a_conn->to_client_data_size >= to_client_data_size_limit) && (a_conn->to_server_data_size >= to_server_data_size_limit )) 1369 | { 1370 | DEBUG(500,"Closing Connection %lu becuase size limits reached", a_conn->id) 1371 | close_connection(a_conn, VORTEX_SIZE_LIMIT_REACHED); 1372 | } 1373 | 1374 | } 1375 | } 1376 | if ((a_tcp->server.count_new) && ((collection == VORTEX_DIRECTION_TO_SERVER) || (collection == VORTEX_DIRECTION_TO_BOTH))) 1377 | { 1378 | 1379 | //see if buffer needs to be realloc'd 1380 | if ( a_tcp->server.count_new+a_conn->to_server_data_size > a_conn->to_server_buffer_size ) 1381 | { 1382 | realloc_old_ptr = a_conn->to_server_data_p; 1383 | realloc_old_size = a_conn->to_server_buffer_size; 1384 | if (realloc_increase_percent == 0 ) 1385 | { 1386 | a_conn->to_server_buffer_size = a_conn->to_server_data_size + a_tcp->server.count_new; 1387 | } 1388 | if ( a_conn->to_server_buffer_size == 0 ) 1389 | { 1390 | a_conn->to_server_buffer_size = base_realloc_size; 1391 | } 1392 | while ( a_tcp->server.count_new+a_conn->to_server_data_size > a_conn->to_server_buffer_size ) 1393 | { 1394 | if ( a_conn->to_server_buffer_size >= realloc_increase_threshold ) 1395 | { 1396 | a_conn->to_server_buffer_size = SIZE_MAX; 1397 | } else 1398 | { 1399 | a_conn->to_server_buffer_size *= realloc_increase_factor; 1400 | } 1401 | } 1402 | a_conn->to_server_data_p = realloc(a_conn->to_server_data_p,a_conn->to_server_buffer_size); 1403 | 1404 | //not used 1405 | //if (realloc_old_ptr != a_conn->to_server_data_p) 1406 | //{ 1407 | // realloc_copy_count++; 1408 | // realloc_copy_bytes += a_conn->to_server_data_size; 1409 | //} 1410 | } 1411 | 1412 | 1413 | //copy server data to connection buffer 1414 | if (a_conn->to_server_data_p == NULL) 1415 | { 1416 | errors.vtx_mem++; 1417 | errors.total++; 1418 | ERROR("Couldn't realloc server data buffer (leaving hole in data) in conn: %lu\n", a_conn->id); 1419 | a_conn->to_server_data_p = realloc_old_ptr; 1420 | a_conn->to_server_buffer_size = realloc_old_size; 1421 | //this stream has a "hole" in it due to realloc failure but currently is not marked as such in any way--these errors should be very rare 1422 | } else 1423 | { 1424 | //Copy the data and set the size accordingly 1425 | memcpy((a_conn->to_server_data_p + a_conn->to_server_data_size), a_tcp->server.data, a_tcp->server.count_new); 1426 | a_conn->to_server_data_size = a_tcp->server.count_new + a_conn->to_server_data_size; 1427 | collected_bytes += (unsigned long int)a_tcp->server.count_new; 1428 | } 1429 | 1430 | //check if connection collection size has been exceeded 1431 | if (a_conn->to_server_data_size >= to_server_data_size_limit ) 1432 | { 1433 | a_conn->to_server_data_size_exceeded += (a_conn->to_server_data_size - to_server_data_size_limit); 1434 | DEBUG(500, "Server max collection size exceeded for connection %lu by %d bytes\n", a_conn->id, a_conn->to_server_data_size_exceeded ); 1435 | //disable further data collection 1436 | a_tcp->server.collect--; 1437 | 1438 | //close this connection if we have exceeded () client and server limits (libnids still tracks it, but we don't) 1439 | if ((a_conn->to_client_data_size >= to_client_data_size_limit) && (a_conn->to_server_data_size >= to_server_data_size_limit )) 1440 | { 1441 | DEBUG(500,"Closing Connection %lu becuase size limits reached", a_conn->id) 1442 | close_connection(a_conn, VORTEX_SIZE_LIMIT_REACHED); 1443 | } 1444 | 1445 | } 1446 | 1447 | } 1448 | 1449 | return; 1450 | }//end of NIDS_DATA 1451 | 1452 | //Should never get here 1453 | DIE("Reached Unreachable State\n"); 1454 | 1455 | return; 1456 | 1457 | }//End of tcp_callback 1458 | 1459 | int main (int argc, char **argv) 1460 | { 1461 | int opt; 1462 | process_name = argv[0]; 1463 | pthread_t stats_thread_h; 1464 | pthread_t error_thread_h; 1465 | pthread_t writer_thread_h; 1466 | int pthread_rc; 1467 | errors.total = 0; 1468 | errors.ip_size = 0; 1469 | errors.ip_frag = 0; 1470 | errors.ip_hdr = 0; 1471 | errors.ip_srcrt = 0; 1472 | errors.tcp_limit = 0; 1473 | errors.tcp_hdr = 0; 1474 | errors.tcp_queue = 0; 1475 | errors.tcp_flags = 0; 1476 | errors.udp_all = 0; 1477 | errors.scan_all = 0; 1478 | errors.vtx_ring = 0; 1479 | errors.vtx_io = 0; 1480 | errors.vtx_mem = 0; 1481 | errors.other = 0; 1482 | 1483 | 1484 | logging_name = "Vortex"; 1485 | 1486 | DEBUG(50, "Starting up Vortex"); 1487 | 1488 | //Set pthread attributes (default to detached) 1489 | //pthread_attr_init(&pthread_attrs); 1490 | //pthread_attr_setdetachstate(&pthread_attrs, PTHREAD_CREATE_DETACHED); 1491 | 1492 | nids_params.n_tcp_streams = 1048576; 1493 | nids_params.n_hosts = 65536; 1494 | 1495 | 1496 | //parse command line options 1497 | DEBUG(100, "Parsing Commnad Line Options"); 1498 | 1499 | //set some defaults 1500 | 1501 | while ((opt = getopt(argc, argv, "hc:f:F:g:G:i:lpr:u:S:C:t:s:H:mwkq:D:M:P:T:E:L:Q:R:n:N:o:O:eK:vx:d")) != -1) 1502 | { 1503 | switch (opt) 1504 | { 1505 | case 'c': 1506 | max_connections = atoi(optarg); 1507 | DEBUG(100,"Max connections set to %lu",max_connections); 1508 | break; 1509 | 1510 | case 'f': 1511 | filter_string = optarg; 1512 | DEBUG(100,"Using packet filter from CLI arg: %s",filter_string); 1513 | break; 1514 | 1515 | case 'F': 1516 | filter_file = optarg; 1517 | DEBUG(100,"Using packet filter from file: %s",filter_file); 1518 | break; 1519 | 1520 | case 'g': 1521 | #ifdef WITH_BSF 1522 | bsf_filter_string = optarg; 1523 | bsf_enabled = 1; 1524 | DEBUG(100,"Using arg filter from CLI arg: %s",bsf_filter_string); 1525 | #else 1526 | DIE("Not compiled with support for BSF. Recompile with support for BSF (-DWITH_BSF)."); 1527 | #endif 1528 | break; 1529 | 1530 | case 'G': 1531 | #ifdef WITH_BSF 1532 | bsf_filter_file = optarg; 1533 | bsf_enabled = 1; 1534 | DEBUG(100,"Using stream filter from file: %s",bsf_filter_file); 1535 | #else 1536 | DIE("Not compiled with support for BSF. Recompile with support for BSF (-DWITH_BSF)."); 1537 | #endif 1538 | break; 1539 | 1540 | case 'i': 1541 | nids_params.device = optarg; 1542 | DEBUG(100,"Using device: %s",nids_params.device); 1543 | break; 1544 | 1545 | case 'l': 1546 | //set STDOUT to line buffering 1547 | #ifdef HAVE_SETLINEBUF 1548 | setlinebuf(stdout); 1549 | #else 1550 | setvbuf(stdout, NULL, _IOLBF, 0); 1551 | #endif 1552 | DEBUG(100,"Ouput set to line buffering"); 1553 | break; 1554 | 1555 | case 'p': 1556 | //Don't put capture in promisc mode 1557 | nids_params.promisc = 0; 1558 | DEBUG(100,"Promisc mode disabled"); 1559 | break; 1560 | 1561 | case 'r': 1562 | nids_params.filename = optarg; 1563 | DEBUG(100,"Reading from capture file: %s",nids_params.filename); 1564 | break; 1565 | 1566 | case 'u': 1567 | user = optarg; 1568 | DEBUG(100,"Switching to user: %s",user); 1569 | break; 1570 | 1571 | case 'S': 1572 | to_server_data_size_limit = (size_t)atoll(optarg); 1573 | if (to_server_data_size_limit > SIZE_MAX) 1574 | { 1575 | DIE("Invalid to server size limit of %llu, must be under %llu", (unsigned long long)to_server_data_size_limit, (unsigned long long)SIZE_MAX); 1576 | } 1577 | DEBUG(100,"Size limit of data to server: %llu", (unsigned long long)to_server_data_size_limit); 1578 | break; 1579 | 1580 | case 'C': 1581 | to_client_data_size_limit = (size_t)atoll(optarg); 1582 | if (to_client_data_size_limit > SIZE_MAX) 1583 | { 1584 | DIE("Invalid to client size limit of %llu, must be under %llu", (unsigned long long)to_server_data_size_limit, (unsigned long long)SIZE_MAX); 1585 | } 1586 | DEBUG(100,"Size limit of data to client: %llu", (unsigned long long)to_server_data_size_limit); 1587 | break; 1588 | 1589 | case 't': 1590 | temp_data_dir = optarg; 1591 | DEBUG(100,"Using temp dir: %s",temp_data_dir); 1592 | break; 1593 | 1594 | case 's': 1595 | //size of TCP streams hash tables (number of TCP streams to follow is 3/4 of this value) 1596 | nids_params.n_tcp_streams = atoi(optarg); 1597 | DEBUG(100,"Set nids parameter n_tcp_streams: %i",nids_params.n_tcp_streams); 1598 | break; 1599 | 1600 | case 'H': 1601 | //number of hosts to track for IP defrag 1602 | nids_params.n_hosts = atoi(optarg); 1603 | DEBUG(100,"Set nids parameter n_hosts: %i",nids_params.n_hosts); 1604 | break; 1605 | 1606 | case 'm': 1607 | //Set libnids to multiproc mode 1608 | nids_params.multiproc = 1; 1609 | DEBUG(100,"Enable NIDS multiproc mode"); 1610 | break; 1611 | 1612 | case 'q': 1613 | //Set libnids queuelimit 1614 | nids_params.queue_limit = atoi(optarg); 1615 | DEBUG(100,"Set nids parameter queue_limit: %i",nids_params.queue_limit); 1616 | break; 1617 | 1618 | case 'D': 1619 | debug_level = atoi(optarg); 1620 | DEBUG(100,"Set debug level to %i",debug_level); 1621 | break; 1622 | 1623 | case 'M': 1624 | snap_len = atoi(optarg); 1625 | DEBUG(100,"Using snap len: %i",snap_len); 1626 | break; 1627 | 1628 | case 'w': 1629 | //Set libnids to enable TCP/IP workaround for non-RFC compliant stacks 1630 | DEBUG(100,"Enabled TCP workarounds"); 1631 | nids_params.tcp_workarounds = 1; 1632 | break; 1633 | 1634 | case 'k': 1635 | //Set libnids to enable TCP/IP workaround for non-RFC compliant stacks 1636 | disable_chksum_ctl(); 1637 | DEBUG(100,"Disable TCP checksum checking"); 1638 | break; 1639 | case 'P': 1640 | poll_rate = atoi(optarg); 1641 | DEBUG(100,"Polling connections at rate: %i",poll_rate); 1642 | break; 1643 | case 'T': 1644 | stats_interval = atoi(optarg); 1645 | DEBUG(100,"Reporting stats at rate: %u",stats_interval); 1646 | break; 1647 | case 'E': 1648 | error_interval = atoi(optarg); 1649 | DEBUG(100,"Reporting error at rate: %u",error_interval); 1650 | break; 1651 | case 'L': 1652 | logging_name = optarg; 1653 | DEBUG(100,"Using logging name of: %s",logging_name); 1654 | break; 1655 | 1656 | case 'Q': 1657 | if (atoi(optarg) > 1) 1658 | { 1659 | conn_ring_size = atoi(optarg); 1660 | } 1661 | DEBUG(100,"Using out queue size of: %i",conn_ring_size); 1662 | break; 1663 | 1664 | case 'R': 1665 | if (atoi(optarg) > 1) 1666 | { 1667 | conn_ring_poll_interval = atoi(optarg); 1668 | } 1669 | DEBUG(100,"Stream output thread using interval of: %i usecs",conn_ring_poll_interval); 1670 | break; 1671 | 1672 | case 'n': 1673 | capture_prio = atoi(optarg); 1674 | DEBUG(100,"Set capture thread priority to: %i",capture_prio); 1675 | break; 1676 | 1677 | case 'N': 1678 | other_prio = atoi(optarg); 1679 | DEBUG(100,"Set other thread priority to: %i",other_prio); 1680 | break; 1681 | 1682 | case 'o': 1683 | capture_cpu = atoi(optarg); 1684 | DEBUG(100,"Set capture thread cpu affinity to: %i",capture_cpu); 1685 | break; 1686 | 1687 | case 'O': 1688 | other_cpu = atoi(optarg); 1689 | DEBUG(100,"Set other thread cpu affinity to: %i",other_cpu); 1690 | break; 1691 | case 'e': 1692 | extended_output = 1; 1693 | DEBUG(100,"Enabling extended output"); 1694 | break; 1695 | 1696 | case 'K': 1697 | tcp_max_idle = atoi(optarg); 1698 | DEBUG(100,"Max TCP IDLE time to: %i",tcp_max_idle); 1699 | break; 1700 | case 'v': 1701 | output_empty_streams = 1; 1702 | DEBUG(100,"Enabling output of empty streams"); 1703 | break; 1704 | 1705 | case 'x': 1706 | realloc_increase_percent = atoi(optarg); 1707 | if ((realloc_increase_percent < 0) || (realloc_increase_percent > REALLOC_INCREASE_PERCENT_MAX)) 1708 | { 1709 | DIE("Invalid value for stream data buffer increase percent. Must be between 0 and %i", REALLOC_INCREASE_PERCENT_MAX); 1710 | } 1711 | 1712 | DEBUG(100,"Stream data buffer increase set to: %i%%",realloc_increase_percent); 1713 | break; 1714 | case 'd': 1715 | disable_temp_file_stomp = 1; 1716 | DEBUG(100,"Disabling modifcation of output file timestamp"); 1717 | break; 1718 | 1719 | case 'h': 1720 | print_usage(); 1721 | exit(0); 1722 | break; 1723 | 1724 | default: 1725 | print_usage(); 1726 | DIE("Invalid option %c", opt); 1727 | break; 1728 | 1729 | } 1730 | } 1731 | 1732 | 1733 | //open syslog connection 1734 | openlog(logging_name, 0, LOG_LOCAL0); 1735 | 1736 | //get libnids syslog function before we override it. 1737 | libnids_syslog=nids_params.syslog; 1738 | nids_params.syslog=vortex_nids_syslog; 1739 | 1740 | //make sure path isn't so long we'll overrun the filename buffers 1741 | //The 2 is for the null char and for / that will need to separate dir from metadata 1742 | if (strnlen(temp_data_dir, PATH_MAX) > ((unsigned int)(PATH_MAX - (METADATA_MAX + 2)))) 1743 | { 1744 | DIE("Specified path of temp directory is too long. The temp directory must be less than %u chars in length", (unsigned int)(PATH_MAX - (METADATA_MAX + 2))); 1745 | } 1746 | 1747 | //too many args 1748 | if (argc > (optind)) 1749 | { 1750 | print_usage(); 1751 | DIE("Invalid command line options!\nYou must quote the filter expression"); 1752 | } 1753 | 1754 | //packet filter, if any is supplied 1755 | if ((filter_file != NULL) || (filter_string != NULL)) 1756 | { 1757 | if ((filter_file != NULL) && (filter_string != NULL)) 1758 | { 1759 | DIE("Can't specify packet filter in CLI arg and file at same time!"); 1760 | } 1761 | 1762 | if (filter_file != NULL) 1763 | { 1764 | 1765 | //Load filter from file 1766 | 1767 | if (read_file_into_buffer(filter_file, &filter_string) == 0) 1768 | { 1769 | DIE("Couldn't Read Filter File %s", filter_file); 1770 | } 1771 | 1772 | } 1773 | 1774 | nids_params.pcap_filter = filter_string; 1775 | DEBUG(300, "Using packet filter: %s", filter_string); 1776 | 1777 | } 1778 | 1779 | 1780 | #ifdef WITH_BSF 1781 | //now deal with bsf style filter 1782 | if (bsf_enabled) 1783 | { 1784 | //make sure we aren't using filter from CLI arg and from file at same time 1785 | if ((bsf_filter_file != NULL) && (bsf_filter_string != NULL)) 1786 | { 1787 | DIE("Can't specify stream filter in CLI arg and file at same time!"); 1788 | } 1789 | 1790 | //make sure filter expression is in bsf_filter_string; 1791 | if (bsf_filter_string == NULL) 1792 | { 1793 | //load filter string from file 1794 | if (read_file_into_buffer(bsf_filter_file, &bsf_filter_string) == 0) 1795 | { 1796 | DIE("Couldn't Read Stream Filter File %s", bsf_filter_file); 1797 | } 1798 | } 1799 | 1800 | 1801 | bsf_desc = bsf_create(); 1802 | if (bsf_desc == NULL) 1803 | { 1804 | printf("Couldn't create BSF!\n"); 1805 | exit(1); 1806 | } 1807 | if (bsf_compile(bsf_desc, bsf_filter_string, 0) == BSF_ERROR_NONE) 1808 | { 1809 | DEBUG(300,"BSF compiled successfully: %s", bsf_filter_string); 1810 | } else 1811 | { 1812 | DIE("BSF compilation failed, please check syntax: %s", bsf_filter_string); 1813 | } 1814 | } 1815 | #endif 1816 | 1817 | //Set collection direction 1818 | if ((to_server_data_size_limit == 0) && (to_client_data_size_limit == 0)) 1819 | { 1820 | DIE("Nothing to do: Both server and client collection limit set to 0B"); 1821 | } 1822 | if ((to_server_data_size_limit > 0) && (to_client_data_size_limit > 0)) 1823 | { 1824 | collection = VORTEX_DIRECTION_TO_BOTH; 1825 | } 1826 | if ((to_server_data_size_limit > 0) && (to_client_data_size_limit == 0)) 1827 | { 1828 | collection = VORTEX_DIRECTION_TO_SERVER; 1829 | } 1830 | if ((to_server_data_size_limit == 0) && (to_client_data_size_limit > 0)) 1831 | { 1832 | collection = VORTEX_DIRECTION_TO_CLIENT; 1833 | } 1834 | 1835 | 1836 | //TODO: 1837 | //check that temp dir exists 1838 | 1839 | //die if capture device and filename are set 1840 | if ((nids_params.device != NULL) && (nids_params.filename != NULL)) 1841 | { 1842 | DIE("Can't specifiy both a capture device and a capture file!"); 1843 | } 1844 | 1845 | //set file stomping if appropriate 1846 | if ((nids_params.filename != NULL) && ( disable_temp_file_stomp == 0 )) 1847 | { 1848 | stomp_temp_file_timestamps = 1; 1849 | } 1850 | 1851 | //set stuff for realloc growth 1852 | realloc_increase_factor = ((float)(100 + realloc_increase_percent))/100; 1853 | DEBUG(100,"Stream data buffer growth factor set to: %f",realloc_increase_factor); 1854 | realloc_increase_threshold = (int)(SIZE_MAX/realloc_increase_factor) - 1; 1855 | 1856 | //Open our own pcap descriptor so we can set the snap length 1857 | if (nids_params.device != NULL) 1858 | { 1859 | pcap_err_buf[0] = '\0'; 1860 | 1861 | 1862 | desc = pcap_open_live(nids_params.device, snap_len, nids_params.promisc, nids_params.pcap_timeout, pcap_err_buf); 1863 | if (desc == NULL) 1864 | { 1865 | DIE("Couldn't open device (%s) for packet capture: %s",nids_params.device,pcap_err_buf); 1866 | } else 1867 | { 1868 | DEBUG(20,"Opened device (%s) for packet capture: %s",nids_params.device,pcap_err_buf); 1869 | nids_params.pcap_desc = desc; 1870 | } 1871 | } 1872 | 1873 | 1874 | if (poll_rate < 1) 1875 | { 1876 | DIE("Poll rate must be greater than or equal to 1"); 1877 | } 1878 | 1879 | //set up ring buffer and start stream writer thread 1880 | //first malloc buffer 1881 | conn_ring = (struct conn_param **) malloc(conn_ring_size * sizeof(struct conn_param *)); 1882 | if (conn_ring == NULL) 1883 | { 1884 | DIE("Couldn't intialize connection ring of size: %i", conn_ring_size); 1885 | } 1886 | //start stream writer thread 1887 | if( (pthread_rc = pthread_create(&writer_thread_h, NULL, conn_writer, NULL)) ) 1888 | { 1889 | DIE("Error starting stream writer thread. return code from pthread_create() is %d\n", pthread_rc); 1890 | } 1891 | 1892 | 1893 | //start stats threads 1894 | if (stats_interval > 0) 1895 | { 1896 | if( (pthread_rc = pthread_create(&stats_thread_h, NULL, stats_thread, NULL)) ) 1897 | { 1898 | WARN("Error Creating stats reporting thread. Stats will not be reported. return code from pthread_create() is %d\n", pthread_rc); 1899 | stats_interval = 0; 1900 | } 1901 | } 1902 | 1903 | //start errors thread 1904 | if (error_interval > 0) 1905 | { 1906 | if( (pthread_rc = pthread_create(&error_thread_h, NULL, errors_thread, NULL)) ) 1907 | { 1908 | WARN("Error Creating error reporting thread. Error counts will not be periodically reported. return code from pthread_create() is %d\n", pthread_rc); 1909 | error_interval = 0; 1910 | } 1911 | } 1912 | 1913 | 1914 | 1915 | //set priority for capture thread before we drop priveledges 1916 | //This is most likely not portable...... 1917 | if (setpriority(PRIO_PROCESS, gettid(), capture_prio) != 0) 1918 | { 1919 | WARN("Couldn't set capture thread priority!"); 1920 | } 1921 | //ok, now lock to a specific processor 1922 | 1923 | if (capture_cpu >= 0) 1924 | { 1925 | cpu_set_t csmask; 1926 | CPU_ZERO(&csmask); 1927 | CPU_SET(capture_cpu, &csmask); 1928 | if (my_sched_setaffinity(gettid(), sizeof(cpu_set_t), &csmask) != 0) 1929 | { 1930 | WARN("Couldn't set processor affinity for capture thread"); 1931 | } 1932 | } 1933 | //Disable Port Scan Detection 1934 | nids_params.scan_num_hosts=0; 1935 | 1936 | 1937 | DEBUG(100,"Using LIBNIDS version %i.%i\n",NIDS_MAJOR,NIDS_MINOR); 1938 | 1939 | if (!nids_init ()) 1940 | { 1941 | DIE("libNIDS init failed: %s",nids_errbuf); 1942 | } 1943 | 1944 | 1945 | //wait for other threads to init before dropping priveledges 1946 | while(!output_thread_init) 1947 | { 1948 | usleep(conn_ring_poll_interval); 1949 | } 1950 | 1951 | while((!error_thread_init) && (error_interval > 0)) 1952 | { 1953 | usleep(conn_ring_poll_interval); 1954 | } 1955 | 1956 | while((!stats_thread_init) && (stats_interval > 0)) 1957 | { 1958 | usleep(conn_ring_poll_interval); 1959 | } 1960 | 1961 | 1962 | //do setuid here! 1963 | //Notice, this program is not designed to be safe on non-linux systems nor be safe as an setuid program. 1964 | if (user != NULL) 1965 | { 1966 | struct passwd *pw_entry = NULL; 1967 | if ((getuid() != 0) && (geteuid() != 0)) 1968 | { 1969 | DIE("You must start as root to switch users"); 1970 | } 1971 | 1972 | //look up user id and group id 1973 | if (!(pw_entry = getpwnam(user))) 1974 | { 1975 | DIE("Could not look up user_id for %s", user); 1976 | } 1977 | //look up additional groups (extra credit) 1978 | if (initgroups(user, pw_entry->pw_gid)) 1979 | { 1980 | DIE("Could not look up additional groups for %s", user); 1981 | } 1982 | //Actually setuid and setgid 1983 | if (setgid(pw_entry->pw_gid)) 1984 | { 1985 | DIE("Could not setgid for %s", user); 1986 | } 1987 | if (setuid(pw_entry->pw_uid)) 1988 | { 1989 | DIE("Could not setuid to %s", user); 1990 | } 1991 | /* Make sure both effective and real user and group were set correctly */ 1992 | if ((getuid() != pw_entry->pw_uid) || (geteuid() != pw_entry->pw_uid) || (getgid() != pw_entry->pw_gid) || (getegid() != pw_entry->pw_gid)) 1993 | { 1994 | DIE("Attempted setuid and setgid didn't work correctly\nPossibly non-Linux or program used suid (no gauruntee of security)"); 1995 | } 1996 | } 1997 | 1998 | 1999 | nids_register_tcp(tcp_callback); 2000 | program_state = 1; 2001 | nids_run(); 2002 | program_state = 2; 2003 | DEBUG(30,"Libnids Run Finished"); 2004 | 2005 | 2006 | #ifdef WITH_BSF 2007 | if (bsf_enabled) 2008 | { 2009 | bsf_destroy(bsf_desc); 2010 | } 2011 | #endif 2012 | //Wait for all the other threads to exit before exiting 2013 | //wait_for_other_threads(); 2014 | 2015 | 2016 | pthread_join(writer_thread_h, NULL); 2017 | if (error_interval > 0) 2018 | { 2019 | pthread_join(error_thread_h, NULL); 2020 | } else 2021 | { 2022 | //Provide some useful output/feedback to users. 2023 | print_errors(); 2024 | if ( errors.tcp_limit > 0) 2025 | { 2026 | WARN("Hint--TCP_LIMIT: Streams dropped due to insufficient connection hash table. Consider increasing connection hash size (-s)."); 2027 | } 2028 | if ( errors.tcp_queue > 0) 2029 | { 2030 | WARN("Hint--TCP_QUEUE: Investigate possible packet loss (if PCAP_LOSS is 0 check ifconfig for RX dropped)."); 2031 | } 2032 | if ( errors.tcp_hdr > connection_id) 2033 | { 2034 | WARN("Hint--TCP_HDR: Possible checksum failures? See disable checksum option (-k)."); 2035 | } 2036 | if ( errors.vtx_ring > 0) 2037 | { 2038 | WARN("Hint--VTX_RING: Streams dropped due to insufficent stream ring buffer. Try increasing ring size (-Q) and/or decreasing poll interval (-R) or increaseing speed of temp dir (-t) (ex. use /dev/shm instead of disk)."); 2039 | } 2040 | 2041 | } 2042 | if (stats_interval > 0) 2043 | { 2044 | pthread_join(stats_thread_h, NULL); 2045 | } else 2046 | { 2047 | //Provide some useful output/feedback to users. 2048 | print_stats(); 2049 | 2050 | if ( closed_connections_limit > 0) 2051 | { 2052 | WARN("Hint--VTX_LIMIT: Streams truncated due to size limits. If not desired, adjust stream size limits accordingly (-C, -S)."); 2053 | } 2054 | 2055 | } 2056 | 2057 | 2058 | return 0; 2059 | } 2060 | --------------------------------------------------------------------------------