├── CHANGES ├── VERSION ├── .gitignore ├── scripts ├── types.zeek ├── Zeek │ └── TsvHttp │ │ ├── __load__.zeek │ │ └── logs-to-http.zeek ├── __preload__.zeek ├── __load__.zeek └── init.zeek ├── tests ├── Makefile ├── tsvhttp │ └── show-plugin.zeek ├── random.seed ├── Scripts │ ├── diff-remove-timestamps │ └── get-zeek-env └── btest.cfg ├── README ├── src ├── tsvhttp.bif ├── Plugin.h ├── Plugin.cc ├── TsvHttp.h └── TsvHttp.cc ├── zkg.meta ├── configure.plugin ├── Makefile ├── CMakeLists.txt ├── COPYING ├── README.md └── configure /CHANGES: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 0.5 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | -------------------------------------------------------------------------------- /scripts/types.zeek: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /tests/Makefile: -------------------------------------------------------------------------------- 1 | 2 | test: 3 | @btest 4 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | This file is only here to prevent the call to ConfigurePackaging() in CMakeLists.txt from failing. 2 | -------------------------------------------------------------------------------- /src/tsvhttp.bif: -------------------------------------------------------------------------------- 1 | # Options for the TsvHttp writer. 2 | 3 | module LogTsvHttp; 4 | 5 | const url: string; 6 | -------------------------------------------------------------------------------- /tests/tsvhttp/show-plugin.zeek: -------------------------------------------------------------------------------- 1 | # @TEST-EXEC: zeek -NN Zeek::TsvHttp |sed -e 's/version.*)/version)/g' >output 2 | # @TEST-EXEC: btest-diff output 3 | -------------------------------------------------------------------------------- /tests/random.seed: -------------------------------------------------------------------------------- 1 | 2983378351 2 | 1299727368 3 | 0 4 | 310447 5 | 0 6 | 1409073626 7 | 3975311262 8 | 34130240 9 | 1450515018 10 | 1466150520 11 | 1342286698 12 | 1193956778 13 | 2188527278 14 | 3361989254 15 | 3912865238 16 | 3596260151 17 | 517973768 18 | 1462428821 19 | 0 20 | 2278350848 21 | 32767 22 | -------------------------------------------------------------------------------- /zkg.meta: -------------------------------------------------------------------------------- 1 | [package] 2 | description = A plugin that streams logs over HTTP, using the native Zeek TSV format. 3 | tags = log writer, logs, streaming, http, plugin 4 | script_dir = scripts/Zeek/TsvHttp 5 | build_command = ./configure --zeek-dist=%(zeek_dist)s && make 6 | test_command = cd tests && btest 7 | depends = 8 | zeek >=3.0.0 9 | -------------------------------------------------------------------------------- /scripts/Zeek/TsvHttp/__load__.zeek: -------------------------------------------------------------------------------- 1 | # 2 | # This is processed when a user explicitly loads the plugin's script module 3 | # through `@load /`. Include code here that 4 | # should execute at that point. This is the most common entry point to 5 | # your plugin's accompanying scripts. 6 | # 7 | 8 | @load ./logs-to-http.zeek 9 | 10 | -------------------------------------------------------------------------------- /scripts/__preload__.zeek: -------------------------------------------------------------------------------- 1 | # 2 | # This is loaded automatically at Zeek startup once the plugin gets activated, 3 | # but before any of the BiFs that the plugin defines become available. 4 | # 5 | # This is primarily for defining types that BiFs already depend on. If you 6 | # need to do any other unconditional initialization, that should go into 7 | # __load__.zeek instead. 8 | # 9 | 10 | @load ./types 11 | 12 | -------------------------------------------------------------------------------- /src/Plugin.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef BRO_PLUGIN_ZEEK_TSVHTTP 3 | #define BRO_PLUGIN_ZEEK_TSVHTTP 4 | 5 | #include "TsvHttp.h" 6 | #include 7 | 8 | namespace plugin { 9 | namespace Zeek_TsvHttp { 10 | 11 | class Plugin : public ::plugin::Plugin 12 | { 13 | protected: 14 | // Overridden from plugin::Plugin. 15 | plugin::Configuration Configure() override; 16 | }; 17 | 18 | extern Plugin plugin; 19 | 20 | } 21 | } 22 | 23 | #endif 24 | -------------------------------------------------------------------------------- /tests/Scripts/diff-remove-timestamps: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | # 3 | # Replace anything which looks like timestamps with XXXs (including the #start/end markers in logs). 4 | 5 | # Get us "modern" regexps with sed. 6 | if [ `uname` == "Linux" ]; then 7 | sed="sed -r" 8 | else 9 | sed="sed -E" 10 | fi 11 | 12 | $sed 's/(0\.000000)|([0-9]{9,10}\.[0-9]{2,8})/XXXXXXXXXX.XXXXXX/g' | \ 13 | $sed 's/^ *#(open|close).(19|20)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g' 14 | -------------------------------------------------------------------------------- /scripts/Zeek/TsvHttp/logs-to-http.zeek: -------------------------------------------------------------------------------- 1 | module LogTsvHttp; 2 | 3 | event zeek_init() &priority=-10 4 | { 5 | if ( url == "" ) 6 | return; 7 | 8 | for ( stream_id in Log::active_streams ) 9 | { 10 | if ( stream_id in exclude_logs || 11 | (|send_logs| > 0 && stream_id !in send_logs) ) 12 | next; 13 | 14 | local filter: Log::Filter = [$name = "default-http", 15 | $writer = Log::WRITER_TSVHTTP]; 16 | Log::add_filter(stream_id, filter); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /configure.plugin: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Hooks to add custom options to the configure script. 4 | # 5 | 6 | plugin_usage() 7 | { 8 | : # Do nothing 9 | # cat <//__load__.zeek. That's processed 8 | # only on explicit `@load /`. 9 | # 10 | 11 | @load ./init.zeek 12 | -------------------------------------------------------------------------------- /src/Plugin.cc: -------------------------------------------------------------------------------- 1 | 2 | #include "Plugin.h" 3 | 4 | namespace plugin { namespace Zeek_TsvHttp { Plugin plugin; } } 5 | 6 | using namespace plugin::Zeek_TsvHttp; 7 | 8 | plugin::Configuration Plugin::Configure() 9 | { 10 | AddComponent(new ::logging::Component("TsvHttp", ::logging::writer::TsvHttp::Instantiate)); 11 | 12 | plugin::Configuration config; 13 | config.name = "Zeek::TsvHttp"; 14 | config.description = "Plugin to POST Zeek logs via HTTP"; 15 | config.version.major = version_major; 16 | config.version.minor = version_minor; 17 | return config; 18 | } 19 | -------------------------------------------------------------------------------- /tests/btest.cfg: -------------------------------------------------------------------------------- 1 | [btest] 2 | TestDirs = tsvhttp 3 | TmpDir = %(testbase)s/.tmp 4 | BaselineDir = %(testbase)s/Baseline 5 | IgnoreDirs = .svn CVS .tmp 6 | IgnoreFiles = *.tmp *.swp #* *.trace .DS_Store 7 | 8 | [environment] 9 | ZEEKPATH=`%(testbase)s/Scripts/get-zeek-env zeekpath` 10 | ZEEK_PLUGIN_PATH=`%(testbase)s/Scripts/get-zeek-env zeek_plugin_path` 11 | ZEEK_SEED_FILE=%(testbase)s/random.seed 12 | PATH=`%(testbase)s/Scripts/get-zeek-env path` 13 | TZ=UTC 14 | LC_ALL=C 15 | TRACES=%(testbase)s/Traces 16 | TMPDIR=%(testbase)s/.tmp 17 | TEST_DIFF_CANONIFIER=%(testbase)s/Scripts/diff-remove-timestamps 18 | -------------------------------------------------------------------------------- /scripts/init.zeek: -------------------------------------------------------------------------------- 1 | module LogTsvHttp; 2 | 3 | export { 4 | # The endpoint to POST logs to, for example http://my.server.com:8080/zeek 5 | const url = "" &redef; 6 | 7 | # Optionally ignore any :zeek:type:`Log::ID` from being sent 8 | const exclude_logs: set[Log::ID] &redef; 9 | 10 | # If you want to explicitly only send certain :zeek:type:`Log::ID` 11 | # streams, add them to this set. If the set remains empty, all will 12 | # be sent. The :zeek:id:`LogTsvHttp::exclude_logs` option 13 | # will remain in effect as well. 14 | const send_logs: set[Log::ID] &redef; 15 | } 16 | 17 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # Convenience Makefile providing a few common top-level targets. 3 | # 4 | 5 | cmake_build_dir=build 6 | arch=`uname -s | tr A-Z a-z`-`uname -m` 7 | 8 | all: build-it 9 | 10 | build-it: 11 | @test -e $(cmake_build_dir)/config.status || ./configure 12 | -@test -e $(cmake_build_dir)/CMakeCache.txt && \ 13 | test $(cmake_build_dir)/CMakeCache.txt -ot `cat $(cmake_build_dir)/CMakeCache.txt | grep ZEEK_DIST | cut -d '=' -f 2`/build/CMakeCache.txt && \ 14 | echo Updating stale CMake cache && \ 15 | touch $(cmake_build_dir)/CMakeCache.txt 16 | 17 | ( cd $(cmake_build_dir) && make ) 18 | 19 | install: 20 | ( cd $(cmake_build_dir) && make install ) 21 | 22 | clean: 23 | ( cd $(cmake_build_dir) && make clean ) 24 | 25 | distclean: 26 | rm -rf $(cmake_build_dir) 27 | 28 | test: 29 | make -C tests 30 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | cmake_minimum_required(VERSION 3.0 FATAL_ERROR) 3 | 4 | project(ZeekPluginTsvHttp) 5 | 6 | include(ZeekPlugin) 7 | 8 | 9 | include(FindCURL) 10 | 11 | message("CURL_INCLUDE_DIRS: ${CURL_INCLUDE_DIRS}") 12 | message("CURL_LIBRARIES: ${CURL_LIBRARIES}") 13 | message("CURL_VERSION_STRING: ${CURL_VERSION_STRING}") 14 | 15 | if ( CURL_FOUND ) 16 | 17 | zeek_plugin_begin(Zeek TsvHttp) 18 | zeek_plugin_cc(src/TsvHttp.cc) 19 | zeek_plugin_cc(src/Plugin.cc) 20 | zeek_plugin_bif(src/tsvhttp.bif) 21 | zeek_plugin_dist_files(README.md CHANGES COPYING VERSION) 22 | zeek_plugin_link_library(${CURL_LIBRARIES}) 23 | zeek_plugin_end() 24 | 25 | file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" VERSION LIMIT_COUNT 1) 26 | 27 | if ("${PROJECT_SOURCE_DIR}" STREQUAL "${CMAKE_SOURCE_DIR}") 28 | # Allows building rpm/deb packages via "make package" in build dir. 29 | include(ConfigurePackaging) 30 | ConfigurePackaging(${VERSION}) 31 | endif () 32 | else () 33 | message(FATAL_ERROR "LibCURL not found.") 34 | endif () 35 | -------------------------------------------------------------------------------- /tests/Scripts/get-zeek-env: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | # 3 | # BTest helper for getting values for Zeek-related environment variables. 4 | 5 | base=`dirname $0` 6 | zeek_dist=`cat ${base}/../../build/CMakeCache.txt | grep ZEEK_DIST | cut -d = -f 2` 7 | 8 | if [ -n "${zeek_dist}" ]; then 9 | if [ "$1" = "zeekpath" ]; then 10 | ${zeek_dist}/build/zeek-path-dev 11 | elif [ "$1" = "zeek_plugin_path" ]; then 12 | ( cd ${base}/../.. && pwd ) 13 | elif [ "$1" = "path" ]; then 14 | echo ${zeek_dist}/build/src:${zeek_dist}/aux/btest:${base}/:${zeek_dist}/aux/zeek-cut:$PATH 15 | else 16 | echo "usage: `basename $0` " >&2 17 | exit 1 18 | fi 19 | else 20 | # Use Zeek installation for testing. In this case zeek-config must be in PATH. 21 | if ! which zeek-config >/dev/null; then 22 | echo "zeek-config not found" >&2 23 | exit 1 24 | fi 25 | 26 | if [ "$1" = "zeekpath" ]; then 27 | zeek-config --zeekpath 28 | elif [ "$1" = "zeek_plugin_path" ]; then 29 | ( cd ${base}/../.. && pwd ) 30 | elif [ "$1" = "path" ]; then 31 | echo ${PATH} 32 | else 33 | echo "usage: `basename $0` " >&2 34 | exit 1 35 | fi 36 | fi 37 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | Copyright (c) 2020 by Looky Labs, Inc 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are met: 5 | 6 | (1) Redistributions of source code must retain the above copyright 7 | notice, this list of conditions and the following disclaimer. 8 | 9 | (2) Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in 11 | the documentation and/or other materials provided with the 12 | distribution. 13 | 14 | (3) Neither the name of Looky Labs, nor 15 | the names of contributors may be used to endorse or promote 16 | products derived from this software without specific prior written 17 | permission. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 23 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 | POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /src/TsvHttp.h: -------------------------------------------------------------------------------- 1 | // See the file "COPYING" in the main distribution directory for copyright. 2 | 3 | #ifndef ZEEK_PLUGIN_TSVHTTP_H 4 | #define ZEEK_PLUGIN_TSVHTTP_H 5 | 6 | #include 7 | 8 | #include "logging/WriterBackend.h" 9 | #include "threading/formatters/Ascii.h" 10 | #include "zlib.h" 11 | #include "Desc.h" 12 | 13 | const int version_major=0; 14 | const int version_minor=5; 15 | 16 | namespace logging { namespace writer { 17 | 18 | class TsvHttp : public WriterBackend { 19 | public: 20 | explicit TsvHttp(WriterFrontend* frontend); 21 | ~TsvHttp() override; 22 | 23 | static WriterBackend* Instantiate(WriterFrontend* frontend) 24 | { return new TsvHttp(frontend); } 25 | 26 | protected: 27 | bool DoInit(const WriterInfo& info, int num_fields, 28 | const threading::Field* const* fields) override; 29 | bool DoWrite(int num_fields, const threading::Field* const* fields, 30 | threading::Value** vals) override; 31 | bool DoSetBuf(bool enabled) override; 32 | bool DoRotate(const char* rotated_path, double open, 33 | double close, bool terminating) override; 34 | bool DoFlush(double network_time) override; 35 | bool DoFinish(double network_time) override; 36 | bool DoHeartbeat(double network_time, double current_time) override; 37 | 38 | private: 39 | enum ConnectionState { 40 | CLOSED = 0, 41 | WAITING = 1, 42 | SENDING_HEADER = 2, 43 | SENDING_DATA = 3, 44 | FINISHING = 4 45 | }; 46 | int connstate; 47 | 48 | void InitConfigOptions(); 49 | void InitFilterOptions(); 50 | 51 | void WriteHeader(const string& path); 52 | void WriteHeaderField(const string& key, const string& value); 53 | string Timestamp(double t); // Uses current time if t is zero. 54 | bool InitFormatter(); 55 | 56 | void CurlSetopts(); 57 | bool CurlConnect(); 58 | bool CurlSendData(); 59 | bool CurlSendHeader(); 60 | static size_t InvokeReadCallback(char *buffer, size_t size, size_t nitems, void *userdata); 61 | size_t CurlReadCallback(char *buffer, size_t size, size_t nitems); 62 | void SwitchBuffers(); 63 | 64 | ODesc databuf1, databuf2, headerbuf; 65 | ODesc* write_buffer, *read_buffer; 66 | const u_char* read_ptr; 67 | unsigned int read_sizeleft; 68 | bool cb_done; 69 | 70 | CURLM* mcurl = NULL; 71 | CURL* curl = NULL; 72 | struct curl_slist *http_headers; 73 | 74 | // hardcoded defaults from Ascii logwriter 75 | const string separator = "\t"; 76 | const string set_separator = ","; 77 | const string empty_field = "(empty)"; 78 | const string unset_field = "-"; 79 | const string meta_prefix = "#"; 80 | 81 | string endpoint; 82 | string path; 83 | 84 | threading::formatter::Formatter* formatter; 85 | }; 86 | } 87 | } 88 | 89 | 90 | #endif 91 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Zeek TSV HTTP Plugin 2 | 3 | A Zeek plugin to POST logs over HTTP. The logs are posted in native 4 | [TSV ASCII format](https://docs.zeek.org/en/stable/scripts/base/frameworks/logging/writers/ascii.zeek.html). The plugin uses HTTP chunked encoding to first post 5 | the Zeek log header then it streams log lines as HTTP chunks as they 6 | become available. 7 | 8 | If the connection closes or resets, the plugin attempts to reconnect 9 | and transmit data where it left off. 10 | 11 | 12 | ## Building and Installing 13 | 14 | The plugin is known to work with with Zeek versions 3.0.0 and newer. 15 | 16 | ### Building 17 | 18 | 19 | 1. Ensure that you have libcurl present on your system (plugin has 20 | been primarily developped/tested against libcurl 7.54, but other 21 | versions should work too). 22 | 23 | 1. Build the plugin using the following commands: 24 | 25 | ```sh 26 | $ ./configure 27 | $ make 28 | ``` 29 | 30 | The configure uses `zeek-config` to determine the path to your Zeek 31 | distribution. If for some reason that is not present you can pass path 32 | via the `--zeek-dist` argument to `configure`. 33 | 34 | 35 | ### Installing 36 | 37 | **From source:** If you've built the plugin yourself following the steps above, `make 38 | install` will install the plugin. (Of course, this requires building 39 | the plugin locally to the Zeek host it will run on). 40 | 41 | 42 | **From binary release:** If you're installing a binary release of the plugin (such as 43 | `Zeek_TsvHttp-0.5.tar.gz`), then do the following after copying the 44 | package on to the Zeek host: 45 | 46 | ```sh 47 | $ sudo mkdir -p $(zeek-config --plugin_dir) 48 | $ cd $(zeek-config --plugin_dir) 49 | $ sudo tar oxzf path/to/plugin/Zeek_TsvHttp-0.5.tar.gz 50 | ``` 51 | 52 | 53 | To verify the installation, run `zeek -N Zeek::TsvHttp` and you will 54 | see the same output as below if the installation was successful. 55 | 56 | ```sh 57 | $ zeek -N Zeek::TsvHttp 58 | Zeek::TsvHttp - Plugin to POST Zeek logs via HTTP (dynamic, version 0.5) 59 | ``` 60 | 61 | ### Install with `zkg` 62 | 63 | To be documented. 64 | 65 | 66 | ## Configure and Run 67 | 68 | Add the following to the end of your `local.zeek` file: 69 | 70 | ``` 71 | @load Zeek/TsvHttp 72 | 73 | # Set this to the URL of your HTTP endpoint 74 | redef LogTsvHttp::url = "http://localhost:9867/space/default/zeek"; 75 | ``` 76 | 77 | By default, all log streams will be sent. 78 | 79 | You can redefine the `exclude_logs` and `send_logs` variables 80 | for finer-grained selection of streams to send. 81 | 82 | For example, to send only the `conn` and `dns` logs: 83 | 84 | 85 | ``` 86 | redef LogTsvHttp::send_logs = set(Conn::LOG, DNS::LOG); 87 | ``` 88 | 89 | 90 | or to send all but the `loaded_scripts` log: 91 | ``` 92 | redef LogTsvHttp::exclude_logs = set(LoadedScripts::LOG); 93 | ``` 94 | 95 | #### Sending logs to different endpoints 96 | 97 | The `LogTsvHttp::url` endpoint can be overridden on a per-log basis 98 | by instantiating a `Log::Filter` and passing the URL in its 99 | configuration table. For example: 100 | 101 | ``` 102 | @load Zeek/TsvHttp 103 | 104 | # Set this to the URL of your HTTP endpoint 105 | redef LogTsvHttp::url = "http://localhost:9867/some/endpoint"; 106 | 107 | event zeek_init() &priority=-10 108 | { 109 | # handles HTTP 110 | local http_filter: Log::Filter = [ 111 | $name = tsv-http", 112 | $writer = Log::WRITER_TSVHTTP, 113 | $path = "http", 114 | $config = table(["url"] = "http://localhost:9877/other/endpoint") 115 | ]; 116 | Log::add_filter(HTTP::LOG, http_filter); 117 | 118 | # handles DNS 119 | local dns_filter: Log::Filter = [ 120 | $name = "tsv-dns", 121 | $writer = Log::WRITER_TSVHTTP, 122 | $path = "dns", 123 | $config = table(["url"] = "http://localhost:9887/and/another/endpoint") 124 | ]; 125 | Log::add_filter(DNS::LOG, dns_filter); 126 | } 127 | ``` 128 | -------------------------------------------------------------------------------- /configure: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Wrapper for viewing/setting options that the plugin's CMake 4 | # scripts will recognize. 5 | # 6 | # Don't edit this. Edit configure.plugin to add plugin-specific options. 7 | # 8 | 9 | set -e 10 | command="$0 $*" 11 | 12 | if [ -e `dirname $0`/configure.plugin ]; then 13 | # Include custom additions. 14 | . `dirname $0`/configure.plugin 15 | fi 16 | 17 | usage() { 18 | 19 | cat 1>&2 </dev/null 2>&1; then 34 | plugin_usage 1>&2 35 | fi 36 | 37 | echo 38 | 39 | exit 1 40 | } 41 | 42 | # Function to append a CMake cache entry definition to the 43 | # CMakeCacheEntries variable 44 | # $1 is the cache entry variable name 45 | # $2 is the cache entry variable type 46 | # $3 is the cache entry variable value 47 | append_cache_entry () { 48 | CMakeCacheEntries="$CMakeCacheEntries -D $1:$2=$3" 49 | } 50 | 51 | # set defaults 52 | builddir=build 53 | zeekdist="" 54 | installroot="default" 55 | CMakeCacheEntries="" 56 | 57 | while [ $# -ne 0 ]; do 58 | case "$1" in 59 | -*=*) optarg=`echo "$1" | sed 's/[-_a-zA-Z0-9]*=//'` ;; 60 | *) optarg= ;; 61 | esac 62 | 63 | case "$1" in 64 | --help|-h) 65 | usage 66 | ;; 67 | 68 | --cmake=*) 69 | CMakeCommand=$optarg 70 | ;; 71 | 72 | --zeek-dist=*) 73 | zeekdist=`cd $optarg && pwd` 74 | ;; 75 | 76 | --install-root=*) 77 | installroot=$optarg 78 | ;; 79 | 80 | --with-binpac=*) 81 | append_cache_entry BinPAC_ROOT_DIR PATH $optarg 82 | binpac_root=$optarg 83 | ;; 84 | 85 | --with-broker=*) 86 | append_cache_entry BROKER_ROOT_DIR PATH $optarg 87 | broker_root=$optarg 88 | ;; 89 | 90 | --with-caf=*) 91 | append_cache_entry CAF_ROOT_DIR PATH $optarg 92 | caf_root=$optarg 93 | ;; 94 | 95 | --with-bifcl=*) 96 | append_cache_entry BifCl_EXE PATH $optarg 97 | ;; 98 | 99 | --enable-debug) 100 | append_cache_entry BRO_PLUGIN_ENABLE_DEBUG BOOL true 101 | ;; 102 | 103 | *) 104 | if type plugin_option >/dev/null 2>&1; then 105 | plugin_option $1 && shift && continue; 106 | fi 107 | 108 | echo "Invalid option '$1'. Try $0 --help to see available options." 109 | exit 1 110 | ;; 111 | esac 112 | shift 113 | done 114 | 115 | if [ -z "$CMakeCommand" ]; then 116 | # prefer cmake3 over "regular" cmake (cmake == cmake2 on RHEL) 117 | if command -v cmake3 >/dev/null 2>&1 ; then 118 | CMakeCommand="cmake3" 119 | elif command -v cmake >/dev/null 2>&1 ; then 120 | CMakeCommand="cmake" 121 | else 122 | echo "This package requires CMake, please install it first." 123 | echo "Then you may use this script to configure the CMake build." 124 | echo "Note: pass --cmake=PATH to use cmake in non-standard locations." 125 | exit 1; 126 | fi 127 | fi 128 | 129 | if [ -z "$zeekdist" ]; then 130 | if type zeek-config >/dev/null 2>&1; then 131 | zeek_config="zeek-config" 132 | else 133 | echo "Either 'zeek-config' must be in PATH or '--zeek-dist=' used" 134 | exit 1 135 | fi 136 | 137 | append_cache_entry BRO_CONFIG_PREFIX PATH `${zeek_config} --prefix` 138 | append_cache_entry BRO_CONFIG_INCLUDE_DIR PATH `${zeek_config} --include_dir` 139 | append_cache_entry BRO_CONFIG_PLUGIN_DIR PATH `${zeek_config} --plugin_dir` 140 | append_cache_entry BRO_CONFIG_CMAKE_DIR PATH `${zeek_config} --cmake_dir` 141 | append_cache_entry CMAKE_MODULE_PATH PATH `${zeek_config} --cmake_dir` 142 | 143 | build_type=`${zeek_config} --build_type` 144 | 145 | if [ "$build_type" = "debug" ]; then 146 | append_cache_entry BRO_PLUGIN_ENABLE_DEBUG BOOL true 147 | fi 148 | 149 | if [ -z "$binpac_root" ]; then 150 | append_cache_entry BinPAC_ROOT_DIR PATH `${zeek_config} --binpac_root` 151 | fi 152 | 153 | if [ -z "$broker_root" ]; then 154 | append_cache_entry BROKER_ROOT_DIR PATH `${zeek_config} --broker_root` 155 | fi 156 | 157 | if [ -z "$caf_root" ]; then 158 | append_cache_entry CAF_ROOT_DIR PATH `${zeek_config} --caf_root` 159 | fi 160 | else 161 | if [ ! -e "$zeekdist/zeek-path-dev.in" ]; then 162 | echo "$zeekdist does not appear to be a valid Zeek source tree." 163 | exit 1 164 | fi 165 | 166 | # BRO_DIST is the canonical/historical name used by plugin CMake scripts 167 | # ZEEK_DIST doesn't serve a function at the moment, but set/provided anyway 168 | append_cache_entry BRO_DIST PATH $zeekdist 169 | append_cache_entry ZEEK_DIST PATH $zeekdist 170 | append_cache_entry CMAKE_MODULE_PATH PATH $zeekdist/cmake 171 | fi 172 | 173 | if [ "$installroot" != "default" ]; then 174 | mkdir -p $installroot 175 | append_cache_entry BRO_PLUGIN_INSTALL_ROOT PATH $installroot 176 | fi 177 | 178 | echo "Build Directory : $builddir" 179 | echo "Zeek Source Directory : $zeekdist" 180 | 181 | mkdir -p $builddir 182 | cd $builddir 183 | 184 | "$CMakeCommand" $CMakeCacheEntries .. 185 | 186 | echo "# This is the command used to configure this build" > config.status 187 | echo $command >> config.status 188 | chmod u+x config.status 189 | -------------------------------------------------------------------------------- /src/TsvHttp.cc: -------------------------------------------------------------------------------- 1 | // See the file "COPYING" in the main distribution directory for copyright. 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include "threading/SerialTypes.h" 8 | 9 | #include 10 | #include 11 | 12 | #include "Plugin.h" 13 | #include "TsvHttp.h" 14 | #include "tsvhttp.bif.h" 15 | 16 | using namespace logging::writer; 17 | using namespace threading; 18 | using threading::Value; 19 | using threading::Field; 20 | 21 | 22 | size_t TsvHttp::InvokeReadCallback(char *buffer, size_t size, size_t nitems, void *userdata) { 23 | 24 | return ((TsvHttp*)userdata)->CurlReadCallback(buffer, size, nitems); 25 | } 26 | 27 | 28 | TsvHttp::TsvHttp(WriterFrontend* frontend) : WriterBackend(frontend) 29 | { 30 | databuf1.Clear(); 31 | databuf2.Clear(); 32 | headerbuf.Clear(); 33 | connstate = CLOSED; 34 | InitConfigOptions(); 35 | InitFilterOptions(); 36 | } 37 | 38 | void TsvHttp::InitConfigOptions() 39 | { 40 | endpoint = BifConst::LogTsvHttp::url->AsString()->CheckString(); 41 | } 42 | 43 | void TsvHttp::InitFilterOptions() 44 | { 45 | const WriterInfo& info = Info(); 46 | 47 | for ( WriterInfo::config_map::const_iterator i = info.config.begin(); i != info.config.end(); ++i ) { 48 | if ( strcmp(i->first, "url") == 0 ) { 49 | endpoint = i->second; 50 | } 51 | } 52 | } 53 | 54 | bool TsvHttp::InitFormatter() 55 | { 56 | 57 | formatter = 0; 58 | delete formatter; 59 | 60 | // Use the default "Zeek logs" format. 61 | databuf1.EnableEscaping(); 62 | databuf1.AddEscapeSequence(separator); 63 | databuf2.EnableEscaping(); 64 | databuf2.AddEscapeSequence(separator); 65 | headerbuf.EnableEscaping(); 66 | headerbuf.AddEscapeSequence(separator); 67 | 68 | formatter::Ascii::SeparatorInfo sep_info(separator, set_separator, unset_field, empty_field); 69 | formatter = new formatter::Ascii(this, sep_info); 70 | 71 | return true; 72 | } 73 | 74 | TsvHttp::~TsvHttp() 75 | { 76 | delete formatter; 77 | } 78 | 79 | void TsvHttp::WriteHeaderField(const string& key, const string& val) 80 | { 81 | string str = meta_prefix + key + separator + val + "\n"; 82 | headerbuf.AddRaw(str); 83 | } 84 | 85 | 86 | 87 | 88 | bool TsvHttp::CurlConnect() 89 | { 90 | if ( curl ) { 91 | CURLMcode mres = curl_multi_remove_handle(mcurl, curl); 92 | if (mres != CURLM_OK) { 93 | Error(Fmt("curl_multi_add_handle() failed: %s", curl_multi_strerror(mres))); 94 | return false; 95 | } 96 | curl_easy_cleanup(curl); 97 | } 98 | 99 | curl = curl_easy_init(); 100 | if ( ! curl ) { 101 | Error("curl_easy_init() failed"); 102 | return false; 103 | } 104 | 105 | CurlSetopts(); 106 | 107 | CURLMcode mres = curl_multi_add_handle(mcurl, curl); 108 | if ( mres != CURLM_OK) { 109 | Error(Fmt("curl_multi_add_handle() failed: %s", curl_multi_strerror(mres))); 110 | return false; 111 | } 112 | 113 | connstate=SENDING_HEADER; 114 | if (!CurlSendHeader()) { 115 | connstate=WAITING; 116 | return false; 117 | } 118 | connstate=SENDING_DATA; 119 | return true; 120 | } 121 | 122 | void TsvHttp::CurlSetopts() 123 | { 124 | 125 | 126 | PLUGIN_DBG_LOG(plugin::Zeek_TsvHttp::plugin, "Endpoint: %s", endpoint.c_str()); 127 | curl_easy_setopt(curl, CURLOPT_URL, endpoint.c_str()); 128 | 129 | curl_easy_setopt(curl, CURLOPT_POST, 1L); 130 | curl_easy_setopt(curl, CURLOPT_READFUNCTION, TsvHttp::InvokeReadCallback); 131 | curl_easy_setopt(curl, CURLOPT_READDATA, this); 132 | // curl_easy_setopt(curl, CURLOPT_UPLOAD_BUFFERSIZE, 16384L); commented pending selecting min version in cmakelists 133 | curl_easy_setopt(curl, CURLOPT_HTTPHEADER, http_headers); 134 | 135 | // will probably want to set these too (like ES plugin) 136 | // curl_easy_setopt(handle, CURLOPT_NOSIGNAL, 1); 137 | // curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, transfer_timeout); 138 | // curl_easy_setopt(handle, CURLOPT_TIMEOUT, transfer_timeout); 139 | // curl_easy_setopt(handle, CURLOPT_DNS_CACHE_TIMEOUT, 60*60); 140 | 141 | } 142 | 143 | 144 | size_t TsvHttp::CurlReadCallback(char *dest, size_t size, size_t nitems) 145 | { 146 | size_t n_read = 0; 147 | size_t max_read = size * nitems; 148 | cb_done = true; 149 | 150 | if (connstate == FINISHING) { 151 | return 0; 152 | } 153 | 154 | if (!read_sizeleft) { 155 | Error(Fmt("CurlReadCallback called with read_sizeleft=0")); 156 | return 0; 157 | } 158 | 159 | /* copy as much as possible from the source to the destination */ 160 | n_read = read_sizeleft; 161 | if(n_read > max_read) 162 | n_read = max_read; 163 | memcpy(dest, read_ptr, n_read); 164 | 165 | read_ptr += n_read; 166 | read_sizeleft -= n_read; 167 | PLUGIN_DBG_LOG(plugin::Zeek_TsvHttp::plugin, "TsvHttp::CurlReadCallback read %zu bytes, %d bytes left", n_read, read_sizeleft); 168 | return n_read; 169 | } 170 | 171 | void TsvHttp::SwitchBuffers() 172 | { 173 | ODesc* tmp = read_buffer; 174 | read_buffer = write_buffer; 175 | write_buffer = tmp; 176 | 177 | read_sizeleft = read_buffer->Len(); 178 | read_ptr = read_buffer->Bytes(); 179 | } 180 | 181 | bool TsvHttp::DoInit(const WriterInfo& info, int num_fields, const Field* const * fields) 182 | { 183 | 184 | InitFormatter(); 185 | 186 | read_buffer = &databuf1; 187 | write_buffer = &databuf2; 188 | read_sizeleft = 0; 189 | 190 | // xxx not thread safe 191 | CURLcode res = curl_global_init(CURL_GLOBAL_NOTHING); // will likely want to init SSL here at some point 192 | 193 | if(res != CURLE_OK) { 194 | PLUGIN_DBG_LOG(plugin::Zeek_TsvHttp::plugin, "curl_global_init() failed: %s", curl_easy_strerror(res)); 195 | return false; 196 | } 197 | 198 | http_headers = curl_slist_append(NULL, "Transfer-Encoding: chunked"); 199 | if (http_headers == NULL) { 200 | Error("curl_slist_append() failed, TsvHttp plugin not starting"); 201 | return false; 202 | } 203 | 204 | // Add an empty Expect header to prevent libcurl from 205 | // automatically adding "Expect: 100-continue", as it does with 206 | // chunked transfers 207 | http_headers = curl_slist_append(http_headers, "Expect:"); 208 | if (http_headers == NULL) { 209 | Error("curl_slist_append() failed, TsvHttp plugin not starting"); 210 | return false; 211 | } 212 | 213 | mcurl = curl_multi_init(); 214 | if ( ! mcurl ) { 215 | Error("curl_multi_init() failed"); 216 | return false; 217 | } 218 | 219 | path = info.path; 220 | WriteHeader(path); 221 | 222 | CurlConnect(); 223 | 224 | Info(Fmt("running version %d.%d", version_major, version_minor)); 225 | return true; 226 | } 227 | 228 | bool TsvHttp::CurlSendData() { 229 | int running_handles=-1; 230 | 231 | if (connstate != SENDING_DATA) { 232 | Error(Fmt("Unexpected connstate %d when sending data", connstate)); 233 | return false; 234 | } 235 | 236 | while (read_sizeleft) { 237 | // Each invocation of curl_multi_perform leads to a read 238 | // callback, as long as the underlying HTTP connection is 239 | // available. We repeat the loop until our read buffer 240 | // drains, unless we don't get a callback, indicating that the 241 | // underlying connection is unavailable, in which case we drop 242 | // the current buffer and return. 243 | // (It might be unavailable because we are still connecting, 244 | // or because the underlying socket write buffer is full). 245 | cb_done=false; 246 | CURLMcode mres = curl_multi_perform(mcurl, &running_handles); 247 | PLUGIN_DBG_LOG(plugin::Zeek_TsvHttp::plugin, "curl_multi_perform done, running handles %d", running_handles); 248 | if(mres != CURLM_OK) { 249 | Error(Fmt("curl_multi_perform() failed: %s", curl_multi_strerror(mres))); 250 | return false; 251 | } 252 | 253 | if (running_handles == 0) { 254 | if (read_sizeleft) 255 | Warning(Fmt("dropping %d bytes due to transfer interruption\n", read_sizeleft)); 256 | 257 | read_sizeleft = 0; 258 | read_buffer->Clear(); 259 | connstate=CLOSED; 260 | return true; 261 | } 262 | 263 | if (!cb_done) { 264 | Warning(Fmt("dropping %d bytes due to write buffer full\n", read_sizeleft)); 265 | read_sizeleft=0; 266 | read_buffer->Clear(); 267 | return true; 268 | } 269 | 270 | } 271 | 272 | PLUGIN_DBG_LOG(plugin::Zeek_TsvHttp::plugin, "Read buffer drained"); 273 | read_buffer->Clear(); 274 | 275 | return true; 276 | } 277 | 278 | bool TsvHttp::CurlSendHeader() { 279 | int running_handles=-1; 280 | read_sizeleft = headerbuf.Len(); 281 | read_ptr = headerbuf.Bytes(); 282 | 283 | // TODO: We should probably do a curl_multi_wait before this loop, to 284 | // avoid busy-looping during connection setup. 285 | while (read_sizeleft) { 286 | CURLMcode mres = curl_multi_perform(mcurl, &running_handles); 287 | PLUGIN_DBG_LOG(plugin::Zeek_TsvHttp::plugin, "curl_multi_perform done, running handles %d", running_handles); 288 | if(mres != CURLM_OK) { 289 | Error(Fmt("curl_multi_perform() failed: %s", curl_multi_strerror(mres))); 290 | read_sizeleft = 0; 291 | return false; 292 | } 293 | 294 | if (running_handles == 0) { 295 | Info(Fmt("No connection or transfer interrupted while sending header. Will try to reconnect later.")); 296 | read_sizeleft = 0; 297 | return false; 298 | } 299 | } 300 | read_sizeleft = 0; 301 | long response_code; 302 | curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); 303 | Warning(Fmt("response code %ld\n", response_code)); 304 | 305 | return true; 306 | } 307 | 308 | void TsvHttp::WriteHeader(const string& path) 309 | { 310 | 311 | string names; 312 | string types; 313 | 314 | for ( int i = 0; i < NumFields(); ++i ) 315 | { 316 | if ( i > 0 ) 317 | { 318 | names += separator; 319 | types += separator; 320 | } 321 | 322 | names += string(Fields()[i]->name); 323 | types += Fields()[i]->TypeName().c_str(); 324 | } 325 | 326 | string str = meta_prefix 327 | + "separator " // Always use space as separator here. 328 | + get_escaped_string(separator, false) 329 | + "\n"; 330 | 331 | headerbuf.AddRaw(str); 332 | 333 | WriteHeaderField("set_separator", get_escaped_string(set_separator, false)); 334 | WriteHeaderField("empty_field", get_escaped_string(empty_field, false)); 335 | WriteHeaderField("unset_field", get_escaped_string(unset_field, false)); 336 | WriteHeaderField("path", get_escaped_string(path, false)); 337 | WriteHeaderField("open", Timestamp(0)); 338 | 339 | WriteHeaderField("fields", names); 340 | WriteHeaderField("types", types); 341 | } 342 | 343 | /** 344 | * Writer-specific method implementing flushing of its output. A writer 345 | * implementation must override this method but it can just 346 | * ignore calls if flushing doesn't align with its semantics. 347 | */ 348 | bool TsvHttp::DoFlush(double network_time) 349 | { 350 | if (connstate == SENDING_DATA) { 351 | SwitchBuffers(); 352 | CurlSendData(); 353 | } 354 | return true; 355 | } 356 | 357 | /** 358 | * Writer-specific method called just before the threading system is 359 | * going to shutdown. It is assumed that once this messages returns, 360 | * the thread can be safely terminated. 361 | */ 362 | bool TsvHttp::DoFinish(double network_time) 363 | { 364 | int running_handles=-1; 365 | Info("DoFinish"); 366 | DoFlush(network_time); 367 | connstate = FINISHING; 368 | // callback will return 0, signaling curl library to send final zero-length chunk. 369 | CURLMcode mres = curl_multi_perform(mcurl, &running_handles); 370 | curl_slist_free_all(http_headers); 371 | 372 | // cleanup order as per https://curl.haxx.se/libcurl/c/curl_multi_cleanup.html 373 | curl_multi_remove_handle(mcurl, curl); 374 | curl_easy_cleanup(curl); 375 | curl_multi_cleanup(mcurl); 376 | 377 | return true; 378 | } 379 | 380 | /** 381 | * Writer-specific output method implementing recording of one log 382 | * entry. 383 | */ 384 | bool TsvHttp::DoWrite(int num_fields, const Field* const * fields, 385 | Value** vals) 386 | { 387 | PLUGIN_DBG_LOG(plugin::Zeek_TsvHttp::plugin, "TsvHttp::DoWrite()"); 388 | 389 | if (connstate == SENDING_HEADER || connstate == FINISHING) { 390 | Error(Fmt("Unexpected connstate %d in DoWrite", connstate)); 391 | return false; 392 | } 393 | if (connstate == CLOSED) { 394 | if (!CurlConnect()) { 395 | return true; 396 | } 397 | } 398 | 399 | // connstate is either WAITING or SENDING_DATA at this point. 400 | if (write_buffer->Len() > (1 << 12)) { 401 | if (read_sizeleft) { 402 | Warning(Fmt("dropping data in DoWrite(): write buffer full (size %d) before read buffer drained (size %d, still %d left)", 403 | write_buffer->Len(), read_buffer->Len(), read_sizeleft)); 404 | return true; 405 | } 406 | if (connstate == WAITING) { 407 | // don't log anything here because we would log too much when disconnected (once per event) 408 | return true; 409 | } 410 | 411 | PLUGIN_DBG_LOG(plugin::Zeek_TsvHttp::plugin, "TsvHttp::DoWrite() write buffer reached limit, sending"); 412 | 413 | SwitchBuffers(); 414 | 415 | // if this fails and we return false, the plugin is disabled (and will be eventually discarded). 416 | if (!CurlSendData()) 417 | return false; 418 | } 419 | 420 | if ( ! formatter->Describe(write_buffer, num_fields, fields, vals) ) 421 | return false; 422 | write_buffer->AddRaw("\n", 1); 423 | 424 | return true; 425 | } 426 | 427 | /** 428 | * Writer-specific method implementing log rotation. Most directly 429 | * this only applies to writers writing into files, which should then 430 | * close the current file and open a new one. However, a writer may 431 | * also trigger other apppropiate actions if semantics are similar. 432 | * Once rotation has finished, the implementation *must* call 433 | * FinishedRotation() to signal the log manager that potential 434 | * postprocessors can now run. 435 | */ 436 | bool TsvHttp::DoRotate(const char* rotated_path, double open, double close, bool terminating) 437 | { 438 | // this is no-op as we're not a file writer 439 | return FinishedRotation(); 440 | } 441 | 442 | /** 443 | * Writer-specific method implementing a change of fthe buffering 444 | * state. If buffering is disabled, the writer should attempt to 445 | * write out information as quickly as possible even if doing so may 446 | * have a performance impact. If enabled (which is the default), it 447 | * may buffer data as helpful and write it out later in a way 448 | * optimized for performance. The current buffering state can be 449 | * queried via IsBuf(). 450 | */ 451 | bool TsvHttp::DoSetBuf(bool enabled) 452 | { 453 | // Nothing to do. 454 | return true; 455 | } 456 | 457 | /** 458 | * Triggered by regular heartbeat messages from the main thread. 459 | */ 460 | bool TsvHttp::DoHeartbeat(double network_time, double current_time) 461 | { 462 | if (connstate == WAITING) { 463 | connstate = CLOSED; 464 | } else if (connstate == SENDING_DATA) { 465 | SwitchBuffers(); 466 | CurlSendData(); 467 | } 468 | return true; 469 | } 470 | 471 | 472 | string TsvHttp::Timestamp(double t) 473 | { 474 | time_t teatime = time_t(t); 475 | 476 | if ( ! teatime ) 477 | { 478 | // Use wall clock. 479 | struct timeval tv; 480 | if ( gettimeofday(&tv, 0) < 0 ) 481 | Error("gettimeofday failed"); 482 | else 483 | teatime = tv.tv_sec; 484 | } 485 | 486 | struct tm tmbuf; 487 | struct tm* tm = localtime_r(&teatime, &tmbuf); 488 | 489 | char tmp[128]; 490 | const char* const date_fmt = "%Y-%m-%d-%H-%M-%S"; 491 | strftime(tmp, sizeof(tmp), date_fmt, tm); 492 | 493 | return tmp; 494 | } 495 | 496 | 497 | --------------------------------------------------------------------------------