├── .github └── workflows │ ├── zeek-matrix.yml │ └── zeek-nightly.yml ├── LICENSE ├── README.md ├── scripts ├── __load__.zeek └── main.zeek ├── testing ├── .gitignore ├── Baseline │ └── tests.logs-content │ │ ├── output.conn │ │ ├── output.files │ │ ├── output.http │ │ └── output.packet_filter ├── Files │ └── random.seed ├── Makefile ├── Scripts │ ├── README │ └── get-zeek-env ├── Traces │ └── http.pcap ├── btest.cfg └── tests │ ├── logs-content.zeek │ ├── logs-disable-default.zeek │ ├── logs-present.zeek │ └── logs-uncompressed.zeek └── zkg.meta /.github/workflows/zeek-matrix.yml: -------------------------------------------------------------------------------- 1 | name: Zeek matrix tests 2 | 3 | on: 4 | push: 5 | pull_request: 6 | 7 | jobs: 8 | test: 9 | name: test-${{ matrix.zeekver }} 10 | runs-on: ubuntu-20.04 11 | strategy: 12 | fail-fast: false 13 | matrix: 14 | zeekver: [zeek, zeek-lts, zeek-nightly] 15 | steps: 16 | - uses: actions/checkout@v3 17 | - uses: zeek/action-zkg-install@v2 18 | with: 19 | zeek_version: ${{ matrix.zeekver }} 20 | - uses: actions/upload-artifact@v3 21 | if: failure() 22 | with: 23 | name: zkg-logs-${{ matrix.zeekver }} 24 | path: ${{ github.workspace }}/.action-zkg-install/artifacts 25 | -------------------------------------------------------------------------------- /.github/workflows/zeek-nightly.yml: -------------------------------------------------------------------------------- 1 | name: Zeek nightly build 2 | 3 | on: 4 | schedule: 5 | - cron: 0 0 * * * 6 | 7 | jobs: 8 | test-nightly: 9 | runs-on: ubuntu-20.04 10 | steps: 11 | - uses: zeek/action-zkg-install@v2 12 | with: 13 | pkg: ${{ github.server_url }}/${{ github.repository }} 14 | zeek_version: zeek-nightly 15 | - uses: actions/upload-artifact@v3 16 | if: failure() 17 | with: 18 | name: zkg-logs 19 | path: ${{ github.workspace }}/.action-zkg-install/artifacts 20 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2018-2023, Corelight, Inc 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 2. Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 14 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 15 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 16 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 17 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 18 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 19 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 20 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 21 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 22 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | JSON Streaming Logs 2 | ------------------- 3 | 4 | This packages makes Zeek write out logs in such a way that it makes life easier 5 | for external log shippers such as `filebeats`, `logstash`, and `splunk_forwarder`. 6 | 7 | The data is structed as JSON with "extension" fields to indicate the time the 8 | log line was written (`_write_ts`) and log type such as `http` or `conn` in a 9 | field named `_path`. Files are rotated in the current log directory without 10 | being compressed so that any log shipper has a while to catch up before the 11 | log file is deleted. 12 | 13 | Logs are named in such a way that a glob in your log shipper configuration 14 | should be able to easily match all of the logs. Each log will have a prefix of 15 | `json_streaming_` so that the http log would have the full name of 16 | `json_streaming_http.log`. 17 | 18 | Loading this script also doesn't impact any other existing logs that Zeek 19 | outputs. If you would like to disable other log output from Zeek, you can change 20 | the `JSONStreaming::disable_default_logs` variable to `T`to disable all of the default 21 | logs. The only potential issue is that your logs will become completely 22 | ephemeral with this change because no logs will be rotated into local storage. 23 | 24 | Contact 25 | ------- 26 | 27 | Please reach out to if you have issues with this script 28 | or if you have thoughts on ways this could better fit into your environment. 29 | -------------------------------------------------------------------------------- /scripts/__load__.zeek: -------------------------------------------------------------------------------- 1 | @load ./main 2 | -------------------------------------------------------------------------------- /scripts/main.zeek: -------------------------------------------------------------------------------- 1 | 2 | module JSONStreaming; 3 | 4 | export { 5 | ## An optional system name for the extension fields if you would 6 | ## like to provide it. 7 | option JSONStreaming::system_name = ""; 8 | 9 | ## If you would like to disable your default logs and only log the 10 | ## "JSON streaming" format of logs set this to `T`. By default this setting 11 | ## will continue logging your logs in whatever format you specified 12 | ## and also log them with the "json_streaming_" prefix and all of the 13 | ## associated settings. 14 | const JSONStreaming::disable_default_logs = F &redef; 15 | 16 | ## If you would like to disable rotation of the "JSON streaming" output log 17 | ## files entirely, set this to `F`. 18 | const JSONStreaming::enable_log_rotation = T &redef; 19 | 20 | ## If rotation is enabled, this is the number of extra files that Zeek will 21 | ## leave laying around so that any process watching the inode can finish. 22 | ## The files will be named with the following scheme: `json_streaming_..log`. 23 | ## So, the first conn log would be named: `json_streaming_conn.1.log`. 24 | const JSONStreaming::extra_files: int = 4 &redef; 25 | 26 | ## If rotation is enabled, this is the rotation interval specifically for the 27 | ## JSON streaming logs. This is set separately since these logs are ephemeral 28 | ## and meant to be immediately carried off to some other storage and search system. 29 | const JSONStreaming::rotation_interval = 15mins &redef; 30 | } 31 | 32 | type JsonStreamingExtension: record { 33 | ## The log stream that this log was written to. 34 | path: string &log; 35 | ## Optionally log a name for the system this log entry came from. 36 | system_name: string &log &optional; 37 | ## Timestamp when the log was written. This is a 38 | ## timestamp as given by most other software. Any 39 | ## other log-specific fields will still be written. 40 | write_ts: time &log; 41 | }; 42 | 43 | function add_json_streaming_log_extension(path: string): JsonStreamingExtension 44 | { 45 | local e = JsonStreamingExtension($path = sub(path, /^json_streaming_/, ""), 46 | $write_ts = network_time()); 47 | 48 | if ( system_name != "" ) 49 | e$system_name = system_name; 50 | 51 | return e; 52 | } 53 | 54 | # We get the log suffix just to be safe. 55 | global log_suffix = getenv("ZEEK_LOG_SUFFIX") == "" ? "log" : getenv("ZEEK_LOG_SUFFIX"); 56 | 57 | function rotate_logs(info: Log::RotationInfo): bool 58 | { 59 | local i = extra_files-1; 60 | while ( i > 0 ) 61 | { 62 | if ( file_size(info$path + "." + cat(i) + "." + log_suffix) >= 0 ) 63 | { 64 | rename(info$path + "." + cat(i) + "." + log_suffix, 65 | info$path + "." + cat(i+1) + "." + log_suffix); 66 | } 67 | --i; 68 | } 69 | 70 | if ( extra_files > 0 ) 71 | { 72 | rename(info$fname, info$path + ".1.log"); 73 | } 74 | else 75 | { 76 | # If no extra files are desired, just remove this file. 77 | unlink(info$fname); 78 | } 79 | 80 | return T; 81 | } 82 | 83 | event zeek_init() &priority=-5 84 | { 85 | local new_filters: set[Log::ID, Log::Filter] = set(); 86 | local filt: Log::Filter; 87 | 88 | for ( stream in Log::active_streams ) 89 | { 90 | for ( filter_name in Log::get_filter_names(stream) ) 91 | { 92 | # This is here because we're modifying the list of filters right now... 93 | if ( /-json-streaming$/ in filter_name ) 94 | next; 95 | 96 | filt = Log::get_filter(stream, filter_name); 97 | 98 | if ( JSONStreaming::disable_default_logs && filter_name == "default" ) 99 | filt$name = "default"; 100 | else 101 | filt$name = filter_name + "-json-streaming"; 102 | 103 | if ( filt?$path ) 104 | filt$path = "json_streaming_" + filt$path; 105 | else if ( filt?$path_func ) 106 | filt$path = "json_streaming_" + filt$path_func(stream, "", []); 107 | 108 | filt$writer = Log::WRITER_ASCII; 109 | 110 | if ( JSONStreaming::enable_log_rotation ) 111 | { 112 | filt$postprocessor = rotate_logs; 113 | filt$interv = rotation_interval; 114 | } 115 | 116 | filt$ext_func = add_json_streaming_log_extension; 117 | filt$ext_prefix = "_"; 118 | 119 | # This works around a bug in Zeek's base logging script 120 | # that sets the default value to an incompatible type. 121 | # It only affects Zeek versions 3.0 and older, but we 122 | # don't have good ways to do version-checking for these, 123 | # so just leave the fix in place. 124 | if ( |filt$config| == 0 ) 125 | filt$config = table_string_of_string(); 126 | 127 | filt$config["use_json"] = "T"; 128 | filt$config["json_timestamps"] = "JSON::TS_ISO8601"; 129 | # Ensure compressed logs are disabled. 130 | filt$config["gzip_level"] = "0"; 131 | 132 | add new_filters[stream, filt]; 133 | } 134 | } 135 | 136 | # Add the filters separately to avoid problems with modifying a set/table 137 | # while it's being iterated over. 138 | for ( [stream, filt] in new_filters ) 139 | { 140 | Log::add_filter(stream, filt); 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /testing/.gitignore: -------------------------------------------------------------------------------- 1 | .btest.failed.dat 2 | .tmp 3 | -------------------------------------------------------------------------------- /testing/Baseline/tests.logs-content/output.conn: -------------------------------------------------------------------------------- 1 | ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. 2 | { 3 | "_path": "conn", 4 | "_system_name": "testsystem", 5 | "_write_ts": "2015-10-16T13:05:35.778708Z", 6 | "id.orig_h": "10.1.9.63", 7 | "id.orig_p": 63526, 8 | "id.resp_h": "54.175.222.246", 9 | "id.resp_p": 80, 10 | "proto": "tcp", 11 | "service": "http" 12 | } 13 | -------------------------------------------------------------------------------- /testing/Baseline/tests.logs-content/output.files: -------------------------------------------------------------------------------- 1 | ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. 2 | { 3 | "_path": "files", 4 | "_system_name": "testsystem", 5 | "_write_ts": "2015-10-16T13:05:35.698604Z", 6 | "filename": "test.json", 7 | "mime_type": "text/json", 8 | "source": "HTTP" 9 | } 10 | -------------------------------------------------------------------------------- /testing/Baseline/tests.logs-content/output.http: -------------------------------------------------------------------------------- 1 | ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. 2 | { 3 | "_path": "http", 4 | "_system_name": "testsystem", 5 | "_write_ts": "2015-10-16T13:05:35.698604Z", 6 | "host": "httpbin.org", 7 | "method": "GET", 8 | "user_agent": "curl/7.45.0" 9 | } 10 | -------------------------------------------------------------------------------- /testing/Baseline/tests.logs-content/output.packet_filter: -------------------------------------------------------------------------------- 1 | ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. 2 | { 3 | "_path": "packet_filter", 4 | "_system_name": "testsystem", 5 | "_write_ts": "1970-01-01T00:00:00.000000Z", 6 | "filter": "ip or not ip", 7 | "node": "zeek" 8 | } 9 | -------------------------------------------------------------------------------- /testing/Files/random.seed: -------------------------------------------------------------------------------- 1 | 2983378351 2 | 1299727368 3 | 0 4 | 310447 5 | 0 6 | 1409073626 7 | 3975311262 8 | 34130240 9 | 1450515018 10 | 1466150520 11 | 1342286698 12 | 1193956778 13 | 2188527278 14 | 3361989254 15 | 3912865238 16 | 3596260151 17 | 517973768 18 | 1462428821 19 | 0 20 | 2278350848 21 | 32767 22 | -------------------------------------------------------------------------------- /testing/Makefile: -------------------------------------------------------------------------------- 1 | 2 | test: 3 | @btest -c btest.cfg 4 | -------------------------------------------------------------------------------- /testing/Scripts/README: -------------------------------------------------------------------------------- 1 | Place helper scripts, such a btest-diff canonifiers, in this directory. 2 | Note that Zeek versions 4.1 and newer include their btest tooling as part 3 | of the installation. Take a look at the folder reported via 4 | 5 | zeek-config --btest_tools_dir 6 | 7 | for scripts, PRNG seeds, and pcaps you might be able to reuse. 8 | -------------------------------------------------------------------------------- /testing/Scripts/get-zeek-env: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | # 3 | # BTest helper for getting values for Zeek-related environment variables. 4 | 5 | # shellcheck disable=SC2002 6 | base="$(dirname "$0")" 7 | zeek_dist=$(cat "${base}/../../build/CMakeCache.txt" 2>/dev/null | grep ZEEK_DIST | cut -d = -f 2) 8 | 9 | if [ -n "${zeek_dist}" ]; then 10 | if [ "$1" = "zeekpath" ]; then 11 | "${zeek_dist}/build/zeek-path-dev" 12 | elif [ "$1" = "zeek_plugin_path" ]; then 13 | (cd "${base}/../.." && pwd) 14 | elif [ "$1" = "path" ]; then 15 | echo "${zeek_dist}/build/src:${zeek_dist}/aux/btest:${base}/:${zeek_dist}/aux/zeek-cut:$PATH" 16 | else 17 | echo "usage: $(basename "$0") " >&2 18 | exit 1 19 | fi 20 | else 21 | # Use Zeek installation for testing. In this case zeek-config must be in PATH. 22 | if ! which zeek-config >/dev/null 2>&1; then 23 | echo "zeek-config not found" >&2 24 | exit 1 25 | fi 26 | 27 | if [ "$1" = "zeekpath" ]; then 28 | zeek-config --zeekpath 29 | elif [ "$1" = "zeek_plugin_path" ]; then 30 | # Combine the local tree and the system-wide path. This allows 31 | # us to test on a local build or an installation made via zkg, 32 | # which squirrels away the build. 33 | echo "$(cd "${base}/../.." && pwd)/build:$(zeek-config --plugin_dir)" 34 | elif [ "$1" = "path" ]; then 35 | echo "${PATH}" 36 | else 37 | echo "usage: $(basename "$0") " >&2 38 | exit 1 39 | fi 40 | fi 41 | -------------------------------------------------------------------------------- /testing/Traces/http.pcap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/corelight/json-streaming-logs/e61088a85b3692b8403de94a815458af9313c344/testing/Traces/http.pcap -------------------------------------------------------------------------------- /testing/btest.cfg: -------------------------------------------------------------------------------- 1 | [btest] 2 | TestDirs = tests 3 | TmpDir = %(testbase)s/.tmp 4 | BaselineDir = %(testbase)s/Baseline 5 | IgnoreDirs = .tmp 6 | IgnoreFiles = *.tmp *.swp #* *.trace .DS_Store 7 | 8 | [environment] 9 | ZEEKPATH=`%(testbase)s/Scripts/get-zeek-env zeekpath` 10 | ZEEK_SEED_FILE=%(testbase)s/Files/random.seed 11 | PATH=`%(testbase)s/Scripts/get-zeek-env path` 12 | PACKAGE=%(testbase)s/../scripts 13 | TZ=UTC 14 | LC_ALL=C 15 | TRACES=%(testbase)s/Traces 16 | TMPDIR=%(testbase)s/.tmp 17 | -------------------------------------------------------------------------------- /testing/tests/logs-content.zeek: -------------------------------------------------------------------------------- 1 | # @TEST-DOC: Verifies properties of the package's resulting JSON logs. This needs jq. 2 | # @TEST-REQUIRES: jq --version 3 | # @TEST-EXEC: zeek -r $TRACES/http.pcap $PACKAGE %INPUT 4 | # Keep only a handful of fields for baselining, this isn't a Zeek internals test. 5 | # @TEST-EXEC: cat json_streaming_conn.log | jq -S '{_path, _write_ts, _system_name, "id.orig_h", "id.resp_h", "id.orig_p", "id.resp_p", proto, service}' >output.conn 6 | # @TEST-EXEC: btest-diff output.conn 7 | # @TEST-EXEC: cat json_streaming_files.log | jq -S '{_path, _write_ts, _system_name, source, mime_type, filename}' >output.files 8 | # @TEST-EXEC: btest-diff output.files 9 | # @TEST-EXEC: cat json_streaming_http.log | jq -S '{_path, _write_ts, _system_name, method, host, user_agent}' >output.http 10 | # @TEST-EXEC: btest-diff output.http 11 | # @TEST-EXEC: cat json_streaming_packet_filter.log | jq -S '{_path, _write_ts, _system_name, node, filter}' >output.packet_filter 12 | # @TEST-EXEC: btest-diff output.packet_filter 13 | 14 | # Set a hostname so we can verify it makes it into the logs: 15 | redef JSONStreaming::system_name = "testsystem"; 16 | 17 | # Turn off log rotation handling because it only kicks in for some of the files: 18 | redef JSONStreaming::enable_log_rotation = F; 19 | -------------------------------------------------------------------------------- /testing/tests/logs-disable-default.zeek: -------------------------------------------------------------------------------- 1 | # @TEST-DOC: Verifies that only the json-streaming logs exist when configured accordingly. 2 | # @TEST-EXEC: zeek -r $TRACES/http.pcap $PACKAGE %INPUT 3 | # @TEST-EXEC: for f in conn files http packet_filter; do test ! -f $f.log; done 4 | # @TEST-EXEC: for f in conn files http packet_filter; do test -f json_streaming_$f.log; done 5 | 6 | # Turn off log rotation handling because it only kicks in for some of the files: 7 | redef JSONStreaming::enable_log_rotation = F; 8 | 9 | # Turn off default logs: 10 | redef JSONStreaming::disable_default_logs = T; 11 | -------------------------------------------------------------------------------- /testing/tests/logs-present.zeek: -------------------------------------------------------------------------------- 1 | # @TEST-DOC: Verifies that Zeek by default writes both the usual logs and the json-streaming ones. 2 | # @TEST-EXEC: zeek -r $TRACES/http.pcap $PACKAGE %INPUT 3 | # @TEST-EXEC: for f in conn files http packet_filter; do test -f $f.log; done 4 | # @TEST-EXEC: for f in conn files http packet_filter; do test -f json_streaming_$f.log; done 5 | 6 | # Turn off log rotation handling because it only kicks in for some of the files: 7 | redef JSONStreaming::enable_log_rotation = F; 8 | -------------------------------------------------------------------------------- /testing/tests/logs-uncompressed.zeek: -------------------------------------------------------------------------------- 1 | # @TEST-DOC: Verifies that the package correctly overrides the compression level. 2 | # @TEST-EXEC: zeek -r $TRACES/http.pcap $PACKAGE %INPUT 3 | # @TEST-EXEC: for f in conn files http packet_filter; do test -f $f.log.gz; done 4 | # @TEST-EXEC: for f in conn files http packet_filter; do test -f json_streaming_$f.log; done 5 | 6 | # Set a compression level for the logs. 7 | redef LogAscii::gzip_level = 5; 8 | 9 | # Turn off log rotation handling because it only kicks in for some of the files: 10 | redef JSONStreaming::enable_log_rotation = F; 11 | -------------------------------------------------------------------------------- /zkg.meta: -------------------------------------------------------------------------------- 1 | [package] 2 | description = JSON streaming logs 3 | tags = logs, json, streaming, stream, filebeat, splunk_forwarder, logstash 4 | script_dir = scripts 5 | test_command = cd testing && btest -d -c btest.cfg 6 | --------------------------------------------------------------------------------