├── LICENSE ├── README.md ├── scripts ├── __load__.zeek ├── preserve_files.zeek └── remote_control.zeek ├── testing ├── .gitignore ├── Baseline │ ├── scripts.test-preserve-files-2 │ │ └── intel.log │ ├── scripts.test-preserve-files-3 │ │ └── intel.log │ ├── scripts.test-preserve-files-4 │ │ └── intel.log │ ├── scripts.test-preserve-files-5 │ │ └── intel.log │ ├── scripts.test-preserve-files-cluster-2 │ │ └── manager-1.intel.log │ ├── scripts.test-preserve-files-cluster │ │ └── manager-1.intel.log │ ├── scripts.test-preserve-files │ │ └── intel.log │ ├── scripts.test-preserve-nofiles-2 │ │ └── intel.log │ ├── scripts.test-preserve-nofiles │ │ └── intel.log │ └── scripts.test-remote-control │ │ └── output ├── btest.cfg ├── random.seed ├── scripts │ ├── test-preserve-files-cluster.zeek │ ├── test-preserve-files.zeek │ ├── test-preserve-nofiles.zeek │ └── test-remote-control.zeek ├── tools │ └── diff-remove-timestamps └── traces │ └── get.trace ├── utils └── intel-mgr.py └── zkg.meta /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016-2022 by Jan Grashoefer 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are met: 5 | 6 | (1) Redistributions of source code must retain the above copyright 7 | notice, this list of conditions and the following disclaimer. 8 | 9 | (2) Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in 11 | the documentation and/or other materials provided with the 12 | distribution. 13 | 14 | (3) Neither the name of the author, nor the names of contributors may 15 | be used to endorse or promote products derived from this software 16 | without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 22 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Intel Extensions 2 | 3 | This package provides extensions for Zeek's intelligence framework. It implements the following functionalities: 4 | 5 | * Remote management of intelligence items (using [broker](https://github.com/zeek/broker)). 6 | * Preservation of files associated with an intel hit. 7 | * ~~Intelligence expiration on per item basis.~~ Per item expiration has been moved to a [separate package](https://github.com/J-Gras/intel-expire). 8 | * ~~Support for `:` indicators.~~ Support for `:` indicators has been moved to a [separate package](https://github.com/J-Gras/intel-seen-more). 9 | 10 | ## Installation 11 | 12 | The scripts are available as package for the [Zeek Package Manager](https://github.com/zeek/package-manager) and can be installed using the following command: `zkg install intel-extensions` 13 | 14 | ## Usage 15 | 16 | None of the scripts is loaded by default, i.e. `zkg load intel-extensions` does not enable any functionality. To load all scripts, add the following to your `local.zeek`: 17 | ``` 18 | @load packages 19 | @load packages/intel-extensions/remote_control.zeek 20 | @load packages/intel-extensions/preserve_files.zeek 21 | 22 | ``` 23 | -------------------------------------------------------------------------------- /scripts/__load__.zeek: -------------------------------------------------------------------------------- 1 | # No script is loaded by default 2 | -------------------------------------------------------------------------------- /scripts/preserve_files.zeek: -------------------------------------------------------------------------------- 1 | ##! This script preserves extracted files in case of an intel hit. 2 | 3 | @load base/frameworks/intel 4 | @load base/frameworks/files 5 | 6 | module Intel; 7 | 8 | export { 9 | ## The prefix where files are preserved. 10 | option preserve_prefix = "./preserved_files/"; 11 | 12 | ## By default files will be preserved if they are in any way 13 | ## associated to an intel hit. By adding intel types to this 14 | ## filter, only files with corresponding hits will be moved. 15 | option preserve_filter: Intel::TypeSet = {}; 16 | 17 | redef record connection += { 18 | ## Indicate whether intel matched in this connection. 19 | intel_hit: bool &default = F; 20 | }; 21 | 22 | ## Preservation state of a file. 23 | type PreserveState: enum { SKIP_FILE, KEEP_FILE, FILE_MOVED }; 24 | 25 | redef record Files::Info += { 26 | ## Indicate whether the file should be moved after extraction. 27 | preserve_state: PreserveState &default = SKIP_FILE; 28 | }; 29 | } 30 | 31 | function move_file(finfo: Files::Info) 32 | { 33 | local ex_file = finfo$extracted; 34 | local ex_path = cat(FileExtract::prefix, ex_file); 35 | local pre_path = cat(preserve_prefix, ex_file); 36 | 37 | # Move files using mv 38 | local ret = system(fmt("mv \"%s\" \"%s\"", 39 | safe_shell_quote(ex_path), 40 | safe_shell_quote(pre_path) 41 | )); 42 | 43 | finfo$preserve_state = FILE_MOVED; 44 | } 45 | 46 | function preserve_match(s: Seen) 47 | { 48 | # Skip if observed type is not of interest 49 | if ( (|preserve_filter| != 0) && # keep all by default 50 | (s$indicator_type !in preserve_filter) ) 51 | return; 52 | 53 | # Hit in the context of a file 54 | if( s?$f ) 55 | { 56 | local fuid = s$f$info$fuid; 57 | # Check whether file analysis is ongoing 58 | if ( Files::file_exists(fuid) ) 59 | { 60 | # Mark file for moving after analysis 61 | local f = Files::lookup_file(s$f$info$fuid); 62 | f$info$preserve_state = KEEP_FILE; 63 | } 64 | else 65 | { 66 | # File analysis already finished 67 | if ( s$f$info?$extracted && 68 | (s$f$info$preserve_state != FILE_MOVED) ) 69 | { 70 | move_file(s$f$info); 71 | } 72 | } 73 | return; 74 | } 75 | 76 | # Hit in the context of a connection 77 | if ( s?$conn ) 78 | { 79 | # Mark connection as intel tainted 80 | s$conn$intel_hit = T; 81 | } 82 | } 83 | 84 | @if ( !Cluster::is_enabled() ) 85 | event Intel::match(s: Seen, items: set[Item]) 86 | { 87 | preserve_match(s); 88 | } 89 | @endif 90 | 91 | @if ( Cluster::is_enabled() && Cluster::local_node_type() == Cluster::WORKER ) 92 | event Intel::match_remote(s: Seen) 93 | { 94 | preserve_match(s); 95 | } 96 | @endif 97 | 98 | event file_state_remove(f: fa_file) 99 | { 100 | if( f$info?$extracted ) 101 | { 102 | if ( f$info$preserve_state == KEEP_FILE ) 103 | { 104 | move_file(f$info); 105 | return; 106 | } 107 | 108 | for ( cid, c in f$conns ) 109 | { 110 | if ( c$intel_hit ) 111 | { 112 | move_file(f$info); 113 | return; 114 | } 115 | } 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /scripts/remote_control.zeek: -------------------------------------------------------------------------------- 1 | ##! This script allows to remove intelligence items using broker. 2 | 3 | module Intel; 4 | 5 | export { 6 | ## Broker port. 7 | option broker_port = 5012/tcp; 8 | ## Broker bind address. 9 | option broker_addr = 127.0.0.1; 10 | 11 | ## Event to raise for intel item query. 12 | global remote_query: event(indicator: string, indicator_type: string); 13 | ## Event to raise for intel item removal. 14 | global remote_remove: event(indicator: string, indicator_type: string); 15 | ## Event to raise for intel item insertion. 16 | global remote_insert: event(indicator: string, indicator_type: string); 17 | } 18 | 19 | global remote_query_reply: event(success: bool, indicator: string); 20 | global remote_remove_reply: event(success: bool, indicator: string); 21 | global remote_insert_reply: event(success: bool, indicator: string); 22 | 23 | redef enum Where += { 24 | # Location used for lookups from remote 25 | Intel::REMOTE, 26 | }; 27 | 28 | global type_tbl: table[string] of Type = { 29 | ["ADDR"] = ADDR, 30 | ["SUBNET"] = SUBNET, 31 | ["URL"] = URL, 32 | ["SOFTWARE"] = SOFTWARE, 33 | ["EMAIL"] = EMAIL, 34 | ["DOMAIN"] = DOMAIN, 35 | ["USER_NAME"] = USER_NAME, 36 | ["CERT_HASH"] = CERT_HASH, 37 | ["PUBKEY_HASH"] = PUBKEY_HASH, 38 | }; 39 | 40 | function compose_seen(indicator: string, indicator_type: Type): Seen 41 | { 42 | local res: Seen = [ 43 | $indicator = indicator, 44 | $indicator_type = indicator_type, 45 | $where = Intel::REMOTE 46 | ]; 47 | 48 | if ( indicator_type == ADDR ) 49 | { 50 | res$host = to_addr(indicator); 51 | } 52 | 53 | return res; 54 | } 55 | 56 | function compose_item(indicator: string, indicator_type: Type): Item 57 | { 58 | local res: Item = [ 59 | $indicator = indicator, 60 | $indicator_type = indicator_type, 61 | $meta = record( 62 | $source = "intel-remote" 63 | ) 64 | ]; 65 | 66 | return res; 67 | } 68 | 69 | event zeek_init() 70 | { 71 | Broker::subscribe("zeek/intel/"); 72 | Broker::listen(fmt("%s", broker_addr), broker_port); 73 | } 74 | 75 | event Intel::remote_query(indicator: string, indicator_type: string) 76 | { 77 | local s = compose_seen(indicator, type_tbl[indicator_type]); 78 | # Lookup indicator and return result 79 | Broker::publish("zeek/intel/query", remote_query_reply, find(s), indicator); 80 | } 81 | 82 | event Intel::remote_remove(indicator: string, indicator_type: string) 83 | { 84 | local item = compose_item(indicator, type_tbl[indicator_type]); 85 | remove(item, T); 86 | # Always indicate success 87 | Broker::publish("zeek/intel/remove", remote_remove_reply, T, indicator); 88 | } 89 | 90 | event Intel::remote_insert(indicator: string, indicator_type: string) 91 | { 92 | local item = compose_item(indicator, type_tbl[indicator_type]); 93 | insert(item); 94 | # Always indicate success 95 | Broker::publish("zeek/intel/insert", remote_insert_reply, T, indicator); 96 | } 97 | -------------------------------------------------------------------------------- /testing/.gitignore: -------------------------------------------------------------------------------- 1 | .tmp 2 | .btest.failed.dat -------------------------------------------------------------------------------- /testing/Baseline/scripts.test-preserve-files-2/intel.log: -------------------------------------------------------------------------------- 1 | ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. 2 | #separator \x09 3 | #set_separator , 4 | #empty_field (empty) 5 | #unset_field - 6 | #path intel 7 | #open XXXX-XX-XX-XX-XX-XX 8 | #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p seen.indicator seen.indicator_type seen.where seen.node matched sources fuid file_mime_type file_desc 9 | #types time string addr port addr port string enum enum string set[enum] set[string] string string string 10 | XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.228.5 59856 192.150.187.43 80 test-filename Intel::FILE_NAME Files::IN_NAME zeek Intel::FILE_NAME source1 FMnxxt3xjVcWNS2141 - http://bro.org/download/CHANGES.bro-aux.txt 11 | #close XXXX-XX-XX-XX-XX-XX 12 | -------------------------------------------------------------------------------- /testing/Baseline/scripts.test-preserve-files-3/intel.log: -------------------------------------------------------------------------------- 1 | ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. 2 | #separator \x09 3 | #set_separator , 4 | #empty_field (empty) 5 | #unset_field - 6 | #path intel 7 | #open XXXX-XX-XX-XX-XX-XX 8 | #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p seen.indicator seen.indicator_type seen.where seen.node matched sources fuid file_mime_type file_desc 9 | #types time string addr port addr port string enum enum string set[enum] set[string] string string string 10 | XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.228.5 59856 192.150.187.43 80 141.142.228.5 Intel::ADDR Conn::IN_ORIG zeek Intel::ADDR source1 - - - 11 | #close XXXX-XX-XX-XX-XX-XX 12 | -------------------------------------------------------------------------------- /testing/Baseline/scripts.test-preserve-files-4/intel.log: -------------------------------------------------------------------------------- 1 | ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. 2 | #separator \x09 3 | #set_separator , 4 | #empty_field (empty) 5 | #unset_field - 6 | #path intel 7 | #open XXXX-XX-XX-XX-XX-XX 8 | #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p seen.indicator seen.indicator_type seen.where seen.node matched sources fuid file_mime_type file_desc 9 | #types time string addr port addr port string enum enum string set[enum] set[string] string string string 10 | XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.228.5 59856 192.150.187.43 80 141.142.228.5 Intel::ADDR Conn::IN_ORIG zeek Intel::ADDR source1 - - - 11 | XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.228.5 59856 192.150.187.43 80 test-filename Intel::FILE_NAME Files::IN_NAME zeek Intel::FILE_NAME source1 FMnxxt3xjVcWNS2141 - http://bro.org/download/CHANGES.bro-aux.txt 12 | XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.228.5 59856 192.150.187.43 80 397168fd09991a0e712254df7bc639ac Intel::FILE_HASH Files::IN_HASH zeek Intel::FILE_HASH source1 FMnxxt3xjVcWNS2141 text/plain http://bro.org/download/CHANGES.bro-aux.txt 13 | #close XXXX-XX-XX-XX-XX-XX 14 | -------------------------------------------------------------------------------- /testing/Baseline/scripts.test-preserve-files-5/intel.log: -------------------------------------------------------------------------------- 1 | ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. 2 | #separator \x09 3 | #set_separator , 4 | #empty_field (empty) 5 | #unset_field - 6 | #path intel 7 | #open XXXX-XX-XX-XX-XX-XX 8 | #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p seen.indicator seen.indicator_type seen.where seen.node matched sources fuid file_mime_type file_desc 9 | #types time string addr port addr port string enum enum string set[enum] set[string] string string string 10 | XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.228.5 59856 192.150.187.43 80 141.142.228.5 Intel::ADDR Conn::IN_ORIG zeek Intel::ADDR source1 - - - 11 | XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.228.5 59856 192.150.187.43 80 test-filename Intel::FILE_NAME Files::IN_NAME zeek Intel::FILE_NAME source1 FMnxxt3xjVcWNS2141 - http://bro.org/download/CHANGES.bro-aux.txt 12 | XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.228.5 59856 192.150.187.43 80 397168fd09991a0e712254df7bc639ac Intel::FILE_HASH Files::IN_HASH zeek Intel::FILE_HASH source1 FMnxxt3xjVcWNS2141 text/plain http://bro.org/download/CHANGES.bro-aux.txt 13 | #close XXXX-XX-XX-XX-XX-XX 14 | -------------------------------------------------------------------------------- /testing/Baseline/scripts.test-preserve-files-cluster-2/manager-1.intel.log: -------------------------------------------------------------------------------- 1 | ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. 2 | #separator \x09 3 | #set_separator , 4 | #empty_field (empty) 5 | #unset_field - 6 | #path intel 7 | #open XXXX-XX-XX-XX-XX-XX 8 | #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p seen.indicator seen.indicator_type seen.where seen.node matched sources fuid file_mime_type file_desc 9 | #types time string addr port addr port string enum enum string set[enum] set[string] string string string 10 | XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.228.5 59856 192.150.187.43 80 141.142.228.5 Intel::ADDR Conn::IN_ORIG worker-1 Intel::ADDR source1 - - - 11 | XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.228.5 59856 192.150.187.43 80 test-filename Intel::FILE_NAME Files::IN_NAME worker-1 Intel::FILE_NAME source1 FMnxxt3xjVcWNS2141 - http://bro.org/download/CHANGES.bro-aux.txt 12 | XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.228.5 59856 192.150.187.43 80 397168fd09991a0e712254df7bc639ac Intel::FILE_HASH Files::IN_HASH worker-1 Intel::FILE_HASH source1 FMnxxt3xjVcWNS2141 text/plain http://bro.org/download/CHANGES.bro-aux.txt 13 | #close XXXX-XX-XX-XX-XX-XX 14 | -------------------------------------------------------------------------------- /testing/Baseline/scripts.test-preserve-files-cluster/manager-1.intel.log: -------------------------------------------------------------------------------- 1 | ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. 2 | #separator \x09 3 | #set_separator , 4 | #empty_field (empty) 5 | #unset_field - 6 | #path intel 7 | #open XXXX-XX-XX-XX-XX-XX 8 | #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p seen.indicator seen.indicator_type seen.where seen.node matched sources fuid file_mime_type file_desc 9 | #types time string addr port addr port string enum enum string set[enum] set[string] string string string 10 | XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.228.5 59856 192.150.187.43 80 397168fd09991a0e712254df7bc639ac Intel::FILE_HASH Files::IN_HASH worker-1 Intel::FILE_HASH source1 FMnxxt3xjVcWNS2141 text/plain http://bro.org/download/CHANGES.bro-aux.txt 11 | #close XXXX-XX-XX-XX-XX-XX 12 | -------------------------------------------------------------------------------- /testing/Baseline/scripts.test-preserve-files/intel.log: -------------------------------------------------------------------------------- 1 | ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. 2 | #separator \x09 3 | #set_separator , 4 | #empty_field (empty) 5 | #unset_field - 6 | #path intel 7 | #open XXXX-XX-XX-XX-XX-XX 8 | #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p seen.indicator seen.indicator_type seen.where seen.node matched sources fuid file_mime_type file_desc 9 | #types time string addr port addr port string enum enum string set[enum] set[string] string string string 10 | XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.228.5 59856 192.150.187.43 80 397168fd09991a0e712254df7bc639ac Intel::FILE_HASH Files::IN_HASH zeek Intel::FILE_HASH source1 FMnxxt3xjVcWNS2141 text/plain http://bro.org/download/CHANGES.bro-aux.txt 11 | #close XXXX-XX-XX-XX-XX-XX 12 | -------------------------------------------------------------------------------- /testing/Baseline/scripts.test-preserve-nofiles-2/intel.log: -------------------------------------------------------------------------------- 1 | ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. 2 | #separator \x09 3 | #set_separator , 4 | #empty_field (empty) 5 | #unset_field - 6 | #path intel 7 | #open XXXX-XX-XX-XX-XX-XX 8 | #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p seen.indicator seen.indicator_type seen.where seen.node matched sources fuid file_mime_type file_desc 9 | #types time string addr port addr port string enum enum string set[enum] set[string] string string string 10 | XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.228.5 59856 192.150.187.43 80 141.142.228.5 Intel::ADDR Conn::IN_ORIG zeek Intel::ADDR source1 - - - 11 | XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.228.5 59856 192.150.187.43 80 test-filename Intel::FILE_NAME Files::IN_NAME zeek Intel::FILE_NAME source1 FMnxxt3xjVcWNS2141 - http://bro.org/download/CHANGES.bro-aux.txt 12 | XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.228.5 59856 192.150.187.43 80 397168fd09991a0e712254df7bc639ac Intel::FILE_HASH Files::IN_HASH zeek Intel::FILE_HASH source1 FMnxxt3xjVcWNS2141 text/plain http://bro.org/download/CHANGES.bro-aux.txt 13 | #close XXXX-XX-XX-XX-XX-XX 14 | -------------------------------------------------------------------------------- /testing/Baseline/scripts.test-preserve-nofiles/intel.log: -------------------------------------------------------------------------------- 1 | ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. 2 | #separator \x09 3 | #set_separator , 4 | #empty_field (empty) 5 | #unset_field - 6 | #path intel 7 | #open XXXX-XX-XX-XX-XX-XX 8 | #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p seen.indicator seen.indicator_type seen.where seen.node matched sources fuid file_mime_type file_desc 9 | #types time string addr port addr port string enum enum string set[enum] set[string] string string string 10 | XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 141.142.228.5 59856 192.150.187.43 80 397168fd09991a0e712254df7bc639ac Intel::FILE_HASH Files::IN_HASH zeek Intel::FILE_HASH source1 FMnxxt3xjVcWNS2141 text/plain http://bro.org/download/CHANGES.bro-aux.txt 11 | #close XXXX-XX-XX-XX-XX-XX 12 | -------------------------------------------------------------------------------- /testing/Baseline/scripts.test-remote-control/output: -------------------------------------------------------------------------------- 1 | ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. 2 | Sent insert command for "1.0.0.1" (ADDR). 3 | Successfully executed insert "1.0.0.1" 4 | Sent query command for "1.0.0.1" (ADDR). 5 | Successfully executed query "1.0.0.1" 6 | Sent remove command for "1.0.0.1" (ADDR). 7 | Successfully executed remove "1.0.0.1" 8 | Sent query command for "1.0.0.1" (ADDR). 9 | Failed to query "1.0.0.1" 10 | Sent remove command for "5.14.4.5" (ADDR). 11 | Successfully executed remove "5.14.4.5" 12 | #separator \x09 13 | #set_separator , 14 | #empty_field (empty) 15 | #unset_field - 16 | #path reporter 17 | #open XXXX-XX-XX-XX-XX-XX 18 | #fields ts level message location 19 | #types time enum string string 20 | XXXXXXXXXX.XXXXXX Reporter::INFO Tried to remove non-existing item '5.14.4.5' (Intel::ADDR). /usr/local/zeek/share/zeek/base/frameworks/intel/./main.zeek, lines 570-571 21 | XXXXXXXXXX.XXXXXX Reporter::INFO received termination signal (empty) 22 | -------------------------------------------------------------------------------- /testing/btest.cfg: -------------------------------------------------------------------------------- 1 | [btest] 2 | TestDirs = scripts 3 | TmpDir = %(testbase)s/.tmp 4 | BaselineDir = %(testbase)s/Baseline 5 | IgnoreDirs = .tmp 6 | IgnoreFiles = *.tmp *.swp 7 | 8 | [environment] 9 | ZEEK_SEED_FILE=%(testbase)s/random.seed 10 | ZEEKPATH=`zeek-config --zeekpath`:%(testbase)s/../scripts 11 | PYTHONPATH=`zeek-config --python_dir` 12 | TZ=UTC 13 | LC_ALL=C 14 | TRACES=%(testbase)s/traces 15 | UTILS=%(testbase)s/../utils 16 | TMPDIR=%(testbase)s/.tmp 17 | TEST_DIFF_CANONIFIER=%(testbase)s/tools/diff-remove-timestamps 18 | -------------------------------------------------------------------------------- /testing/random.seed: -------------------------------------------------------------------------------- 1 | 2983378351 2 | 1299727368 3 | 0 4 | 310447 5 | 0 6 | 1409073626 7 | 3975311262 8 | 34130240 9 | 1450515018 10 | 1466150520 11 | 1342286698 12 | 1193956778 13 | 2188527278 14 | 3361989254 15 | 3912865238 16 | 3596260151 17 | 517973768 18 | 1462428821 19 | 0 20 | 2278350848 21 | 32767 22 | -------------------------------------------------------------------------------- /testing/scripts/test-preserve-files-cluster.zeek: -------------------------------------------------------------------------------- 1 | # @TEST-SERIALIZE: comm 2 | # 3 | # @TEST-EXEC: mkdir preserved_files 4 | # @TEST-EXEC: btest-bg-run manager-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager-1 zeek %INPUT 5 | # @TEST-EXEC: btest-bg-run worker-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -r $TRACES/get.trace %INPUT 6 | # @TEST-EXEC: btest-bg-wait -k 13 7 | # @TEST-EXEC: btest-diff manager-1/intel.log 8 | # @TEST-EXEC: test -e preserved_files/extract-1362692527.009512-HTTP-FMnxxt3xjVcWNS2141 9 | 10 | # @TEST-START-FILE cluster-layout.zeek 11 | redef Cluster::nodes = { 12 | ["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp], 13 | ["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1"], 14 | }; 15 | # @TEST-END-FILE 16 | 17 | # Scenario: Hit on FILE_HASH 18 | @load preserve_files 19 | @load frameworks/files/extract-all-files 20 | @load frameworks/files/hash-all-files 21 | @load frameworks/intel/seen/file-hashes 22 | 23 | redef Log::default_rotation_interval = 0secs; 24 | redef Intel::preserve_prefix = "../preserved_files/"; 25 | 26 | module Intel; 27 | 28 | # Manager 29 | 30 | @if ( Cluster::local_node_type() == Cluster::MANAGER ) 31 | event Cluster::node_up(name: string, id: string) 32 | { 33 | # Insert the data once all workers are connected. 34 | if ( Cluster::worker_count == 1 ) 35 | { 36 | Intel::insert([$indicator="397168fd09991a0e712254df7bc639ac", 37 | $indicator_type=Intel::FILE_HASH, $meta=[$source="source1"]]); 38 | } 39 | } 40 | @endif 41 | 42 | # Worker 43 | 44 | @if ( Cluster::local_node_type() == Cluster::WORKER ) 45 | event zeek_init() 46 | { 47 | suspend_processing(); 48 | } 49 | 50 | event Intel::insert_indicator(item: Intel::Item) 51 | { 52 | # Run test on worker-1 when item has been inserted 53 | if ( Cluster::node == "worker-1" ) 54 | continue_processing(); 55 | } 56 | @endif 57 | 58 | # Shutdown logic 59 | 60 | event die() 61 | { 62 | terminate(); 63 | } 64 | 65 | event Intel::log_intel(rec: Intel::Info) 66 | { 67 | if ( "source1" in rec$sources ) 68 | schedule 2sec { die() }; 69 | } 70 | 71 | event Cluster::node_down(name: string, id: string) 72 | { 73 | # Cascading termination 74 | schedule 2sec { die() }; 75 | } 76 | 77 | # @TEST-START-NEXT 78 | 79 | # Scenario: Multiple hits on the same file 80 | @load preserve_files 81 | @load frameworks/files/extract-all-files 82 | @load frameworks/files/hash-all-files 83 | @load frameworks/intel/seen/file-hashes 84 | @load frameworks/intel/seen/file-names 85 | @load frameworks/intel/seen/conn-established 86 | 87 | redef Log::default_rotation_interval = 0secs; 88 | redef Intel::preserve_prefix = "../preserved_files/"; 89 | 90 | module Intel; 91 | 92 | # Manager 93 | 94 | @if ( Cluster::local_node_type() == Cluster::MANAGER ) 95 | event Cluster::node_up(name: string, id: string) 96 | { 97 | # Insert the data once all workers are connected. 98 | if ( Cluster::worker_count == 1 ) 99 | { 100 | Intel::insert([$indicator="397168fd09991a0e712254df7bc639ac", 101 | $indicator_type=Intel::FILE_HASH, $meta=[$source="source1"]]); 102 | Intel::insert([$indicator="test-filename", 103 | $indicator_type=Intel::FILE_NAME, $meta=[$source="source1"]]); 104 | Intel::insert([$indicator="141.142.228.5", 105 | $indicator_type=Intel::ADDR, $meta=[$source="source1"]]); 106 | } 107 | } 108 | @endif 109 | 110 | # Worker 111 | 112 | @if ( Cluster::local_node_type() == Cluster::WORKER ) 113 | event zeek_init() 114 | { 115 | suspend_processing(); 116 | } 117 | 118 | global worker_data = 0; 119 | event Intel::insert_indicator(item: Intel::Item) 120 | { 121 | # Run test on worker-1 when all items have been inserted 122 | if ( Cluster::node == "worker-1" ) 123 | { 124 | ++worker_data; 125 | if ( worker_data == 3 ) 126 | continue_processing(); 127 | } 128 | } 129 | 130 | event file_new(f: fa_file) &priority=10 131 | { 132 | f$info$filename = "test-filename"; 133 | } 134 | @endif 135 | 136 | # Shutdown logic 137 | 138 | event die() 139 | { 140 | terminate(); 141 | } 142 | 143 | event Intel::log_intel(rec: Intel::Info) 144 | { 145 | if ( "source1" in rec$sources ) 146 | schedule 2sec { die() }; 147 | } 148 | 149 | event Cluster::node_down(name: string, id: string) 150 | { 151 | # Cascading termination 152 | schedule 2sec { die() }; 153 | } 154 | -------------------------------------------------------------------------------- /testing/scripts/test-preserve-files.zeek: -------------------------------------------------------------------------------- 1 | # @TEST-EXEC: mkdir preserved_files 2 | # @TEST-EXEC: zeek -r $TRACES/get.trace preserve_files %INPUT 3 | # @TEST-EXEC: btest-diff intel.log 4 | # @TEST-EXEC: test -e preserved_files/extract-1362692527.009512-HTTP-FMnxxt3xjVcWNS2141 5 | 6 | # Keep FILE_HASH hit (match after analysis finished) 7 | @load frameworks/files/extract-all-files 8 | @load frameworks/files/hash-all-files 9 | @load frameworks/intel/seen/file-hashes 10 | 11 | event zeek_init() 12 | { 13 | Intel::insert([$indicator="397168fd09991a0e712254df7bc639ac", 14 | $indicator_type=Intel::FILE_HASH, $meta=[$source="source1"]]); 15 | } 16 | 17 | # @TEST-START-NEXT 18 | 19 | # Keep FILE_NAME hit (match in the beginning of file analysis) 20 | @load frameworks/files/extract-all-files 21 | @load frameworks/intel/seen/file-names 22 | 23 | event zeek_init() 24 | { 25 | Intel::insert([$indicator="test-filename", 26 | $indicator_type=Intel::FILE_NAME, $meta=[$source="source1"]]); 27 | } 28 | 29 | event file_new(f: fa_file) &priority=10 30 | { 31 | f$info$filename = "test-filename"; 32 | } 33 | 34 | # @TEST-START-NEXT 35 | 36 | # Keep ADDR hit (match before file analysis starts) 37 | @load frameworks/files/extract-all-files 38 | @load frameworks/intel/seen/conn-established 39 | 40 | event zeek_init() 41 | { 42 | Intel::insert([$indicator="141.142.228.5", 43 | $indicator_type=Intel::ADDR, $meta=[$source="source1"]]); 44 | } 45 | 46 | # @TEST-START-NEXT 47 | 48 | # Multiple hits on the same file 49 | @load frameworks/files/extract-all-files 50 | @load frameworks/files/hash-all-files 51 | @load frameworks/intel/seen/file-hashes 52 | @load frameworks/intel/seen/file-names 53 | @load frameworks/intel/seen/conn-established 54 | 55 | event zeek_init() 56 | { 57 | Intel::insert([$indicator="397168fd09991a0e712254df7bc639ac", 58 | $indicator_type=Intel::FILE_HASH, $meta=[$source="source1"]]); 59 | Intel::insert([$indicator="test-filename", 60 | $indicator_type=Intel::FILE_NAME, $meta=[$source="source1"]]); 61 | Intel::insert([$indicator="141.142.228.5", 62 | $indicator_type=Intel::ADDR, $meta=[$source="source1"]]); 63 | } 64 | 65 | event file_new(f: fa_file) &priority=10 66 | { 67 | f$info$filename = "test-filename"; 68 | } 69 | 70 | # @TEST-START-NEXT 71 | 72 | # Multiple hits on the same file using a filter 73 | @load frameworks/files/extract-all-files 74 | @load frameworks/files/hash-all-files 75 | @load frameworks/intel/seen/file-hashes 76 | @load frameworks/intel/seen/file-names 77 | @load frameworks/intel/seen/conn-established 78 | 79 | redef Intel::preserve_filter = {Intel::DOMAIN, Intel::ADDR}; 80 | 81 | event zeek_init() 82 | { 83 | Intel::insert([$indicator="397168fd09991a0e712254df7bc639ac", 84 | $indicator_type=Intel::FILE_HASH, $meta=[$source="source1"]]); 85 | Intel::insert([$indicator="test-filename", 86 | $indicator_type=Intel::FILE_NAME, $meta=[$source="source1"]]); 87 | Intel::insert([$indicator="141.142.228.5", 88 | $indicator_type=Intel::ADDR, $meta=[$source="source1"]]); 89 | } 90 | 91 | event file_new(f: fa_file) &priority=10 92 | { 93 | f$info$filename = "test-filename"; 94 | } 95 | -------------------------------------------------------------------------------- /testing/scripts/test-preserve-nofiles.zeek: -------------------------------------------------------------------------------- 1 | # @TEST-EXEC: mkdir preserved_files 2 | # @TEST-EXEC: zeek -r $TRACES/get.trace preserve_files %INPUT 3 | # @TEST-EXEC: btest-diff intel.log 4 | # @TEST-EXEC-FAIL: test -e preserved_files/extract-1362692527.009512-HTTP-FMnxxt3xjVcWNS2141 5 | 6 | # Hit on FILE_HASH using a filter without FILE_HASH 7 | @load frameworks/files/extract-all-files 8 | @load frameworks/files/hash-all-files 9 | @load frameworks/intel/seen/file-hashes 10 | 11 | redef Intel::preserve_filter = {Intel::DOMAIN}; 12 | 13 | event zeek_init() 14 | { 15 | Intel::insert([$indicator="397168fd09991a0e712254df7bc639ac", 16 | $indicator_type=Intel::FILE_HASH, $meta=[$source="source1"]]); 17 | } 18 | 19 | # @TEST-START-NEXT 20 | 21 | # Multiple hits on the same file using a filter 22 | @load frameworks/files/extract-all-files 23 | @load frameworks/files/hash-all-files 24 | @load frameworks/intel/seen/file-hashes 25 | @load frameworks/intel/seen/file-names 26 | @load frameworks/intel/seen/conn-established 27 | 28 | redef Intel::preserve_filter = {Intel::DOMAIN}; 29 | 30 | event zeek_init() 31 | { 32 | Intel::insert([$indicator="397168fd09991a0e712254df7bc639ac", 33 | $indicator_type=Intel::FILE_HASH, $meta=[$source="source1"]]); 34 | Intel::insert([$indicator="test-filename", 35 | $indicator_type=Intel::FILE_NAME, $meta=[$source="source1"]]); 36 | Intel::insert([$indicator="141.142.228.5", 37 | $indicator_type=Intel::ADDR, $meta=[$source="source1"]]); 38 | } 39 | 40 | event file_new(f: fa_file) &priority=10 41 | { 42 | f$info$filename = "test-filename"; 43 | } 44 | -------------------------------------------------------------------------------- /testing/scripts/test-remote-control.zeek: -------------------------------------------------------------------------------- 1 | # @TEST-SERIALIZE: comm 2 | # 3 | # @TEST-EXEC: btest-bg-run zeek-proc zeek remote_control %INPUT 4 | # @TEST-EXEC: $UTILS/intel-mgr.py insert 1.0.0.1 ADDR > output 5 | # @TEST-EXEC: $UTILS/intel-mgr.py query 1.0.0.1 ADDR >> output 6 | # @TEST-EXEC: $UTILS/intel-mgr.py remove 1.0.0.1 ADDR >> output 7 | # @TEST-EXEC: $UTILS/intel-mgr.py query 1.0.0.1 ADDR >> output 8 | # @TEST-EXEC: $UTILS/intel-mgr.py remove 5.14.4.5 ADDR >> output 9 | # @TEST-EXEC: cat zeek-proc/reporter.log >> output 10 | # @TEST-EXEC: btest-bg-wait -k 10 11 | # @TEST-EXEC: btest-diff output 12 | 13 | redef exit_only_after_terminate = T; 14 | 15 | event Intel::remote_remove(indicator: string, indicator_type: string) 16 | { 17 | if ( indicator == "5.14.4.5" ) 18 | terminate(); 19 | } 20 | -------------------------------------------------------------------------------- /testing/tools/diff-remove-timestamps: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | # 3 | # Replace anything which looks like timestamps with XXXs (including the #start/end markers in logs). 4 | 5 | # Get us "modern" regexps with sed. 6 | if [ `uname` == "Linux" ]; then 7 | sed="sed -r" 8 | else 9 | sed="sed -E" 10 | fi 11 | 12 | # The first seds uses a "basic" regexp, the 3rd a "modern:. 13 | sed 's/"ts":[0-9]\{13\},/"ts":XXXXXXXXXXXXX,/g' | \ 14 | sed 's/[0-9]\{10\}\.[0-9]\{2,8\}/XXXXXXXXXX.XXXXXX/g' | \ 15 | $sed 's/^ *#(open|close).(19|20)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g' -------------------------------------------------------------------------------- /testing/traces/get.trace: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J-Gras/intel-extensions/351591ae033dd044024ca6811e00cd7b7538fa24/testing/traces/get.trace -------------------------------------------------------------------------------- /utils/intel-mgr.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import broker 4 | 5 | from argparse import ArgumentParser 6 | 7 | operations = ( 8 | 'query', 9 | 'remove', 10 | 'insert') 11 | 12 | intel_types = ( 13 | 'ADDR', 14 | 'SUBNET', 15 | 'URL', 16 | 'SOFTWARE', 17 | 'EMAIL', 18 | 'DOMAIN', 19 | 'USER_NAME', 20 | 'CERT_HASH', 21 | 'PUBKEY_HASH') 22 | 23 | 24 | 25 | def get_arguments(): 26 | parser = ArgumentParser(description='This script allows to manage' 27 | ' intelligence indicators of a Zeek instance using broker.') 28 | parser.add_argument('operation', metavar='OPERATION', type=str.lower, 29 | choices=operations, help='Operation to execute') 30 | parser.add_argument('indicator', metavar='INDICATOR', type=str, 31 | help='Intel indicator') 32 | parser.add_argument('indicator_type', metavar='TYPE', type=str.upper, 33 | choices=intel_types, help='Intel indicator\'s type') 34 | parser.add_argument('-p', metavar='PORT', type=int, default=5012, 35 | dest='port', help='Broker port (default: 5012)') 36 | parser.add_argument('-a', metavar='IP', type=str, default='127.0.0.1', 37 | dest='host', help='Broker host (default: 127.0.0.1)') 38 | return parser.parse_args() 39 | 40 | 41 | def main(): 42 | args = get_arguments() 43 | op = args.operation 44 | 45 | ep_zeek = broker.Endpoint() 46 | sub_zeek_intel = ep_zeek.make_subscriber(f"zeek/intel/{op}") 47 | sub_zeek_state = ep_zeek.make_status_subscriber(True) 48 | ep_zeek.peer(args.host, args.port, retry=0) 49 | 50 | # Establish connection 51 | #TODO: see https://github.com/zeek/broker/issues/18 52 | for st in sub_zeek_state.get(1, 0.5): 53 | if not (type(st) == broker.Status and st.code() == broker.SC.PeerAdded): 54 | print(f"Failed to establish connection! ({st.code()})") 55 | return 56 | 57 | # Send operation 58 | evt_op = broker.zeek.Event(f"Intel::remote_{op}", 59 | args.indicator, 60 | args.indicator_type) 61 | ep_zeek.publish(f"zeek/intel/{op}", evt_op) 62 | print(f"Sent {op} command for \"{args.indicator}\" ({args.indicator_type}).") 63 | 64 | # Await reply 65 | for (topic, data) in sub_zeek_intel.get(1, 2.0): 66 | evt_rep = broker.zeek.Event(data) 67 | if evt_rep.name() != f"Intel::remote_{op}_reply": 68 | print("Received unexpected event.") 69 | return 70 | success = evt_rep.args()[0] 71 | indicator = evt_rep.args()[1] 72 | if success: 73 | print(f"Successfully executed {op} \"{indicator}\"") 74 | else: 75 | print(f"Failed to {op} \"{indicator}\"") 76 | return 77 | 78 | print("Request timed out."); 79 | return 80 | 81 | if __name__ == '__main__': 82 | main() 83 | -------------------------------------------------------------------------------- /zkg.meta: -------------------------------------------------------------------------------- 1 | [package] 2 | description = Extensions for Zeek's intelligence framework. 3 | credits = Jan Grashoefer 4 | tags = intel, remote control, preserve files 5 | script_dir = scripts 6 | executables = utils/intel-mgr.py 7 | test_command = cd testing && btest -d 8 | depends = 9 | zeek >=3.0 10 | --------------------------------------------------------------------------------