├── README.md ├── bash ├── README.md ├── ipdr_bulkquery.sh └── ipdr_userquery.sh ├── lua ├── INDEX.md ├── README.md ├── backend_scripts │ ├── README.md │ ├── alerts │ │ ├── README.md │ │ ├── alert_filter.lua │ │ ├── print_alerts.lua │ │ └── print_alerts2.lua │ ├── alienvault-otx │ │ ├── README.md │ │ ├── avhit.png │ │ ├── check-leveldb.lua │ │ └── otx2leveldb.rb │ ├── blacklist-keys │ │ ├── README.md │ │ ├── mac.lua │ │ └── shadowhammer-blacklist.lua │ ├── cisco-umbrella-1m │ │ ├── README.md │ │ ├── cu-dns-extractor.lua │ │ └── cu-flow-marker.lua │ ├── critical-stack │ │ ├── README.md │ │ ├── compile-cs.lua │ │ ├── critical-stack-checker.lua │ │ ├── master-public.bro.dat │ │ └── tris_leveldb.lua │ ├── dyndns-alert │ │ ├── README.md │ │ ├── dynamic-dns.txt │ │ ├── dyndns-alert.lua │ │ └── trie.lua │ ├── elasticsearch │ │ ├── README.md │ │ ├── es_curl_flow.lua │ │ └── es_socket_flow.lua │ ├── enginemonitor │ │ ├── README.md │ │ ├── snmp.lua │ │ ├── snmp_cg.lua │ │ ├── snmp_walkpoll.lua │ │ └── snmp_walkpoll_key.lua │ ├── firehol │ │ ├── README.md │ │ ├── firehol.lua │ │ ├── firehol_level1.netset │ │ ├── iprangemap.lua │ │ └── iprangemap.md │ ├── flow-tag-enrich │ │ ├── README.md │ │ └── add_flow_version.lua │ ├── flows │ │ ├── README.md │ │ ├── ftracker1.lua │ │ ├── ip2location │ │ │ ├── README.md │ │ │ ├── cg_ip2loc_asn.lua │ │ │ ├── cg_ip2loc_cc.lua │ │ │ ├── cg_ip2loc_proxy.lua │ │ │ ├── cg_ip2loc_region.lua │ │ │ ├── compile_ip2loc.lua │ │ │ ├── ip2location.lua │ │ │ └── samples │ │ │ │ ├── IP2LOCATION-LITE-ASN.CSV │ │ │ │ ├── IP2LOCATION-LITE-DB3.CSV │ │ │ │ └── IP2PROXY-LITE-PX2.CSV │ │ ├── longssh-whitelist.lua │ │ ├── sg1.lua │ │ ├── sg2.lua │ │ ├── sg3.lua │ │ ├── ssh_alerts.lua │ │ └── tcpanalysis │ │ │ └── latency_monitor.lua │ ├── flowsizes │ │ ├── README.md │ │ └── flows_size.lua │ ├── fts │ │ ├── README.md │ │ ├── c2-x509-fts.lua │ │ ├── detect_sha1.lua │ │ ├── ocsp_check.lua │ │ └── ocsp_check_async.lua │ ├── geoflows │ │ ├── README.md │ │ ├── cg_ip2loc_asn.lua │ │ ├── cg_ip2loc_cc.lua │ │ ├── cg_ip2loc_proxy.lua │ │ ├── cg_ip2loc_region.lua │ │ ├── cg_ip2loc_state.lua │ │ ├── ftrie.lua │ │ ├── geoflows.lua │ │ └── libftrie.so │ ├── metricmonitors │ │ ├── README.md │ │ ├── cgmon.lua │ │ ├── cgmon_2.lua │ │ ├── cgmon_3.lua │ │ └── delta-change-alert │ │ │ ├── README.md │ │ │ └── mac_traffic_tracker.lua │ ├── netflow_generator │ │ ├── README.md │ │ ├── nfffi.lua │ │ └── nfgen.lua │ ├── passive-dns │ │ ├── README.md │ │ ├── flowtag-passive-dns.lua │ │ ├── helpers │ │ │ ├── README.md │ │ │ ├── test_1.lua │ │ │ └── tris_leveldb.lua │ │ └── passive-dns-creator.lua │ ├── resources │ │ ├── README.md │ │ ├── filehash.lua │ │ ├── printcertchain.lua │ │ ├── sha256_x509.lua │ │ └── ssl.lua │ ├── roca │ │ ├── README.md │ │ ├── cert01.pem │ │ ├── cert05.pem │ │ ├── roca.lua │ │ └── rocafile.lua │ ├── snmpasync │ │ ├── README.md │ │ ├── async_tasks.lua │ │ ├── bulkwalk_cmd.lua │ │ └── snmp_walkpoll_key.lua │ ├── strelka │ │ ├── README.md │ │ ├── strelka-json.lua │ │ ├── strelka-resource.lua │ │ ├── strelka1.png │ │ └── strelka2.png │ ├── ua-parser │ │ ├── README.md │ │ ├── cg_browser.lua │ │ ├── cg_device.lua │ │ ├── cg_os.lua │ │ ├── regexes.yaml │ │ ├── tinyyaml.lua │ │ ├── ua-extractor.lua │ │ ├── ua-resource-monitor.lua │ │ └── ua-resource.lua │ ├── unixsocket_api │ │ ├── README.md │ │ ├── rubyexample │ │ │ ├── apimon.lua │ │ │ └── rbench.rb │ │ └── trisul_unsock_api.lua │ └── url-latency-monitor │ │ ├── README.md │ │ └── urlmon.lua ├── frontend_scripts │ ├── INDEX.md │ ├── alerts │ │ ├── README.md │ │ └── socialalert.lua │ ├── buffer │ │ ├── README.md │ │ └── tcphdr.lua │ ├── custom_alerts │ │ ├── README.md │ │ └── my-alerts.lua │ ├── custom_resources │ │ ├── README.md │ │ └── pdf-files.lua │ ├── fileextract │ │ ├── README.md │ │ ├── filex_ramfs_basic.lua │ │ ├── filex_stream.lua │ │ ├── filter.lua │ │ ├── fx_largeimage.lua │ │ ├── fx_video_chunk.lua │ │ ├── malware-cymru-alert.lua │ │ ├── malware-cymru.lua │ │ ├── save_content_types.lua │ │ ├── save_content_types_sha256.lua │ │ ├── save_exe.lua │ │ ├── save_exe_simple.lua │ │ ├── save_exe_streaming.lua │ │ ├── saveall.lua │ │ └── sha256.lua │ ├── heartbleed │ │ ├── README.md │ │ ├── tls-heartbeat-2.lua │ │ ├── tlsrec1.png │ │ └── tlsrec2.png │ ├── httpserver │ │ ├── README.md │ │ └── httpsvr.lua │ ├── inputfilter │ │ ├── README.md │ │ ├── barnyard2_unixsocket.lua │ │ ├── flowimport │ │ │ ├── README.md │ │ │ ├── flowimport.lua │ │ │ └── flowimport_v6.lua │ │ ├── fortigate-log │ │ │ ├── README.md │ │ │ ├── flowimport.lua │ │ │ ├── flowimport_v6.lua │ │ │ └── fortigate-log.lua │ │ ├── kiwisyslog.lua │ │ ├── kiwisyslog_old.lua │ │ ├── lanl-cyber │ │ │ ├── README.md │ │ │ ├── flowimport.lua │ │ │ ├── flows.txt │ │ │ ├── la1.png │ │ │ ├── la2.png │ │ │ ├── la3.png │ │ │ └── lanlflow.lua │ │ ├── purelua_pcap.lua │ │ ├── silk │ │ │ ├── README.md │ │ │ ├── flowimport.lua │ │ │ └── silk.lua │ │ ├── snort_unixsocket.lua │ │ ├── suricata_eve.lua │ │ ├── suricata_eve_unixsocket.lua │ │ └── vast.lua │ ├── natsyslog │ │ ├── README.md │ │ ├── flowkey.lua │ │ ├── ipfix-protocol.lua │ │ ├── ipfixnatctr.lua │ │ ├── natsyslogctr.lua │ │ ├── raw-syslogs.lua │ │ ├── store-tagged-only.lua │ │ ├── sweepbuf.lua │ │ └── syslog-protocol.lua │ ├── packetstore │ │ ├── README.md │ │ ├── nopcap_tls.lua │ │ ├── skip-ssh.lua │ │ └── skip-youtube │ │ │ ├── README.md │ │ │ └── skip_youtube.lua │ ├── pingtunnel │ │ ├── README.md │ │ ├── hamming-check.lua │ │ ├── icmp-echo-check.lua │ │ ├── mrumap.lua │ │ └── ping-alert.png │ ├── protocol_handlers │ │ └── flow_director.lua │ ├── protocol_tree │ │ ├── README.md │ │ ├── cg_protocol_tree.lua │ │ ├── ptree_ethernet.lua │ │ └── sweepbuf.lua │ ├── reassembly │ │ ├── README.md │ │ ├── flow_attributes.lua │ │ ├── ftp.lua │ │ ├── haSSH │ │ │ ├── README.md │ │ │ ├── hassh-counters.lua │ │ │ ├── hassh1.gif │ │ │ ├── md5ffi.lua │ │ │ ├── pdurecord.lua │ │ │ ├── ssh-dissect.lua │ │ │ ├── ssh-spy.lua │ │ │ └── sweepbuf.lua │ │ ├── ja3 │ │ │ ├── README.md │ │ │ ├── jahash.lua │ │ │ └── prints │ │ │ │ ├── README.md │ │ │ │ ├── csv_toja3.rb │ │ │ │ ├── ja3fingerprint.json │ │ │ │ ├── mk_ja3fingerprint.rb │ │ │ │ ├── tofingerprints.rb │ │ │ │ └── toja3.rb │ │ ├── reass_filter.lua │ │ ├── revssh │ │ │ ├── README.md │ │ │ └── rev-ssh.lua │ │ ├── save_payloads │ │ │ ├── README.md │ │ │ └── savetcp.lua │ │ ├── ssh-analyzer │ │ │ ├── README.md │ │ │ ├── ssh-spy.lua │ │ │ ├── ssh_alert_group.lua │ │ │ └── ssh_dissect.lua │ │ ├── tls-sni │ │ │ ├── README.md │ │ │ ├── sni-tls.lua │ │ │ └── sni.lua │ │ ├── tls-version-count │ │ │ ├── README.md │ │ │ ├── counter_pk_algos.lua │ │ │ ├── counter_sig_algos.lua │ │ │ ├── sweepbuf.lua │ │ │ ├── tlscertalgos.lua │ │ │ └── tlsversions.lua │ │ └── tlsalert.lua │ ├── segmentsmack │ │ ├── README.md │ │ ├── sweepbuf.lua │ │ └── tcp-segment-smack-detect.lua │ └── simplecounter │ │ ├── README.md │ │ └── rstcounter.lua ├── hub_scripts │ └── radius-aaa │ │ ├── README.md │ │ ├── exporter │ │ ├── README.md │ │ └── export-radacct.py │ │ ├── radiusparse_aaadump.lua │ │ ├── radiusparse_alepo.lua │ │ ├── radiusparser-radacct.lua │ │ ├── radiusparser1.lua │ │ └── radiusparser2.lua ├── libs │ ├── README.md │ ├── geodb │ │ ├── README.md │ │ ├── compile_geolite2.lua │ │ ├── compile_ip2loc.lua │ │ └── helpers │ │ │ ├── csv.lua │ │ │ ├── ip6.lua │ │ │ ├── ipprefixdb.lua │ │ │ ├── querytool.lua │ │ │ └── tris_leveldb.lua │ ├── ipprefixdb │ │ ├── README.md │ │ ├── helpers │ │ │ ├── querytool.lua │ │ │ └── tris_leveldb.lua │ │ ├── ip6.lua │ │ ├── ipprefixdb.lua │ │ └── test.lua │ └── leveldb │ │ ├── README.md │ │ ├── test.lua │ │ └── tris_leveldb.lua ├── skeletons │ ├── README.md │ ├── alert_monitor.lua │ ├── cg_monitor.lua │ ├── engine_monitor.lua │ ├── filex_monitor.lua │ ├── flow_tracker.lua │ ├── fts_monitor.lua │ ├── input_filter.lua │ ├── message_monitor.lua │ ├── new_alert_group.lua │ ├── new_counter_group.lua │ ├── new_resource_group.lua │ ├── packet_storage.lua │ ├── protocol_handler.lua │ ├── reassembly_handler.lua │ ├── resource_monitor.lua │ ├── session_monitor.lua │ └── simple_counter.lua ├── techniques │ ├── README.md │ ├── ac-httplog.lua │ ├── broadcast-state.lua │ ├── httplog.lua │ ├── latloss.lua │ ├── mrumap │ │ ├── README.md │ │ ├── mrumap.lua │ │ └── test.lua │ ├── queue.lua │ ├── re2http.lua │ └── tagflow.lua └── tutorial │ ├── README.md │ ├── tutorial1 │ ├── README.md │ ├── hello.lua │ └── tutorial.pcap │ ├── tutorial2 │ ├── 16minutes.pcap │ ├── README.md │ └── pktlen.lua │ ├── tutorial3 │ ├── README.md │ └── dnp3_tcp.lua │ └── tutorial4 │ ├── README.md │ ├── csv.lua │ ├── step1.lua │ ├── step2.lua │ ├── step3.lua │ └── urlhaus.txt └── trp ├── INDEX.md ├── README.md ├── apt1-detect ├── README.md ├── dnss.txt ├── ip-ranges.txt ├── md5s.txt ├── search_fqdn.rb ├── search_fqdn_adv.rb ├── search_keyspace.rb ├── search_md5.rb ├── search_md5_adv.rb └── search_text.rb ├── cert-extract ├── README.md └── certx.rb ├── cert-search ├── README.md └── eccerts.rb ├── cginfo ├── README.md ├── cginfo.rb └── cginfoall.rb ├── elephantflows └── ftracker.rb ├── flows ├── README.md ├── flows_2.rb ├── pcap-for-flowid.rb ├── pcap-for-resourceid.rb ├── pcap_allflows.rb ├── query-by-flowid.rb ├── query-flow-file_zmq.rb ├── query-flow_zmq.rb └── query_pdp.rb ├── fts └── README.md ├── getpcap ├── README.md ├── daypcaps.rb ├── getpackets.rb └── getpackets2.rb ├── getvolume ├── README.md ├── get_total_volume.rb └── getvolume_zmq.rb ├── helloworld ├── README.md └── hello.rb ├── helpers └── model_utils.rb ├── hourlystats ├── README.md ├── hourlystats.rb └── hourlystats2.rb ├── icsissl ├── README.md └── checknotary.rb ├── ids ├── README.md └── save-pcap.rb ├── ioc-sweeper ├── 469aed6f-941c-4a1e-b471-3a3e80cbcc2e.ioc ├── README.md └── iocsweep.rb ├── keyspace ├── README.md └── active_keys.rb ├── proto-tree └── ptree.rb ├── python ├── README.md ├── cginfo.py ├── counter_group_topper_request.py ├── counter_item_request.py ├── top_internal_external_hosts.py ├── trp.proto └── trp_pb2.py ├── resources └── sweep_intel.rb ├── stats └── cistats.rb ├── toppers ├── README.md ├── topper_trend.rb └── toppers_zmq.rb ├── trp.proto ├── trp_proto.md └── youtube ├── README.md ├── youtube_titles.rb └── youtube_vids.rb /README.md: -------------------------------------------------------------------------------- 1 | Welcome to Trisul Scripts Repository 2 | ==================================== 3 | 4 | Trisul Network Analytics https://www.trisul.org offers a scriptable platform upon which you can build your own network and security analysis tools. 5 | 6 | There are two APIs available. 7 | 8 | > The JA3 TLS Fingerprints database has now moved to a new repository ja3prints 9 | 10 | 11 | ## LUA API 12 | ----------- 13 | 14 | Lua scripts are used to customize the Trisul Real Time analytics engine. 15 | 16 | The LUA API is documented at https://www.trisul.org/docs/lua 17 | 18 | 19 | ## TRP API 20 | ----------- 21 | 22 | Trisul Remote Protocol use scripts in Ruby, Javascript, or Python to query and hunt through historical data. 23 | 24 | The TRP API is documented at https://docs.trisul.org/docs/trp/ 25 | 26 | -------------------------------------------------------------------------------- /bash/README.md: -------------------------------------------------------------------------------- 1 | # tool_qstreamflow helper scripts 2 | 3 | 4 | As far as possible use the scripts provided with Trisul Network Analytics IPDR 5 | in the data directory `/usr/local/share/trisul-hub` 6 | 7 | These scripts are intended for illustration of tool_qstreamflow 8 | 9 | 10 | 11 | See *man tool_qstreamflow* 12 | -------------------------------------------------------------------------------- /lua/INDEX.md: -------------------------------------------------------------------------------- 1 | LUA API - samples 2 | ------------------ 3 | 4 | LUA allows you to plug into various points in the Trisul C engine. 5 | 6 | 7 | This directory consists of a number of working samples. 8 | 9 | 10 | All the features of native C plugins are automatically available 11 | to the LUA Plugins as well. These include statistical sketching (top-n, cardinatliy) traffic monitoring flow taggers, packet searches etc. 12 | 13 | File | Path |Description 14 | --- | --- |--- 15 | socialalert.lua|forntend/alerts|Generates an ALERT when you access social networks like Facebook,Twitter 16 | tcphdr.lua|buffer|Prints the TCP header - does not actually meter anything 17 | rstcounter.lua|forntend/counters|Count Number of RST packets seen 18 | hello.lua|hello|Basic working script, just prints hello 19 | hello2.lua|hello|Calls a bunch of methods on T.host inside onload 20 | httpsvr.lua|httpserver|Counts HTTP traffic per HTTP Server 21 | pktlen.lua|packetlen|orking sample adds a new counter group called "Packet Length" 22 | ac-httplog.lua|techniques|HTTP Logger , this time using Aho-Corasik T.ac(..) 23 | httplog.lua|techniques|A customizable HTTP logger, correlates requests and responses 24 | queue.lua|techniques|simple way to implement queues and double queues 25 | re2http.lua|techniques|Use RE2 regex to capture select HTTP headers and log them 26 | tagflow.lua|techniques|Tags user agent with old java verson 27 | -------------------------------------------------------------------------------- /lua/README.md: -------------------------------------------------------------------------------- 1 | Trisul LUA API Samples 2 | ====================== 3 | 4 | The Trisul LUA API allows you to program the Trisul real time network analytics engine. 5 | 6 | This repository contains a number of working samples demonstrating various types of scripts. 7 | 8 | Where do I start? 9 | ----------------- 10 | 11 | The first three places you need to know are : 12 | 13 | 1. The [Trisul LUA API Documentation](http://www.trisul.org/docs/lua) 14 | 2. The `tutorials` directory contains scripts that are used in the [Trisul LUA tutorials](http://www.trisul.org/doca/lua/tutorial1.html) 15 | 3. The `skeletons` directory contains well commented skeletons of various types of scripts you can use to get started. 16 | 17 | 18 | ### Directories 19 | 20 | Next, the actual scripts are split into two directories. 21 | 22 | In Trisul, LUA scripts that run on the "fast" path are called Frontend scripts and those that run on the "metric streams" are called Backend scripts. For more see [Frontend vs Backend](http://www.trisul.org/docs/lua/basics.html#frontend_and_backend_scripts) 23 | 24 | The directories here are organized that way 25 | 26 | 1. Directory `frontend` - **Frontend scripts** Contains samples of frontend scripts. Examples are TCP Reassembly, HTTP File Reconstruction etc 27 | 2. Directory `backend` - **Backend scripts** Contains samples of flow monitors, passive DNS, etc 28 | 29 | 30 | Getting help 31 | ------------ 32 | 33 | Create an issue here on Github 34 | -------------------------------------------------------------------------------- /lua/backend_scripts/alerts/README.md: -------------------------------------------------------------------------------- 1 | Print blacklist alerts 2 | ============= 3 | 4 | -------------------------------------------------------------------------------- /lua/backend_scripts/alerts/alert_filter.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- alert_filter.lua skeleton 3 | -- 4 | -- TYPE: BACKEND SCRIPT 5 | -- PURPOSE: Listen to alert activity 6 | -- DESCRIPTION: Process streaming alerts as they come in and also plug into 7 | -- the "flush" activity when alerts are forwarded to the 8 | -- Hub nodes for storage. 9 | -- 10 | 11 | TrisulPlugin = { 12 | 13 | -- the ID block, you can skip the fields marked 'optional ' 14 | -- 15 | id = { 16 | name = " Badfellas alert group Monitor", 17 | description = "Process alert events in the badfellas alert group ", -- optional 18 | author = "Unleash", -- optional 19 | version_major = 1, -- optional 20 | version_minor = 0, -- optional 21 | }, 22 | 23 | -- WHEN CALLED : your LUA script is loaded into Trisul 24 | onload = function() 25 | end, 26 | 27 | -- WHEN CALLED : your LUA script is unloaded / detached from Trisul 28 | onunload = function() 29 | end, 30 | 31 | -- the alert_monitor block 32 | -- you only need to define the functions you need 33 | alert_monitor = { 34 | 35 | 36 | -- which alert group do you want to monitor 37 | -- each alert group is identified by a GUID login as admin > profile > Alert groups to view 38 | alert_guid = '{5E97C3A3-41DB-4E34-92C3-87C904FAB83E}', 39 | 40 | 41 | 42 | 43 | 44 | 45 | -- WHEN CALLED: when each alert is flushed to the hub node (default every 60 secs) 46 | -- return false if you dont want to save this alert, true to save 47 | flushfilter = function(engine, alert) 48 | -- Block the ALIENVAULT alert type 49 | if alert:sigid() == "ALIENVAULT" then 50 | return false; 51 | else 52 | return true; 53 | end 54 | end, 55 | 56 | 57 | }, 58 | 59 | } 60 | -------------------------------------------------------------------------------- /lua/backend_scripts/alerts/print_alerts.lua: -------------------------------------------------------------------------------- 1 | -- print_alerts.lua 2 | -- 3 | -- doesnt do much , just prints all alert details to screen 4 | -- 5 | -- Run Trisul in "nodemon" (note its not nodaemon ! ) mode so 6 | -- the terminal is available for printing 7 | -- 8 | -- "trisulctl_probe testbench run " should do it 9 | -- 10 | 11 | TrisulPlugin = { 12 | 13 | id = { 14 | name = "alert printer", 15 | description = "sample to just print alerts" 16 | }, 17 | 18 | 19 | alert_monitor = { 20 | 21 | alert_guid = '{5E97C3A3-41DB-4e34-92C3-87C904FAB83E}', 22 | 23 | -- a new alert was seen - print all details to screen 24 | -- 25 | onnewalert = function(engine,alert) 26 | 27 | 28 | print("timestamp ".. alert:timestamp()) 29 | print("date and time ".. os.date('%c', alert:timestamp())) 30 | print("source_ip ".. alert:source_ip()) 31 | print("source_port ".. alert:source_port()) 32 | print("dest ip ".. alert:destination_ip()) 33 | print("dest port ".. alert:destination_port()) 34 | print("sigid ".. alert:sigid()) 35 | print("class ".. alert:classification()) 36 | print("priority ".. alert:priority()) 37 | print("message ".. alert:message()) 38 | 39 | end, 40 | 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /lua/backend_scripts/alienvault-otx/avhit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trisulnsm/trisul-scripts/bef85dea72016ef58a98246649c4cf7c4f3faca5/lua/backend_scripts/alienvault-otx/avhit.png -------------------------------------------------------------------------------- /lua/backend_scripts/alienvault-otx/check-leveldb.lua: -------------------------------------------------------------------------------- 1 | -- check IOC against a levelDB database 2 | -- 3 | -- TYPE: BACKEND SCRIPT 4 | -- 5 | 6 | local LDB=require'tris_leveldb' 7 | 8 | TrisulPlugin = { 9 | 10 | -- id block 11 | -- 12 | id = { 13 | name = "Intel IOC Check", 14 | description = "Check all items for hit in a LevelDB database ", 15 | }, 16 | 17 | onload = function() 18 | 19 | end, 20 | 21 | -- dont store the Intel harvested in backend, save space 22 | resource_monitor = { 23 | 24 | resource_guid = '{EE1C9F46-0542-4A7E-4C6A-55E2C4689419}', 25 | 26 | onbeginflush=function(engine) 27 | T.leveldb = LDB.new() 28 | local ok,errmsg=T.leveldb:open("/usr/local/share/trisul-probe/plugins/trisul-intel.leveldb."..engine:instanceid()) 29 | if not ok then 30 | T.logerror("Unable to open the leveldb file e="..errmsg) 31 | T.leveldb=nil 32 | end 33 | 34 | 35 | end, 36 | 37 | 38 | onendflush=function() 39 | if T.leveldb then T.leveldb:close() end 40 | end, 41 | 42 | -- 43 | -- always return false from flushfilter 44 | -- dont want to save 45 | -- 46 | flushfilter = function(engine, resource) 47 | if not T.leveldb then return false end 48 | 49 | local hit = T.leveldb:get(resource:label()) 50 | if hit then 51 | engine:add_alert("{B5F1DECB-51D5-4395-B71B-6FA730B772D9}", 52 | resource:flow():id(), 53 | "ALIENVAULT-HIT", 54 | 1, 55 | "Resource ".. resource:uri().. 56 | " IOC "..resource:label().. 57 | " hit an AlientVault OTX IOC. JSON".. hit) 58 | end 59 | 60 | return false 61 | end, 62 | 63 | }, 64 | } 65 | 66 | -------------------------------------------------------------------------------- /lua/backend_scripts/alienvault-otx/otx2leveldb.rb: -------------------------------------------------------------------------------- 1 | require 'faraday' 2 | require 'json' 3 | require 'leveldb' 4 | 5 | 6 | raise "Usage: otx2leveldb.rb apikey output-leveldb-path" unless ARGV.length == 2 7 | 8 | apikey = ARGV[0] 9 | outputdb = ARGV[1] 10 | server = "https://otx.alienvault.com" 11 | 12 | 13 | conn = Faraday.new(:url => server) do |faraday| 14 | faraday.request :url_encoded # form-encode POST params 15 | faraday.adapter Faraday.default_adapter # make requests with Net::HTTP 16 | end 17 | 18 | 19 | # validation - check API key validity 20 | url= "/api/v1/users/me" 21 | response = conn.get do |req| 22 | req.url url 23 | req.headers['X-OTX-API-KEY'] = apikey 24 | end 25 | 26 | unless JSON.parse(response.body)["user_id"] 27 | print("Invalid api key response from #{server} : #{apikey}\n") 28 | return 29 | end 30 | 31 | 32 | # download all the subscribed pulses 33 | response = conn.get do |req| 34 | req.url "/api/v1/pulses/subscribed" 35 | req.headers['X-OTX-API-KEY'] = apikey 36 | #req.params={modified_since:Time.now-86400} 37 | end 38 | rows = [] 39 | responses = JSON.parse(response.body) 40 | results = responses["results"] 41 | 42 | 43 | # write to level DB key => JSON of alienvault JSON pulse 44 | db = LevelDB::DB.new(outputdb) 45 | results.each do |data| 46 | pulse = {pulse_id:data["id"],pulse_name:data["name"],tlp:data["tlp"]} 47 | data["indicators"].each_with_index do |i,idx| 48 | db.put i.delete("indicator"),pulse.merge(i).to_json() 49 | end 50 | end 51 | 52 | 53 | # Dump keys to console 54 | db.each do |k,v| 55 | print("#{k}\n") 56 | end 57 | 58 | db.close() 59 | -------------------------------------------------------------------------------- /lua/backend_scripts/blacklist-keys/README.md: -------------------------------------------------------------------------------- 1 | # Blacklist 2 | 3 | Uses the Trisul Lua `onnewkey` API method to very efficiently check for blacklisted keys from any counter group. 4 | 5 | 6 | ## Shadowhammer 7 | 8 | The mac.lua script shows how you can trigger a real time alert whenever a MAC address from the [Operation ShadowHammer attack](https://securelist.com/operation-shadowhammer/89992/) shows up. 9 | 10 | 11 | The `onnewkey` method from the Trisul LUA Script API](https://trisul.org/docs/lua/cg_monitor.html#function_onnewkey) is used to check for newly seen keys. 12 | Behind the scenes Trisul uses a efficient first-seen algorithm to dramatically reduce the frequency of this method. 13 | Hence this has negligible CPU usage requirements. 14 | 15 | 16 | ## Adapt 17 | 18 | You can adapt this to check any counter group key, IP addresses, HTTP Host names, TLS SNI, Cipher suits, MAC addresses, Countries, ASN, etc. 19 | All you have to do is replace the GUID in 20 | 21 | 22 | ````lua 23 | 24 | counter_guid = "" 25 | 26 | ```` 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /lua/backend_scripts/blacklist-keys/mac.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- mac.lua 3 | -- 4 | -- TYPE: BACKEND SCRIPT 5 | -- PURPOSE: Use static blacklist for MAC addresses 6 | -- DESCRIPTION: Alert when a MAC address from blacklist shows up 7 | -- 8 | 9 | Blacklist = require'shadowhammer-blacklist' 10 | 11 | 12 | TrisulPlugin = { 13 | 14 | id = { 15 | name = "MAC blacklist", 16 | description = "plugin to MAC counter group and watch for onnewkey", 17 | }, 18 | 19 | -- cg_monitor block 20 | -- 21 | cg_monitor = { 22 | 23 | -- monitor counter group MAC 24 | counter_guid = "{4B09BD22-3B99-40FC-8215-94A430EA0A35}", 25 | 26 | -- As soon as a new key is seen , new keys repeat every X hours 27 | onnewkey = function(engine, timestamp, key) 28 | 29 | local m = Blacklist[key] 30 | if m then 31 | 32 | T.logdebug("Found MAC in user specified Blacklist "..key) 33 | print("Found MAC in user specified Blacklist "..key) 34 | 35 | engine:add_alert("{B5F1DECB-51D5-4395-B71B-6FA730B772D9}" , 36 | nil,"FireHOL",2,"Found alert in FireHOL range "..tostring(m)) 37 | end 38 | end, 39 | 40 | }, 41 | } 42 | 43 | -------------------------------------------------------------------------------- /lua/backend_scripts/cisco-umbrella-1m/README.md: -------------------------------------------------------------------------------- 1 | Using Cisco OpenDNS Umbrella 1M List to mark flows 2 | ============ 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /lua/backend_scripts/critical-stack/compile-cs.lua: -------------------------------------------------------------------------------- 1 | -- compiles a Critical-Stack intel data file 2 | -- created for Bro-IDS into a LevelDB database suitable for TrisulNSM 3 | 4 | local LDB=require'tris_leveldb' 5 | 6 | if #arg ~= 2 then 7 | print("Usage : luajit compile_cs.lua master-publc.bro.dat critical-stack.trisul.0") 8 | return 9 | end 10 | 11 | local input_file = arg[1] 12 | local output_leveldb = arg[2] 13 | 14 | -- open dn 15 | local db1 = LDB.new() 16 | db1:open(output_leveldb) 17 | 18 | -- Process each line 19 | print("Processing ".. input_file) 20 | local nitems=0 21 | local f = io.open(input_file) 22 | for line in f:lines() do 23 | if line:find("%s*#$") == nil then 24 | local _,_,k = line:find("(%S+)") 25 | db1:put(k, line:gsub("\t"," ") ) 26 | nitems = nitems + 1 27 | end 28 | end 29 | 30 | db1:close() 31 | 32 | print("Compiled "..nitems.." indicators into output database ".. output_leveldb) 33 | 34 | -------------------------------------------------------------------------------- /lua/backend_scripts/critical-stack/critical-stack-checker.lua: -------------------------------------------------------------------------------- 1 | -- TYPE: BACKEND SCRIPT 2 | -- PURPOSE: Checks harvested Indicators against a CriticalStack database 3 | -- 4 | 5 | local LDB=require'tris_leveldb' 6 | 7 | TrisulPlugin = { 8 | 9 | -- id block 10 | -- 11 | id = { 12 | name = "Critical-Stack check", 13 | description = "Check all harvested IOC in leveldb", 14 | }, 15 | 16 | onload = function() 17 | T.LDB = nil 18 | T.permanent_error=false 19 | end, 20 | 21 | -- dont store the Intel harvested in backend, save space 22 | resource_monitor = { 23 | 24 | resource_guid = '{EE1C9F46-0542-4A7E-4C6A-55E2C4689419}', 25 | 26 | -- 27 | -- do you want to save these Harvested resources on the Hub 28 | -- 29 | flushfilter = function(engine, resource) 30 | 31 | if T.permanent_error then return false end 32 | 33 | if T.LDB == nil then 34 | T.LDB = LDB.new() 35 | local compiled_db = T.env.get_config("App>DataDirectory") .. "/plugins/critical-stack.trisul." .. engine:id() 36 | local f,err= T.LDB:open( compiled_db ) 37 | if not f then 38 | T.logerror("Error opening critical stack LevelDB err="..err) 39 | T.logerror("DB file="..compiled_db) 40 | T.permanent_error=true 41 | return false 42 | else 43 | T.loginfo("Opened successfully "..compiled_db) 44 | end 45 | end 46 | 47 | local match_val = T.LDB:get(resource:label()) 48 | if match_val then 49 | engine:add_alert("{B5F1DECB-51D5-4395-B71B-6FA730B772D9}", -- user alerts : group 50 | resource:flow():id(), 51 | "CRIT-STACK", 52 | 2, 53 | match_val ) 54 | end 55 | return false 56 | end, 57 | }, 58 | } 59 | 60 | -------------------------------------------------------------------------------- /lua/backend_scripts/dyndns-alert/dynamic-dns.txt: -------------------------------------------------------------------------------- 1 | 3d-game.com #dtdns.com 2 | 4irc.com #dtdns.com 3 | b0ne.com #dtdns.com 4 | bbsindex.com #dtdns.com 5 | chatnook.com #dtdns.com 6 | darktech.org #dtdns.com 7 | denani.si #afraid.org 8 | denaros.com.au #afraid.org 9 | dena-sudrajat.us #afraid.org 10 | dencoliensales.com #afraid.org 11 | dendi-cyber.org #afraid.org 12 | dendox.org #afraid.org 13 | fitnessdieta.com #afraid.org 14 | gearenergy.ca #afraid.org 15 | gearenergy.com #afraid.org 16 | gearhardtminecraft.com #afraid.org 17 | gearplanetary.com #afraid.org 18 | geasspvp.tk #afraid.org 19 | geban.com.br #afraid.org 20 | gebish.org #afraid.org 21 | geborgen-geboren.ch #afraid.org 22 | geborgengeboren.ch #afraid.org 23 | gebrbumann.ch #afraid.org 24 | gebyarmp3.com #afraid.org 25 | gecco.org.za #afraid.org 26 | geckobungalow.com #afraid.org 27 | geckocrew.com #afraid.org 28 | gecko.rs #afraid.org 29 | -------------------------------------------------------------------------------- /lua/backend_scripts/dyndns-alert/dyndns-alert.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- Alert on Dynamic DNS 3 | -- 4 | -- TYPE: BACKEND SCRIPT 5 | -- PURPOSE: Watches DNS resource stream and does suffix matching against known partial doms 6 | -- requires 7 | -- 1. trie.lua (basic trie to store DNS names) 8 | -- 2. dynamic-dns.txt (a basic feed file) 9 | -- 10 | TrisulPlugin = { 11 | 12 | -- id block 13 | -- 14 | id = { 15 | name = "Alert on Dyndns", 16 | description = "Alert on dyndns and numeric only names", 17 | }, 18 | 19 | -- Load the feed here 20 | onload = function() 21 | 22 | T.dnstrie = require'trie' 23 | 24 | -- load the intel 25 | -- each line in this format -- domain # provider (3d-game.com #dtdns.com) 26 | local line_number = 1 27 | for line in io.lines("dynamic-dns.txt") do 28 | local domain_name, provider = line:match("%s*(%S+)%s*#(.*)") 29 | T.dnstrie:add(domain_name, provider .. " intel source line ".. line_number ); 30 | line_number = line_number + 1 31 | end 32 | 33 | T.log("Loaded ".. line_number .. " dynamic domains from intel file "); 34 | 35 | end, 36 | 37 | -- resource_monitor block 38 | -- 39 | resource_monitor = { 40 | 41 | -- DNS resources 42 | resource_guid = '{D1E27FF0-6D66-4E57-BB91-99F76BB2143E}', 43 | 44 | -- WHEN CALLED : a new resource is seen (immediately) 45 | onnewresource = function(engine, resource ) 46 | local hit = T.dnstrie:get( resource:uri()) 47 | if hit then 48 | T.log("Dynamic DNS hit for ".. resource:uri() ) 49 | engine:add_alert( "{F69C2462-ECEA-45B8-B1CB-F90342D37A4F}", 50 | resource:flow():id(), 51 | hit, 52 | 1, 53 | "Request for dyndns resource "..resource:uri()) 54 | end 55 | end, 56 | 57 | }, 58 | 59 | } 60 | -------------------------------------------------------------------------------- /lua/backend_scripts/dyndns-alert/trie.lua: -------------------------------------------------------------------------------- 1 | -- Simple trie for DNS 2 | -- inspired from Kubernetes at https://raw.githubusercontent.com/kubernetes/contrib/master/ingress/controllers/nginx/lua/trie.lua 3 | -- 4 | 5 | local _M = {} 6 | 7 | local mt = { 8 | __index = _M 9 | } 10 | 11 | 12 | function _M.new() 13 | local t = { } 14 | return setmetatable(t, mt) 15 | end 16 | 17 | function _M.add(t, domain, source_intel ) 18 | 19 | -- reverse the domain and split by . 20 | local l = t 21 | for p in domain:reverse():gmatch("[a-zA-Z0-9\\-]+") do 22 | if not l[p] then 23 | l[p] = {} 24 | l.__value=nil 25 | end 26 | l = l[p] 27 | end 28 | l.__value = source_intel 29 | end 30 | 31 | function _M.get(t, key) 32 | 33 | 34 | -- this may be nil 35 | local l = t 36 | 37 | 38 | -- reverse the domain and split by . 39 | local l = t 40 | for p in key:reverse():gmatch("[a-zA-Z\\-]+") do 41 | if l[p] then 42 | l = l[p] 43 | else 44 | break 45 | end 46 | end 47 | 48 | -- may be nil 49 | return l.__value 50 | end 51 | 52 | return _M 53 | -------------------------------------------------------------------------------- /lua/backend_scripts/elasticsearch/README.md: -------------------------------------------------------------------------------- 1 | Send network flows to elastisearch 2 | ============= 3 | 4 | 1. es_curl_flow.lua 5 | ------------- 6 | Send network flows to elastic search using curl linux tool 7 | 8 | 9 | 10 | 2. es_socket_flow.lua 11 | -------------- 12 | Send netflowk flows to elasticsearch using elasticsearch-lua plugin 13 | 14 | https://github.com/DhavalKapil/elasticsearch-lua 15 | 16 | **Prerequisites** 17 | 18 | 1. elasticsearch-lua 19 | 20 | -------------------------------------------------------------------------------- /lua/backend_scripts/enginemonitor/README.md: -------------------------------------------------------------------------------- 1 | Engine-Monitor scripts 2 | ====== 3 | 4 | 5 | Scripts that run at start and end of each stream window. 6 | 7 | See (Engine Monitor API)[https://www.trisul.org/docs/lua/engine_monitor.html] 8 | 9 | 10 | >> Like all Backend Scripts you can install and uninstall and edit script on a Live system. 11 | 12 | Two scripts are included here. 13 | 14 | 15 | 1. snmp.lua : Simple SNMP poller that adds metrics for specific ports (hardcoded in lua script) 16 | 2. snmp_port_bw.lua : A more sophisticated version that reads a SQLITE database for ports, agents, and community (SNMPv2) 17 | 18 | 19 | Usage 20 | ===== 21 | 22 | Easiest way : Just copy the scripts to the `local-lua` directory on the probe. The directory is typically at `/usr/local/var/lib/trisul/domain0/probe0/context0/config/local-lua` 23 | 24 | For other deployment options see ("Installing LUA Scripts")[https://www.trisul.org/docs/lua/basics.html#installing_and_uninstalling] -------------------------------------------------------------------------------- /lua/backend_scripts/enginemonitor/snmp.lua: -------------------------------------------------------------------------------- 1 | -- .lua 2 | -- 3 | -- snmp.lua 4 | -- Hooks on to the 1 minute ontimer() mechanism 5 | -- to query SNMP and feed counters into Trisul 6 | -- 7 | 8 | -- // {9781db2c-f78a-4f7f-a7e8-2b1a9a7be71a} 9 | -- define_guid(<>, 10 | -- 0x9781db2c, 0xf78a, 0x4f7f, 0xa7, 0xe8, 0x2b, 0x1a, 0x9a, 0x7b, 0xe7, 0x1a); 11 | 12 | function get_if_octets( dir, port_ifindex) 13 | local oid = (dir == "in") and 6 or 10 14 | local h = io.popen("snmpget -v2c -c 'community' 172.16.0.1 .1.3.6.1.2.1.31.1.1.1."..oid.."."..port_ifindex) 15 | local val = h:read("*a") 16 | print("val = "..val) 17 | h:close() 18 | local ctrval = val:match("(%d+)%s*$") 19 | return tonumber(ctrval); 20 | 21 | end 22 | 23 | TrisulPlugin = { 24 | 25 | id = { 26 | name = "SNMP Interface", 27 | description = "Per Interface Stats : Key Agent:IfIndex ", 28 | author = "Unleash", 29 | version_major = 1, 30 | version_minor = 0, 31 | }, 32 | 33 | 34 | onload = function() 35 | end, 36 | 37 | 38 | onunload = function() 39 | end, 40 | 41 | engine_monitor = { 42 | onbeginflush = function(engine, tv) 43 | 44 | local ports = { 1,2,3,4,23,24,625}; 45 | 46 | for i,port in ipairs(ports) do 47 | 48 | local val_in = get_if_octets("in", port) 49 | local val_out = get_if_octets("out", port) 50 | 51 | engine:update_counter( "{9781db2c-f78a-4f7f-a7e8-2b1a9a7be71a}", 52 | "if-"..port, 0, val_in + val_out ); 53 | 54 | engine:update_counter( "{9781db2c-f78a-4f7f-a7e8-2b1a9a7be71a}", 55 | "if-"..port, 1, val_in ); 56 | 57 | engine:update_counter( "{9781db2c-f78a-4f7f-a7e8-2b1a9a7be71a}", 58 | "if-"..port, 2, val_out ); 59 | 60 | T.log(T.K.loglevel.DEBUG, "SNMP vals tv="..(tv+10).."port="..port.." in="..val_in.." out="..val_out); 61 | end 62 | 63 | end 64 | }, 65 | 66 | 67 | } 68 | 69 | -------------------------------------------------------------------------------- /lua/backend_scripts/enginemonitor/snmp_cg.lua: -------------------------------------------------------------------------------- 1 | -- .lua 2 | -- 3 | -- snmp.lua 4 | -- Hooks on to the 1 minute ontimer() mechanism 5 | -- to query SNMP and feed counters into Trisul 6 | -- 7 | 8 | -- // {9781db2c-f78a-4f7f-a7e8-2b1a9a7be71a} 9 | -- define_guid(<>, 10 | -- 0x9781db2c, 0xf78a, 0x4f7f, 0xa7, 0xe8, 0x2b, 0x1a, 0x9a, 0x7b, 0xe7, 0x1a); 11 | 12 | TrisulPlugin = { 13 | 14 | id = { 15 | name = "SNMP Interface", 16 | description = "Per Interface Stats : Key Agent:IfIndex ", 17 | author = "Unleash", 18 | version_major = 1, 19 | version_minor = 0, 20 | }, 21 | 22 | countergroup = { 23 | control = { 24 | guid = "{9781db2c-f78a-4f7f-a7e8-2b1a9a7be71a}", 25 | name = "SNMP-Interface", 26 | description = "Traffic using SNMP input ", 27 | bucketsize = 60, 28 | }, 29 | 30 | meters = { 31 | { 0, T.K.vartype.DELTA_RATE_COUNTER, 20, "bytes", "Total BW", "Bps" }, 32 | { 1, T.K.vartype.DELTA_RATE_COUNTER, 20, "bytes", "In Octets", "Bps" }, 33 | { 2, T.K.vartype.DELTA_RATE_COUNTER, 20, "bytes", "Out Octets", "Bps" }, 34 | }, 35 | }, 36 | 37 | 38 | } 39 | 40 | -------------------------------------------------------------------------------- /lua/backend_scripts/firehol/README.md: -------------------------------------------------------------------------------- 1 | FireHOL.lua - Trisul Plugin to alert on FireHOL matches 2 | ================================= 3 | 4 | This LUA script tracks all newly seen IPs against the excellent FireHOL Cyber Crime Feeds at http://iplists.firehol.org/ 5 | 6 | 7 | Using 8 | ----- 9 | 10 | This is also available as a Trisul App for free download, but if you want to use this script separately 11 | 12 | Download the FireHOL IP Ranges file and put it in this directory 13 | 14 | ```` 15 | cd /usr/local/share/trisul-probe 16 | curl -O https://iplists.firehol.org/files/firehol_level1.netset 17 | ```` 18 | 19 | Place the 2 lua scripts in the local-lua directory 20 | 21 | ```` 22 | 23 | cp firehol.lua /usr/local/var/lib/trisul-probe/domain0/probe0/context0/config/local-lua 24 | cp iprangemap.lua /usr/local/var/lib/trisul-probe/domain0/probe0/context0/config/local-lua 25 | 26 | ```` 27 | 28 | The script will be automatically picked up by a LIVE instance of Trisul within 1 minute. 29 | 30 | 31 | Output Alert 32 | ------------ 33 | 34 | When a match is seen you will immediately get an alert. 35 | Log on to Trisul , then go to Alerts > Show All > User Alerts > View Real Time 36 | 37 | -------------------------------------------------------------------------------- /lua/backend_scripts/firehol/firehol.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- firehol.lua 3 | -- 4 | -- TYPE: BACKEND SCRIPT 5 | -- PURPOSE: Scan using FIREHOL IP CRIME list http://iplists.firehol.org/ 6 | -- DESCRIPTION: NOW ! Trisul users can just plugin this script automatically 7 | -- scan all their Traffic against the excellent (low false positive) 8 | -- FIREHOL list 9 | -- The good thing about this list is - if an alert fires there is 10 | -- a high likelyhood you need to take action. 11 | -- 12 | -- DEV NOTE: We use a very fast custom built IP Range Map in LUA to match entire 13 | -- ip spaces. 14 | -- 15 | -- Level 3 alert when IP seen, Level 1 alert when Data transfer happens 16 | -- 17 | 18 | local FH = require'iprangemap' 19 | local FIREHOL_FILENAME = "firehol_level1.netset" 20 | 21 | TrisulPlugin = { 22 | 23 | id = { 24 | name = "FireHOL tracker", 25 | description = "Scans host traffic vs this excellent list ", 26 | }, 27 | 28 | -- load the list 29 | onload = function() 30 | T.fhole = FH.new() 31 | local firehol_intel_file = T.env.get_config("App>DataDirectory") + "/" + FIREHOL_FILENAME 32 | local status,errormsg = T.fhole:load(firehol_intel_file) 33 | if status == false then 34 | T.logerror("Error loading filehol list msg="..errormsg) 35 | return false 36 | end 37 | end, 38 | 39 | -- cg_monitor block 40 | -- 41 | cg_monitor = { 42 | 43 | -- monitor all hosts 44 | counter_guid = "{4CD742B1-C1CA-4708-BE78-0FCA2EB01A86}", 45 | 46 | -- As soon as a new key is seen , new keys repeat every X hours 47 | onnewkey = function(engine, timestamp, key) 48 | local m = fh:lookup(key) 49 | if m then 50 | T.log("Found IP in FireHOL Blacklist"..key) 51 | engine:add_alert("{B5F1DECB-51D5-4395-B71B-6FA730B772D9}" , 52 | nil,"FireHOL",2,"Found alert in FireHOL range "..tostring(m)) 53 | end 54 | end, 55 | 56 | }, 57 | } 58 | 59 | -------------------------------------------------------------------------------- /lua/backend_scripts/firehol/iprangemap.md: -------------------------------------------------------------------------------- 1 | LUA IP range lookup 2 | =================== 3 | 4 | [iprangemap.lua](https://github.com/trisulnsm/trisul-scripts/blob/master/lua/backend_scripts/firehol/iprangemap.lua) is a simple fast lookup library to check an IP address against a database of IP ranges. 5 | 6 | We at Trisul Network Analytics are using this for checking an IP against blacklists. 7 | 8 | 9 | Usage 10 | ----- 11 | 12 | The following example 13 | 1. adds 3 IP subnets 14 | 2. one IP 15 | 3. checks match 16 | 4. prints match 17 | 18 | 19 | Try with LuaJIT 20 | 21 | ````lua 22 | local RMAP=require'iprangemap' 23 | 24 | local rm = RMAP.new() 25 | rm:add("103.229.217.0/24") 26 | rm:add("74.207.22..122") -- bad form , will be rejected 27 | rm:add("180.179.120.65") 28 | rm:add("29.212.22..0/255") 29 | rm:add("23.251.224.0/19") 30 | 31 | rm:resort() -- not needed if you add in sorted order 32 | 33 | rm:dump() 34 | 35 | -- check(ip) true or false 36 | print(rm:check("103.229.217.2") ) -- prints true 37 | print(rm:check("10.29.7.2")) -- prints false 38 | 39 | -- print matching range 40 | local match = rm:lookup("23.251.224.4") 41 | print("Found match=".. tostring(match)) 42 | 43 | ```` 44 | 45 | 46 | 47 | Examples 48 | -------- 49 | 50 | Check how we use this to check the FireHOL ranges in `firehol.lua` 51 | -------------------------------------------------------------------------------- /lua/backend_scripts/flow-tag-enrich/README.md: -------------------------------------------------------------------------------- 1 | # Flow Tag Enrich 2 | 3 | Add the following flow tags 4 | 5 | 1. IP version `[ipver]=version`. Example `tag=[ipver]6` to get IP v6 flows 6 | -------------------------------------------------------------------------------- /lua/backend_scripts/flow-tag-enrich/add_flow_version.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- session_group_monitor.lua skeleton 3 | -- 4 | -- TYPE: BACKEND SCRIPT 5 | -- PURPOSE: session (flow) updates. IP flows are a type of sesssion group 6 | -- DESCRIPTION: Handle flow related streaming metrics and listen to flows as they 7 | -- are flushed to the database (hub) node. 8 | -- 9 | TrisulPlugin = { 10 | 11 | id = { 12 | name = "Add Version to Flow", 13 | description = "Add tag IPv6 or IPv4 flows", 14 | }, 15 | 16 | 17 | -- sg_monitor block 18 | -- sg = session group 19 | sg_monitor = { 20 | 21 | -- that guid refers to IPv4/IPv6 flows (you can skip the session_guid field if you want its the default ) 22 | session_guid = '{99A78737-4B41-4387-8F31-8077DB917336}', -- optional 23 | 24 | -- WHEN CALLED: before a flow is flushed to the Hub node 25 | onflush = function(engine, flow) 26 | if flow:flow():ipa():len()==32 then 27 | flow:add_tag("[ipver]6") 28 | else 29 | flow:add_tag("[ipver]4") 30 | end 31 | end, 32 | 33 | }, 34 | 35 | } 36 | -------------------------------------------------------------------------------- /lua/backend_scripts/flows/README.md: -------------------------------------------------------------------------------- 1 | # Backend Scripts - Flows 2 | 3 | These scripts demonstrates working with flows on the metrics backend. 4 | 5 | ## Index of scripts 6 | 7 | 8 | Filename | What it does | Demonstrates 9 | ---------------------|--------------------------------------------------|---------------------------- 10 | sg1.lua | Prints all new flow keys to the console | Minimal script demonstrates usage of onnewflow 11 | sg2.lua | Write files - one per backend instance - and prints flow details into it | Work with multiple instances of script, file io, global variables 12 | sg3.lua | Working example - prints flow details like sourceip,port,destip,total bytes, interval | Navigating flow object model, writing to files, using string and time formatting | 13 | 14 | 15 | 16 | ## Running 17 | 18 | Place these scripts in these locations and restart trisul-probe . Normally you want to do the first (local-lua) when testing. 19 | 20 | 21 | Node | Directory | Remarks | 22 | -----|-----------|---------| 23 | On Probe | ````/usr/local/var/lib/trisul/domain0/probe0/context0/config/local-lua```` | local-lua scripts apply to that probe and context only. Normally you want to do this when experimenting 24 | On Probe | /usr/local/lib/trisul-probe/plugins/lua | Script loaded for all instances on that node (machine) | 25 | On Hub | /usr/local/lib/trisul-hub/domain0/hub0/profileX/lua | All probes that use that profileX will get the lua script | 26 | 27 | -------------------------------------------------------------------------------- /lua/backend_scripts/flows/ftracker1.lua: -------------------------------------------------------------------------------- 1 | -- FlowTracker_PROD.lua 2 | -- 3 | -- Only tracks UDP flows in subnet 10.10.22.0/24 4 | -- The metric returned is total_bytes. Therefore this tracker snapshots Top-K flows 5 | -- matching this rule. 6 | -- 7 | TrisulPlugin = { 8 | 9 | id = { 10 | name = "FlowTracker-SUBNET1", 11 | description = "Only subnets a,b but excluding production apps", 12 | }, 13 | 14 | 15 | flowtracker = { 16 | 17 | control = { 18 | name = "Subnet1-FT", 19 | description = "UDP flows from 10.10.22 net", 20 | bucketsize = 300, -- 5 minutes streaming window 21 | count = 100 -- 100 top flows meeting criteria 22 | }, 23 | 24 | 25 | -- return the metric you want to track associated with this flow 26 | -- you can examine all fields of the flow and use other factors 27 | -- to compute a number 28 | -- 29 | -- return 0 or nil means this flow isnt of interest and not tracked 30 | -- 31 | getmetric = function(engine,f) 32 | 33 | if f:flow():protocol() == "11" and 34 | f:flow():ipa_readable():match("^10.10.22") or f:flow():ipz_readable():match("^10.10.22") or 35 | then 36 | print("Metric for flow protocol ".. f:flow():protocol()) 37 | return f:az_bytes() + f:za_bytes() 38 | end 39 | 40 | end, 41 | 42 | }, 43 | 44 | 45 | } 46 | 47 | -------------------------------------------------------------------------------- /lua/backend_scripts/flows/ip2location/README.md: -------------------------------------------------------------------------------- 1 | ip2location.lua 2 | =================== 3 | 4 | Script to meter 5 | - Country 6 | - ASN 7 | - Region 8 | - City 9 | 10 | 11 | Add Flow edges to each of them 12 | -------------------------------------------------------------------------------- /lua/backend_scripts/flows/ip2location/cg_ip2loc_asn.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- IP2Location : ASN numbers 3 | -- 4 | -- 5 | TrisulPlugin = { 6 | 7 | -- the ID block, you can skip the fields marked 'optional ' 8 | -- 9 | id = { name = "IP2Loc ASN", }, 10 | 11 | 12 | -- countergroup block 13 | -- 14 | countergroup = { 15 | 16 | control = { 17 | guid = "{EF44F11F-B90B-4B24-A9F5-86482C51D125}", 18 | name = "IP2Loc ASN", 19 | bucketsize = 60, 20 | }, 21 | 22 | meters = { 23 | { 0, T.K.vartype.RATE_COUNTER, 10, 0, "Total", "Bps", "Bps" }, 24 | { 1, T.K.vartype.RATE_COUNTER, 10, 0, "Upload To", "Bps", "Bps" }, 25 | { 2, T.K.vartype.RATE_COUNTER, 10, 0, "Download From", "Bps", "Bps" }, 26 | }, 27 | 28 | }, 29 | } 30 | -------------------------------------------------------------------------------- /lua/backend_scripts/flows/ip2location/cg_ip2loc_cc.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- IP2Location : Country code counters 3 | -- 4 | -- 5 | TrisulPlugin = { 6 | 7 | -- the ID block, you can skip the fields marked 'optional ' 8 | -- 9 | id = { name = "IP2Loc Country", }, 10 | 11 | 12 | -- countergroup block 13 | -- 14 | countergroup = { 15 | 16 | control = { 17 | guid = "{F962527D-985D-42FD-91D5-DA39F4D2A222}", 18 | name = "IP2Loc Country", 19 | bucketsize = 60, 20 | }, 21 | 22 | meters = { 23 | { 0, T.K.vartype.RATE_COUNTER, 10, 0, "Total", "Bps", "Bps" }, 24 | { 1, T.K.vartype.RATE_COUNTER, 10, 0, "Upload To", "Bps", "Bps" }, 25 | { 2, T.K.vartype.RATE_COUNTER, 10, 0, "Download From", "Bps", "Bps" }, 26 | }, 27 | 28 | }, 29 | } 30 | -------------------------------------------------------------------------------- /lua/backend_scripts/flows/ip2location/cg_ip2loc_proxy.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- IP2Location : Proxy 3 | -- 4 | -- 5 | TrisulPlugin = { 6 | 7 | -- the ID block, you can skip the fields marked 'optional ' 8 | -- 9 | id = { name = "IP2Loc Proxy", }, 10 | 11 | 12 | -- countergroup block 13 | -- 14 | countergroup = { 15 | 16 | control = { 17 | guid = "{2DCA13EB-0EB3-46F6-CAA2-9989EA904051}", 18 | name = "IP2Loc Proxy", 19 | bucketsize = 60, 20 | }, 21 | 22 | meters = { 23 | { 0, T.K.vartype.RATE_COUNTER, 10, 0, "Total", "Bps", "Bps" }, 24 | { 1, T.K.vartype.RATE_COUNTER, 10, 0, "Upload To", "Bps", "Bps" }, 25 | { 2, T.K.vartype.RATE_COUNTER, 10, 0, "Download From", "Bps", "Bps" }, 26 | }, 27 | 28 | }, 29 | } 30 | -------------------------------------------------------------------------------- /lua/backend_scripts/flows/ip2location/cg_ip2loc_region.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- IP2Location : Region state+city 3 | -- 4 | -- 5 | TrisulPlugin = { 6 | 7 | -- the ID block, you can skip the fields marked 'optional ' 8 | -- 9 | id = { name = "IP2Loc City", }, 10 | 11 | 12 | -- countergroup block 13 | -- 14 | countergroup = { 15 | 16 | control = { 17 | guid = "{E85FEB77-942C-411D-DF12-5DFCFCF2B932}", 18 | name = "IP2Loc City", 19 | bucketsize = 60, 20 | }, 21 | 22 | meters = { 23 | { 0, T.K.vartype.RATE_COUNTER, 10, 0, "Total", "Bps", "Bps" }, 24 | { 1, T.K.vartype.RATE_COUNTER, 10, 0, "Upload To", "Bps", "Bps" }, 25 | { 2, T.K.vartype.RATE_COUNTER, 10, 0, "Download From", "Bps", "Bps" }, 26 | }, 27 | 28 | }, 29 | } 30 | -------------------------------------------------------------------------------- /lua/backend_scripts/flows/longssh-whitelist.lua: -------------------------------------------------------------------------------- 1 | -- LongLivedSSH-exclude hosts in white list 2 | -- 3 | -- The metric returned is duration. Therefore this tracker snapshots Top-K flows 4 | -- Use a whitelist pair to exlude known conversation endpts 5 | -- 6 | local dbg=require'debugger' 7 | TrisulPlugin = { 8 | 9 | id = { 10 | name = "FlowTracker-SSHWHITELIST", 11 | description = "SSH by duration exclude whitelisted pairs ", 12 | }, 13 | 14 | 15 | onload=function() 16 | T.whitelistpairs = { 17 | {"192.168.1.11", "138.68.45.27"} 18 | }; 19 | 20 | end, 21 | 22 | 23 | flowtracker = { 24 | 25 | control = { 26 | name = "SSH-WHITELIST", 27 | description = "ssh duration whitelist", 28 | bucketsize = 300, -- 5 minutes streaming window 29 | count = 100 -- 100 top flows meeting criteria 30 | }, 31 | 32 | 33 | -- return the metric you want to track associated with this flow 34 | -- 35 | -- return 0 or nil means this flow isnt of interest and not tracked 36 | -- 37 | getmetric = function(engine,f) 38 | 39 | local fid = f:flow() 40 | if fid:portz_readable() == "22" then 41 | 42 | for _, p in ipairs(T.whitelistpairs) do 43 | local ip1 = p[1] 44 | local ip2 = p[2] 45 | 46 | if fid:ipa_readable() == ip1 and fid:ipz_readable() == ip2 or 47 | fid:ipz_readable() == ip1 and fid:ipa_readable() == ip2 then 48 | 49 | -- flow is whitelisted ignore 50 | 51 | else 52 | 53 | -- non whitelisted SSH flow , return duration as tracker metric 54 | local start_ts,last_ts = f:time_window() 55 | return last_ts - start_ts 56 | end 57 | end 58 | 59 | end 60 | 61 | end, 62 | 63 | }, 64 | 65 | 66 | } 67 | 68 | -------------------------------------------------------------------------------- /lua/backend_scripts/flows/sg1.lua: -------------------------------------------------------------------------------- 1 | TrisulPlugin = { 2 | 3 | id = { 4 | name = "sess group mon - print flows 1", 5 | description = "prints each invocation of onnewflow Minimal version prints flow id", 6 | }, 7 | 8 | 9 | sg_monitor = { 10 | 11 | session_guid = '{99A78737-4B41-4387-8F31-8077DB917336}', 12 | 13 | onnewflow = function(engine, newflow) 14 | print( newflow:flow():id()) 15 | end, 16 | 17 | 18 | }, 19 | 20 | } 21 | -------------------------------------------------------------------------------- /lua/backend_scripts/flows/sg2.lua: -------------------------------------------------------------------------------- 1 | TrisulPlugin = { 2 | 3 | id = { 4 | name = "sess group mon - prints flows", 5 | description = "writes to a file each new flow ", 6 | }, 7 | 8 | onload = function() 9 | T.flowlogfile = io.open("/tmp/flog."..T.contextid,"w") 10 | T.count=1; 11 | end, 12 | 13 | onunload=function() 14 | T.flowlogfile:close() 15 | end, 16 | 17 | 18 | sg_monitor = { 19 | 20 | session_guid = '{99A78737-4B41-4387-8F31-8077DB917336}', 21 | 22 | onnewflow = function(engine, newflow) 23 | T.flowlogfile:write( ""..T.count.." "..newflow:flow():id().."\n"); 24 | T.count = T.count+1 25 | end, 26 | 27 | }, 28 | 29 | } 30 | -------------------------------------------------------------------------------- /lua/backend_scripts/flows/sg3.lua: -------------------------------------------------------------------------------- 1 | TrisulPlugin = { 2 | 3 | id = { 4 | name = "sess group mon - onflush", 5 | description = "Write to a file full flow details ", 6 | }, 7 | 8 | onload = function() 9 | T.flowlogfile = io.open("/tmp/fdetails."..T.contextid,"w") 10 | T.count=1; 11 | end, 12 | 13 | onunload=function() 14 | T.flowlogfile:close() 15 | end, 16 | 17 | 18 | sg_monitor = { 19 | 20 | session_guid = '{99A78737-4B41-4387-8F31-8077DB917336}', 21 | 22 | onbeginflush = function(engine,ts) 23 | T.flowlogfile:write( "Flushing flows at ts="..ts.." time="..os.date("%c",ts).."\n") 24 | end, 25 | 26 | onflush = function(engine,flw) 27 | 28 | local ftuples = flw:flow() 29 | 30 | local start_tm, end_tm = flw:time_window() 31 | 32 | local fstr = string.format("%5d %2s %-15s %-6s %-15s %-6s %8d %20s %5d secs\n", 33 | T.count, 34 | ftuples:protocol(), 35 | ftuples:ipa_readable(), ftuples:porta_readable(), 36 | ftuples:ipz_readable(), ftuples:portz_readable(), 37 | flw:az_bytes() + flw:za_bytes(), 38 | os.date("%c",start_tm), 39 | end_tm-start_tm 40 | ); 41 | 42 | 43 | T.flowlogfile:write(fstr) 44 | 45 | T.count = T.count + 1 46 | 47 | end, 48 | 49 | onendflush = function(engine) 50 | T.flowlogfile:write( "----------------------------------------\n\n"); 51 | end 52 | 53 | }, 54 | 55 | } 56 | -------------------------------------------------------------------------------- /lua/backend_scripts/flows/ssh_alerts.lua: -------------------------------------------------------------------------------- 1 | -- ssh_alerts.lua 2 | -- 3 | -- monitor flows being flushed - then generate an alert if there is > 15K on each side 4 | -- this could indicate a successful SSH session 5 | -- 6 | -- Also create a new alert group called SSH alerts use that to push it and also push it 7 | -- to the External IDS alert group 8 | -- 9 | -- 10 | TrisulPlugin = { 11 | 12 | id = { 13 | name = "SSH alerts", 14 | description = "When successful SSH happens (simple version)", 15 | }, 16 | 17 | alertgroup = { 18 | control = { 19 | guid = '{0409EAC7-1E60-43D3-C0FA-A87429F99728}', 20 | name = 'SSH Alerts', 21 | description = 'Alerts on SSH login and transfer' 22 | } 23 | }, 24 | 25 | sg_monitor = { 26 | 27 | onflush = function(engine, newflow) 28 | 29 | -- p-0016 appears in flowkey format when ssh involved 30 | -- you can also newflow:flow():porta_readable() = '22' (for port 22) etc 31 | local flowkey = newflow:flow():id() 32 | 33 | if flowkey:match("p-0016") then 34 | 35 | print("On engine "..engine:instanceid().." found ssh session " .. newflow:flow():to_s()); 36 | 37 | local total_bytes = newflow:az_bytes() + newflow:za_bytes() 38 | 39 | if total_bytes < 1000 then 40 | engine:add_alert('{0409EAC7-1E60-43D3-C0FA-A87429F99728}', 41 | newflow:flow():id(), 42 | 'SSHXFER', 43 | 1, 44 | "ssh session transferring ".. total_bytes.." bytes, check it out"); 45 | engine:add_alert_full("{5E97C3A3-41DB-4E34-92C3-87C904FAB83E}", 46 | newflow:flow():id(), 47 | 'SSHXFER', 48 | 'attack', 49 | 1, 50 | "ssh session transferring ".. total_bytes.." bytes, check it out", 51 | "FIRE") 52 | 53 | end 54 | 55 | end 56 | end, 57 | 58 | 59 | }, 60 | 61 | } 62 | -------------------------------------------------------------------------------- /lua/backend_scripts/flowsizes/README.md: -------------------------------------------------------------------------------- 1 | Flow Sizes Analytics 2 | ==================== 3 | 4 | This script measures the relationship between flow sizes and the data transferred. 5 | 6 | 7 | -------------------------------------------------------------------------------- /lua/backend_scripts/flowsizes/flows_size.lua: -------------------------------------------------------------------------------- 1 | -- flows sizes vs volume distribution 2 | -- 3 | TrisulPlugin = { 4 | 5 | id = { 6 | name = "Flowsizes", 7 | description = "flow size analytics", -- optional 8 | }, 9 | 10 | 11 | sg_monitor = { 12 | 13 | onflush = function(engine, flow) 14 | -- packet size into log10 range 15 | local totalbytes = flow:az_bytes() + flow:za_bytes() 16 | if bit.bor(flow:state(),0x09C0) ~= 0 and totalbytes > 0 then 17 | local key = tostring(math.ceil( math.log10(totalbytes))) .. " Dig" 18 | engine:update_counter( "{91D08D08-B846-4C28-1FC9-A2C419DCC605}", key, 0, 1) 19 | engine:update_counter( "{91D08D08-B846-4C28-1FC9-A2C419DCC605}", key, 1, totalbytes) 20 | end 21 | end, 22 | }, 23 | 24 | 25 | countergroup = { 26 | 27 | control = { 28 | guid = "{91D08D08-B846-4C28-1FC9-A2C419DCC605}", 29 | name = "Flow Sizes", 30 | description = "Flow sizes distribution", 31 | bucketsize = 60, 32 | }, 33 | 34 | -- meters table 35 | meters = { 36 | { 0, T.K.vartype.COUNTER, 50, 30, "Flows", "flows", "Flws" }, 37 | { 1, T.K.vartype.COUNTER, 50, 30, "Bytes", "bytes", "Bytes" }, 38 | }, 39 | }, 40 | } 41 | 42 | -------------------------------------------------------------------------------- /lua/backend_scripts/fts/README.md: -------------------------------------------------------------------------------- 1 | FTS Stream Scripts 2 | ================== 3 | 4 | FTS are Full Text Document streams generated by the Trisul Fast Path and made available to you in the Slow Path (called Backend scripts). 5 | 6 | The documentation for samples in the 'fts' dir are at [FTS Monitor Docs](http://trisul.org/docs/lua/fts_monitor.html) . 7 | 8 | The built in FTS Documents are 9 | 10 | 1. HTTP Headers 11 | 2. SSL Certificates 12 | 3. DNS Records 13 | 14 | 15 | The main 'trick' to write effective Trisul LUA scripts on the FTS stream is two fold 16 | 17 | 1. Identify the structure of the documents 18 | 2. Use a Regex or someother method to capture what you want 19 | 20 | To identify the structure of a document, you can either use the Web Trisul UI under Resources > FTS or you can write a simple 21 | script to dump the document on screen. 22 | 23 | 24 | Samples 25 | ------- 26 | 27 | The sample scripts here demonstrate the approach. 28 | 29 | 30 | Note 31 | ---- 32 | 33 | All FTS Scripts work only if you have the corresponding FTS type enabled in trisulProbeConfig.xml 34 | 35 | The config file is usually found in `/usr/local/etc/trisul-probe/domain0/probe0/context0/trisulProbeConfig.xml` 36 | -------------------------------------------------------------------------------- /lua/backend_scripts/fts/c2-x509-fts.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- c2-x509-fts.lua 3 | -- 4 | -- TYPE: BACKEND SCRIPT 5 | -- PURPOSE: Full text search documents streaming 6 | -- DESCRIPTION: Fun script to C2 Channel that transfers chunks of data using a 7 | -- X509 certificate extension "Subject Key Identifier" 8 | -- Writes each chunk to file in streaming manner 9 | -- 10 | 11 | -- helper : to write the Hex to binary file chunk 12 | function string.hex2bin(str) 13 | return (str:gsub('..', function (cc) 14 | return string.char(tonumber(cc, 16)) 15 | end)) 16 | end 17 | 18 | TrisulPlugin = { 19 | 20 | -- id block 21 | -- 22 | id = { 23 | name = "C2-X509 FTS demo", 24 | description = "pull out C2 binariy", 25 | }, 26 | 27 | onload = function() 28 | T.count = 0; 29 | end, 30 | 31 | -- pull out large SubjectKeyIdentifier into "chunk files" 32 | fts_monitor = { 33 | 34 | -- 9E.. refers to SSL Certs FTS (you can get this from Trisul UI) 35 | fts_guid = '{9FEB8ADE-ADBB-49AD-BC68-C6A02F389C71}', 36 | 37 | 38 | -- WHEN CALLED : a new FTS Document is seen 39 | onnewfts = function(engine, fts ) 40 | 41 | local _,_,ski = fts:text():find("X509v3 Subject Key Identifier:%s*(%S+)") 42 | if ski and ski:len() > 32 then 43 | T.count = T.count + 1 44 | local hexski = ski:gsub("[:%s]","") 45 | local outf = io.open("/tmp/c2ski-"..engine:instanceid().."-"..T.count,"w") 46 | outf:write(hexski:hex2bin()) 47 | outf:close() 48 | end 49 | 50 | end, 51 | }, 52 | } 53 | 54 | 55 | -------------------------------------------------------------------------------- /lua/backend_scripts/geoflows/README.md: -------------------------------------------------------------------------------- 1 | # geoflows - use a Geo database to track flow activity 2 | 3 | This script uses the libftrie.so from the [ftrie](https://github.com/trisulnsm/ftrie) project to track the number of active flows per country and city. 4 | 5 | 6 | ## shared library libftrie.so 7 | 8 | File libfrie.so is Ubuntu 16.04 , you can build the ftrie project for your own platform 9 | Copy `libftrie.so` file to the lib directory. 10 | 11 | ```lua 12 | cp libftrie.so /usr/local/lib/trisul-probe/ 13 | ```` 14 | 15 | 16 | -------------------------------------------------------------------------------- /lua/backend_scripts/geoflows/cg_ip2loc_asn.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- IP2Location : ASN numbers 3 | -- 4 | -- 5 | TrisulPlugin = { 6 | 7 | -- the ID block, you can skip the fields marked 'optional ' 8 | -- 9 | id = { name = "IP2Loc ASN", description ="ASN metrics"}, 10 | 11 | 12 | -- countergroup block 13 | -- 14 | countergroup = { 15 | 16 | control = { 17 | guid = "{EF44F11F-B90B-4B24-A9F5-86482C51D125}", 18 | name = "IP2Loc ASN", 19 | description = "Meters ASN", 20 | bucketsize = 60, 21 | }, 22 | 23 | meters = { 24 | { 0, T.K.vartype.RATE_COUNTER, 40, 40, "Total", "Total", "Bps" }, 25 | { 1, T.K.vartype.RATE_COUNTER, 40, 40, "Upload To", "Upload To", "Bps" }, 26 | { 2, T.K.vartype.RATE_COUNTER, 40, 40, "Download From", "Download From", "Bps" }, 27 | { 3, T.K.vartype.COUNTER, 40, 40, "Flows", "Flows", "flows" }, 28 | }, 29 | 30 | }, 31 | } 32 | -------------------------------------------------------------------------------- /lua/backend_scripts/geoflows/cg_ip2loc_cc.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- IP2Location : Country code counters 3 | -- 4 | -- 5 | TrisulPlugin = { 6 | 7 | -- the ID block, you can skip the fields marked 'optional ' 8 | -- 9 | id = { name = "IP2Loc Country", description="Country Metrics"}, 10 | 11 | 12 | -- countergroup block 13 | -- 14 | countergroup = { 15 | 16 | control = { 17 | guid = "{F962527D-985D-42FD-91D5-DA39F4D2A222}", 18 | name = "IP2Loc Country", 19 | description = "Meters Country", 20 | bucketsize = 60, 21 | }, 22 | 23 | meters = { 24 | { 0, T.K.vartype.RATE_COUNTER, 40, 40, "Total", "Total", "Bps" }, 25 | { 1, T.K.vartype.RATE_COUNTER, 40, 40, "Upload To", "Upload To", "Bps" }, 26 | { 2, T.K.vartype.RATE_COUNTER, 40, 40, "Download From", "Download From", "Bps" }, 27 | { 3, T.K.vartype.COUNTER, 40, 40, "Flows", "Flows", "flows" }, 28 | }, 29 | 30 | }, 31 | } 32 | -------------------------------------------------------------------------------- /lua/backend_scripts/geoflows/cg_ip2loc_proxy.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- IP2Location : Proxy 3 | -- 4 | -- 5 | TrisulPlugin = { 6 | 7 | -- the ID block, you can skip the fields marked 'optional ' 8 | -- 9 | id = { name = "IP2Loc Proxy", description="Proxy Metrics"}, 10 | 11 | 12 | -- countergroup block 13 | -- 14 | countergroup = { 15 | 16 | control = { 17 | guid = "{2DCA13EB-0EB3-46F6-CAA2-9989EA904051}", 18 | name = "IP2Loc Proxy", 19 | description = "Meters Proxy", 20 | bucketsize = 60, 21 | }, 22 | 23 | meters = { 24 | { 0, T.K.vartype.RATE_COUNTER, 40, 40, "Total", "Total", "Bps" }, 25 | { 1, T.K.vartype.RATE_COUNTER, 40, 40, "Upload To", "Upload To", "Bps" }, 26 | { 2, T.K.vartype.RATE_COUNTER, 40, 40, "Download From", "Download From", "Bps" }, 27 | { 3, T.K.vartype.COUNTER, 40, 40, "Flows", "Flows", "flows" }, 28 | }, 29 | 30 | }, 31 | } 32 | -------------------------------------------------------------------------------- /lua/backend_scripts/geoflows/cg_ip2loc_region.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- IP2Location : Region state+city 3 | -- 4 | -- 5 | TrisulPlugin = { 6 | 7 | -- the ID block, you can skip the fields marked 'optional ' 8 | -- 9 | id = { name = "IP2Loc City", description="City metrics"}, 10 | 11 | 12 | -- countergroup block 13 | -- 14 | countergroup = { 15 | 16 | control = { 17 | guid = "{E85FEB77-942C-411D-DF12-5DFCFCF2B932}", 18 | name = "IP2Loc City", 19 | description = "Meters city", 20 | bucketsize = 60, 21 | }, 22 | 23 | meters = { 24 | { 0, T.K.vartype.RATE_COUNTER, 40, 40, "Total", "Total", "Bps" }, 25 | { 1, T.K.vartype.RATE_COUNTER, 40, 40, "Upload To", "Upload To", "Bps" }, 26 | { 2, T.K.vartype.RATE_COUNTER, 40, 40, "Download From", "Download From", "Bps" }, 27 | { 3, T.K.vartype.COUNTER, 40, 40, "Flows", "Flows", "flows" }, 28 | }, 29 | 30 | }, 31 | } 32 | -------------------------------------------------------------------------------- /lua/backend_scripts/geoflows/cg_ip2loc_state.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- IP2Location : State 3 | -- 4 | -- 5 | TrisulPlugin = { 6 | 7 | -- the ID block, you can skip the fields marked 'optional ' 8 | -- 9 | id = { name = "IP2Loc State", description="State within country metrics"}, 10 | 11 | 12 | -- countergroup block 13 | -- 14 | countergroup = { 15 | 16 | control = { 17 | guid = "{5C28445E-19E3-499E-E14D-E4CC7128B62B}", 18 | name = "IP2Loc State", 19 | description = "Meters state", 20 | bucketsize = 60, 21 | }, 22 | 23 | meters = { 24 | { 0, T.K.vartype.RATE_COUNTER, 40, 40, "Total", "Total", "Bps" }, 25 | { 1, T.K.vartype.RATE_COUNTER, 40, 40, "Upload To", "Upload To", "Bps" }, 26 | { 2, T.K.vartype.RATE_COUNTER, 40, 40, "Download From", "Download From", "Bps" }, 27 | { 3, T.K.vartype.COUNTER, 40, 40, "Flows", "Flows", "flows" }, 28 | }, 29 | 30 | }, 31 | } 32 | -------------------------------------------------------------------------------- /lua/backend_scripts/geoflows/libftrie.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trisulnsm/trisul-scripts/bef85dea72016ef58a98246649c4cf7c4f3faca5/lua/backend_scripts/geoflows/libftrie.so -------------------------------------------------------------------------------- /lua/backend_scripts/metricmonitors/README.md: -------------------------------------------------------------------------------- 1 | Metric Monitors 2 | =========== 3 | 4 | 5 | Metric monitors dont work with packets/ flows but rather with 6 | streaming metrics. You can monitor either the raw metric 7 | updates (1sec latency) or keep track of processed and aggregated 8 | metric streams 9 | 10 | 11 | 12 | 1. cgmon.lua 13 | -------------- 14 | 15 | Monitors metric being streamed out to DB from a counter group 16 | 17 | 18 | 2. cgmon-2.lua 19 | ----------- 20 | 21 | Monitors real time (1s) streaming updates to counter group 22 | 23 | 24 | 3. cgmon-3.lua 25 | ----------- 26 | 27 | Monitors Topper Sketches as they are streamed out to DB 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /lua/backend_scripts/metricmonitors/cgmon.lua: -------------------------------------------------------------------------------- 1 | -- .lua 2 | -- 3 | -- cgmon.lua 4 | -- Monitors counter group activity 5 | -- 6 | -- 7 | TrisulPlugin = { 8 | 9 | id = { 10 | name = "CG Monitor ", 11 | description = "Example of monitoring counter group activity ", 12 | author = "Unleash", 13 | version_major = 1, 14 | version_minor = 0, 15 | }, 16 | 17 | 18 | onload = function() 19 | print("onload : from cgmon.lua"..T.contextid) 20 | end, 21 | 22 | onunload = function() 23 | print("onunload: bye cgmon.lua"..T.contextid) 24 | end, 25 | 26 | -- 27 | -- Monitor attaches itself to a counter group and gets called for 28 | -- all keys matching the regex 29 | -- 30 | cg_monitor = { 31 | 32 | counter_guid = "{C51B48D4-7876-479E-B0D9-BD9EFF03CE2E}", 33 | 34 | onbeginflush = function(dbengine,tvsec) 35 | print(string.format("onbeginflush [%d]", T.contextid )) 36 | end, 37 | 38 | onflush= function(dbengine,ts,key,metrics ) 39 | print(string.format("onflush [%d] %s", T.contextid , key )) 40 | end, 41 | 42 | onendflush = function(dbengine) 43 | print(string.format("onendflush [%d]", T.contextid )) 44 | end, 45 | 46 | }, 47 | 48 | } 49 | 50 | -------------------------------------------------------------------------------- /lua/backend_scripts/metricmonitors/cgmon_2.lua: -------------------------------------------------------------------------------- 1 | -- .lua 2 | -- 3 | -- cgmon2.lua 4 | -- Monitors counter group activity - demonstrates the update(..) method 5 | -- 6 | -- 7 | TrisulPlugin = { 8 | 9 | id = { 10 | name = "CG Monitor ", 11 | description = "Example of monitoring counter group activity ", 12 | author = "Unleash", 13 | version_major = 1, 14 | version_minor = 0, 15 | }, 16 | 17 | 18 | onload = function() 19 | end, 20 | 21 | onunload = function() 22 | end, 23 | 24 | -- 25 | -- Monitor attaches itself to a counter group and gets called for 26 | -- all keys matching the regex 27 | -- 28 | cg_monitor = { 29 | 30 | counter_guid = "{C51B48D4-7876-479E-B0D9-BD9EFF03CE2E}", 31 | 32 | onupdate = function(dbengine,key,tvsec,metrics) 33 | print(string.format("onupdate [%d] %d %10s %s", T.contextid, tvsec, key, table.concat(metrics,' ') )) 34 | end, 35 | 36 | }, 37 | 38 | } 39 | 40 | -------------------------------------------------------------------------------- /lua/backend_scripts/metricmonitors/cgmon_3.lua: -------------------------------------------------------------------------------- 1 | -- .lua 2 | -- 3 | -- cgmon3.lua 4 | -- Monitors counter group activity - demonstrates tapping into Topper Flush 5 | -- 6 | -- 7 | TrisulPlugin = { 8 | 9 | id = { 10 | name = "CG Monitor ", 11 | description = "Example of monitoring counter group activity ", 12 | author = "Unleash", 13 | version_major = 1, 14 | version_minor = 0, 15 | }, 16 | 17 | 18 | -- 19 | -- Monitor attaches itself to a counter group and gets called for 20 | -- various events in the counter group lifecycle 21 | -- 22 | cg_monitor = { 23 | 24 | counter_guid = "{4CD742B1-C1CA-4708-BE78-0FCA2EB01A86}", 25 | 26 | onbegintopperflush = function(dbengine,tvsec,meterid) 27 | print(string.format("----------- Topper Flush [%d] Meter %d Time %d ", T.contextid, meterid, tvsec)) 28 | end, 29 | 30 | ontopperflush = function(dbengine,key,metric) 31 | print(string.format("ontopperflush [%d] %s %d", T.contextid, key, metric)) 32 | end, 33 | 34 | onendtopperflush = function(dbengine,metric) 35 | 36 | end, 37 | 38 | }, 39 | 40 | } 41 | 42 | -------------------------------------------------------------------------------- /lua/backend_scripts/metricmonitors/delta-change-alert/README.md: -------------------------------------------------------------------------------- 1 | Technique to detect and alert on DELTA change on any metric 2 | =========================================================== 3 | 4 | ## script mac-traffic-tracker.lua 5 | 6 | Give a list of MACs to monitor and the following 7 | 8 | ```` 9 | mac_table = { 10 | ["00:1C:C0:B9:B9:10"] = {delta=0.5, alert_str="SYSTEMUNDER_TEST_77" , last_val=0, last_val_tm=0} , 11 | ["00:1B:57:41:71:75"] = {delta=0.5, alert_str="SEMIND_FUTURES" , last_val=0, last_val_tm=0} , 12 | }, 13 | ```` 14 | 15 | 1. `delta` : What % delta UP or down do you want to alert on 16 | 2. `alert_str` : Useful string to include with the alert, use an asset name or handler 17 | 3. `last_val,last_tm` : leave it at 0 18 | 19 | 20 | Inserting this script into Trisul Backend LUA will generate an alert whenever the usage (1-min) 21 | goes beyond `delta*value_in_previous_interval`.We have it in production for monitoring steady multicast feed for Financial Market applications. Works great. 22 | 23 | ## Usage 24 | 25 | Download this script into the local lua directory on the probe. To see where that directory is located. 26 | 27 | `trisulctl_probe list lua default@probe0` 28 | 29 | You dont need to restart trisul. Backend scripts are injected into live system. Removing the lua file uninstalls the script automatically. 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /lua/backend_scripts/passive-dns/flowtag-passive-dns.lua: -------------------------------------------------------------------------------- 1 | -- flow-tag 2 | -- 3 | -- TYPE: BACKEND SCRIPT 4 | -- PURPOSE: Tags flows with domain name. 5 | -- DESCRIPTION: Using a real time passive DNS database , we tag flows as they are flushed 6 | -- with the domain name. Top level for COM, ORG, NET and 3 level for othersj 7 | -- 8 | -- This script does the following 9 | -- 1. leveldb - uses LUAJIT FFI to build a LEVELDB backend 10 | -- 2. fts monitor - listens to DNS and updates the ldb CNAME/A 11 | -- 12 | -- 13 | 14 | local leveldb=require'tris_leveldb' 15 | TrisulPlugin = { 16 | 17 | id = { 18 | name = "pDNS tagger", 19 | description = "Passive DNS Flow Tagger ", 20 | }, 21 | 22 | -- we listen to onmessage for pDNS attach event 23 | message_subscriptions = { '{4349BFA4-536C-4310-C25E-E7C997B92244}' }, 24 | 25 | onmessage=function(msgid, msg) 26 | local dbaddr = msg:match("newleveldb=(%S+)") 27 | _,T.LevelReader = leveldb.from_addr(dbaddr); 28 | end, 29 | 30 | -- session_group : flow monitor 31 | -- as flows as flushed we tag em 32 | sg_monitor = { 33 | 34 | -- 35 | -- for each IP we look up the pDNS and add the tag 36 | -- 37 | onflush = function(engine,flw) 38 | 39 | if T.LevelReader==nil then return; end 40 | 41 | local ipa = flw:flow():ipa_readable() 42 | local ipz = flw:flow():ipz_readable() 43 | 44 | local p1 = T.LevelReader( ipa) 45 | local p2 = T.LevelReader( ipz) 46 | 47 | if p1 then 48 | local last_two = p1:match('(%w+%.%a+)$') 49 | -- print("Adding tag for "..ipa.." = "..last_two) 50 | flw:add_tag( last_two) 51 | end 52 | 53 | if p2 then 54 | local last_two = p2:match('(%w+%.%a+)$') 55 | -- print("Adding tag for "..ipz.." = "..last_two) 56 | flw:add_tag( last_two) 57 | end 58 | 59 | end, 60 | 61 | 62 | }, 63 | 64 | } 65 | -------------------------------------------------------------------------------- /lua/backend_scripts/passive-dns/helpers/test_1.lua: -------------------------------------------------------------------------------- 1 | -- test1.lua 2 | -- Test the level db interface 3 | 4 | 5 | local LevelDB=require'helpers/tris_leveldb' 6 | 7 | 8 | local l_writer, l_reader, l_closer, l_deleter = LevelDB.from_addr( LevelDB.open("Test111.ldb") ) 9 | 10 | l_writer("k1", " fooo") 11 | l_writer("k2", " baaarr") 12 | l_writer("192.168.2.79", " key is ubuntu64 test") 13 | 14 | 15 | local val 16 | 17 | val = l_reader("k1") 18 | print("k1 value is " .. val) 19 | 20 | val= l_reader("farbaz") 21 | if val then 22 | print("farbaz value is " .. val) 23 | else 24 | print("farbaz value is nil") 25 | end 26 | 27 | 28 | l_writer("gmail.com", "72.203.203.11") 29 | val = l_reader("gmail.com") 30 | print("gmail.com value is " .. val) 31 | 32 | 33 | local ret,err= l_deleter("gmail.com") 34 | if ret then 35 | print("Deleted key") 36 | else 37 | print("Err deleting ".. err) 38 | end 39 | 40 | 41 | val = l_reader("gmail.com") 42 | if val then 43 | print("gmail.com value is " .. val) 44 | else 45 | print("gmail.com value is NIL " ) 46 | end 47 | 48 | ret,err= l_deleter("gmail.com") 49 | if ret then 50 | print("Deleted key") 51 | else 52 | print("Err deleting ".. err) 53 | end 54 | 55 | 56 | 57 | l_closer() 58 | 59 | 60 | -- dump the DB 61 | print("DB Contents-----") 62 | LevelDB.dump("Test111.ldb") 63 | 64 | 65 | -------------------------------------------------------------------------------- /lua/backend_scripts/resources/README.md: -------------------------------------------------------------------------------- 1 | File hash resources 2 | ============= 3 | 4 | 1. filehash.lua 5 | ------------- 6 | Print all the hashes seen 7 | 8 | 9 | 2. sha256_x509.lua 10 | -------------- 11 | 1.How to save the certificates to filesystem 12 | 2.Feed back the new SHA256 certificate hash into Trisul resources pipelines 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /lua/backend_scripts/resources/filehash.lua: -------------------------------------------------------------------------------- 1 | -- filehash.lua 2 | -- 3 | -- Sample skeleton script for monitoring resources 4 | 5 | -- 1. Attaches to the "File Hash" resource group identified 6 | -- by the GUID {9781db2c-f78a-4f7f-a7e8-2b1a9a7be71a} 7 | -- 8 | -- 2. Prints all the hashes seen - field resource:uri() 9 | -- 10 | -- 11 | 12 | TrisulPlugin = { 13 | 14 | id = { 15 | name = "Prints File Hashes seen ", 16 | description = "Sample script that just prints new file hashes as they are seen ", 17 | }, 18 | 19 | 20 | resource_monitor = { 21 | 22 | -- 23 | -- The guid {978.. below represents the File Hashes resource 24 | -- Login as admin/admin , then context0 > Resource Groups to view list of 25 | -- installed resource GUIDs 26 | -- 27 | resource_guid = '{9781db2c-f78a-4f7f-a7e8-2b1a9a7be71a}', 28 | 29 | onnewresource = function(engine, newresource ) 30 | print("timestamp ".. os.date('%c',newresource:timestamp())) 31 | print("hash = ".. newresource:uri()) 32 | print("label = ".. newresource:label()) 33 | print("flow = ".. newresource:flow():to_s()) 34 | end, 35 | 36 | }, 37 | 38 | } 39 | -------------------------------------------------------------------------------- /lua/backend_scripts/resources/printcertchain.lua: -------------------------------------------------------------------------------- 1 | -- dumpcerts.lua 2 | -- 3 | -- Print cert chains 4 | -- 5 | 6 | TrisulPlugin = { 7 | 8 | id = { 9 | name = "Prints SSL Cert resources ", 10 | description = "Just prints SSL Certs ", 11 | }, 12 | 13 | 14 | resource_monitor = { 15 | 16 | -- want the SSL 17 | resource_guid = '{5AEE3F0B-9304-44BE-BBD0-0467052CF468}', 18 | 19 | -- a new resource was seen - print all details to screen 20 | onnewresource = function(engine, newresource ) 21 | 22 | for cert in newresource:label():gmatch("%-*BEGIN CERTIFICATE.-END CERTIFICATE%-*") do 23 | print(cert) 24 | end 25 | 26 | end, 27 | 28 | }, 29 | 30 | } 31 | -------------------------------------------------------------------------------- /lua/backend_scripts/resources/ssl.lua: -------------------------------------------------------------------------------- 1 | -- ssl.lua 2 | -- 3 | -- Sample skeleton script for monitoring resources 4 | 5 | -- 6 | -- 7 | 8 | 9 | TrisulPlugin = { 10 | 11 | id = { 12 | name = "Prints SSL Cert resources ", 13 | description = "Just prints SSL Certs ", 14 | }, 15 | 16 | 17 | resource_monitor = { 18 | 19 | -- want the SSL 20 | resource_guid = function() 21 | for name ,guid in pairs(T.resourcegroups) do 22 | if name:match("SSL") then return guid; end 23 | end 24 | end, 25 | 26 | -- a new resource was seen - print all details to screen 27 | onnewresource = function(engine, newresource ) 28 | print("---------------------------------------------------------------------------------------------") 29 | print("timestamp ".. os.date('%c',newresource:timestamp())) 30 | print("certsubject = ".. newresource:uri()) 31 | print("flow = ".. newresource:flow():to_s()) 32 | end, 33 | 34 | }, 35 | 36 | } 37 | -------------------------------------------------------------------------------- /lua/backend_scripts/roca/cert01.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIEkjCCA3qgAwIBAgIQCgFBQgAAAVOFc2oLheynCDANBgkqhkiG9w0BAQsFADA/ 3 | MSQwIgYDVQQKExtEaWdpdGFsIFNpZ25hdHVyZSBUcnVzdCBDby4xFzAVBgNVBAMT 4 | DkRTVCBSb290IENBIFgzMB4XDTE2MDMxNzE2NDA0NloXDTIxMDMxNzE2NDA0Nlow 5 | SjELMAkGA1UEBhMCVVMxFjAUBgNVBAoTDUxldCdzIEVuY3J5cHQxIzAhBgNVBAMT 6 | GkxldCdzIEVuY3J5cHQgQXV0aG9yaXR5IFgzMIIBIjANBgkqhkiG9w0BAQEFAAOC 7 | AQ8AMIIBCgKCAQEAnNMM8FrlLke3cl03g7NoYzDq1zUmGSXhvb418XCSL7e4S0EF 8 | q6meNQhY7LEqxGiHC6PjdeTm86dicbp5gWAf15Gan/PQeGdxyGkOlZHP/uaZ6WA8 9 | SMx+yk13EiSdRxta67nsHjcAHJyse6cF6s5K671B5TaYucv9bTyWaN8jKkKQDIZ0 10 | Z8h/pZq4UmEUEz9l6YKHy9v6Dlb2honzhT+Xhq+w3Brvaw2VFn3EK6BlspkENnWA 11 | a6xK8xuQSXgvopZPKiAlKQTGdMDQMc2PMTiVFrqoM7hD8bEfwzB/onkxEz0tNvjj 12 | /PIzark5McWvxI0NHWQWM6r6hCm21AvA2H3DkwIDAQABo4IBfTCCAXkwEgYDVR0T 13 | AQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwfwYIKwYBBQUHAQEEczBxMDIG 14 | CCsGAQUFBzABhiZodHRwOi8vaXNyZy50cnVzdGlkLm9jc3AuaWRlbnRydXN0LmNv 15 | bTA7BggrBgEFBQcwAoYvaHR0cDovL2FwcHMuaWRlbnRydXN0LmNvbS9yb290cy9k 16 | c3Ryb290Y2F4My5wN2MwHwYDVR0jBBgwFoAUxKexpHsscfrb4UuQdf/EFWCFiRAw 17 | VAYDVR0gBE0wSzAIBgZngQwBAgEwPwYLKwYBBAGC3xMBAQEwMDAuBggrBgEFBQcC 18 | ARYiaHR0cDovL2Nwcy5yb290LXgxLmxldHNlbmNyeXB0Lm9yZzA8BgNVHR8ENTAz 19 | MDGgL6AthitodHRwOi8vY3JsLmlkZW50cnVzdC5jb20vRFNUUk9PVENBWDNDUkwu 20 | Y3JsMB0GA1UdDgQWBBSoSmpjBH3duubRObemRWXv86jsoTANBgkqhkiG9w0BAQsF 21 | AAOCAQEA3TPXEfNjWDjdGBX7CVW+dla5cEilaUcne8IkCJLxWh9KEik3JHRRHGJo 22 | uM2VcGfl96S8TihRzZvoroed6ti6WqEBmtzw3Wodatg+VyOeph4EYpr/1wXKtx8/ 23 | wApIvJSwtmVi4MFU5aMqrSDE6ea73Mj2tcMyo5jMd6jmeWUHK8so/joWUoHOUgwu 24 | X4Po1QYz+3dszkDqMp4fklxBwXRsW10KXzPMTZ+sOPAveyxindmjkW8lGy+QsRlG 25 | PfZ+G6Z6h7mjem0Y+iWlkYcV4PIWL1iwBi8saCbGS5jN2p8M+X+Q7UNKEkROb3N6 26 | KOqkqm57TH2H3eDJAkSnh6/DNFu0Qg== 27 | -----END CERTIFICATE----- 28 | -------------------------------------------------------------------------------- /lua/backend_scripts/roca/cert05.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIICpTCCAYwCCQCEcnuDiu6k0DANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDDAls 3 | b2NhbGhvc3QwHhcNMTcxMDE2MTk0NjEwWhcNMTgxMDE2MTk0NjEwWjAUMRIwEAYD 4 | VQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQGp 5 | xaaH24+yow+C3/jdoVTCke8nsEsl4YW4qyDExqX2BIWf5Z7eVKCvfA5yiIAK1xk7 6 | tzGlvx4RcNehxmkw1B6j6yHe33uK9dQqkcNPcgRblu/VfLGCMB3cwmln5VCOmriJ 7 | mG1X6Dff5w+Z/DfEmPQ8F8O5sGTPcERD16Az4DqCFpeqvzD7Ke59M7N2eWWokFjj 8 | s5dUvTocAH0oojsM+nn51xGsbkxLT3ewY0sezWhZGvA8fVO2Kl8tkRY9TvT48mgx 9 | rkYe2F7F94jFEj/8Esh0IP3P6h0bSlLDUhgk+zmYNIc5tiqYbY52siibwuuLwG0T 10 | gUeFG4OgcYPSEFWfkGMhAgMBAAEwDQYJKoZIhvcNAQELBQADggECAAGZiTHt6Hgd 11 | yDjR9aH9O5oNvtmTPf6tJtKhIu1VEnWjxpXK35Dzmw5AvSSL4EVB/nAYx63eonfI 12 | 4BLM3YOPLFzrIJn6LE37LMug90vCz8zPOChWR7HTgroLyh3O2nPjvJL/SuoBya6g 13 | OwKg6gqF4Kmc4Q3BAY8QiDnQP3pIxF46WKq+r6/QqILuZRE9W7SwjTBP1jKmVztM 14 | 3tyL9kVl7RbRA9/ovXHlEfkN7lJqqYKMZvJZPsxm6VJQRsW1rl55bI8Vqys2DOnx 15 | Nm01jtxuMVukAWEj4J8ExCKnd5EhRO/fbOfbwUVythhIsevkdXUC3PiZhDcK+Cge 16 | PbJmeM8KehS3 17 | -----END CERTIFICATE----- -------------------------------------------------------------------------------- /lua/backend_scripts/roca/rocafile.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- rocafile.lua :small test program 3 | -- 4 | -- PURPOSE: check PEM files for vulnerability 5 | -- DESCRIPTION: usage 'luajit rocafile.lua example01.pem' 6 | -- 7 | 8 | 9 | require'roca' 10 | 11 | T = {} 12 | 13 | TrisulPlugin.onload() 14 | 15 | local pem_file = arg[1] 16 | print("Checking PEM file ".. pem_file) 17 | 18 | -- run cmd to extract the modulus from X509 19 | local pipein = io.popen( "openssl x509 -in ".. pem_file.." -text") 20 | local certchaindump= pipein:read("*a") 21 | 22 | for m in certchaindump:gmatch("Modulus:([0-9a-fA-F:%s]*)Exponent") do 23 | 24 | local h = m:gsub("[:%s]","") 25 | print("modulus_hex="..h) 26 | 27 | if TrisulPlugin.is_vulnerable(h) then 28 | print("VULNERABLE...") 29 | else 30 | print("NOT VULNERABLE...") 31 | end 32 | 33 | end 34 | 35 | -------------------------------------------------------------------------------- /lua/backend_scripts/snmpasync/README.md: -------------------------------------------------------------------------------- 1 | Async SNMP 2 | ========== 3 | 4 | 5 | -------------------------------------------------------------------------------- /lua/backend_scripts/snmpasync/bulkwalk_cmd.lua: -------------------------------------------------------------------------------- 1 | -- return { key, value } 2 | function do_bulk_walk( agent, version, community, oid ) 3 | command = "snmpbulkwalk" 4 | if version == "1" then command="snmpwalk" end 5 | local tstart = os.time() 6 | local ofile = os.tmpname() 7 | os.execute(command.." -r 1 -O q -t 3 -v"..version.." -c '"..community.."' "..agent.." "..oid.. " > "..ofile) 8 | --print(command.." -r 1 -O q -t 3 -v"..version.." -c '"..community.."' "..agent.." "..oid) 9 | 10 | local ret = { } 11 | local h=io.open(ofile) 12 | for oneline in h:lines() 13 | do 14 | local k,v = oneline:match("%.(%d+)%s+(.+)") 15 | if k then 16 | ret[agent.."_"..k] = v:gsub('"','') 17 | else 18 | print("ERROR in snmp output line="..oneline) 19 | end 20 | end 21 | h:close() 22 | os.remove(ofile) 23 | 24 | print("Done with agent "..agent.." elapsed secs="..os.time()-tstart) 25 | return ret 26 | end 27 | 28 | -------------------------------------------------------------------------------- /lua/backend_scripts/strelka/strelka-resource.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- strelka_resource : A new resource type to hold the Strelka JSON results document 3 | -- 4 | -- 5 | TrisulPlugin = { 6 | 7 | id = { 8 | name = "Strelka Scan", 9 | description = "Scan results ", 10 | }, 11 | 12 | -- resourcegroup block 13 | -- 14 | resourcegroup = { 15 | control = { 16 | guid = "{8A3E3EE5-0194-4B3C-9400-39BE9E7F7A11}", 17 | name = "Strelka Scan", 18 | description = "Output JSON document from Strelka scan", 19 | }, 20 | }, 21 | } 22 | -------------------------------------------------------------------------------- /lua/backend_scripts/strelka/strelka1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trisulnsm/trisul-scripts/bef85dea72016ef58a98246649c4cf7c4f3faca5/lua/backend_scripts/strelka/strelka1.png -------------------------------------------------------------------------------- /lua/backend_scripts/strelka/strelka2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trisulnsm/trisul-scripts/bef85dea72016ef58a98246649c4cf7c4f3faca5/lua/backend_scripts/strelka/strelka2.png -------------------------------------------------------------------------------- /lua/backend_scripts/ua-parser/cg_browser.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- UA-Browser counter group 3 | -- 4 | -- 5 | TrisulPlugin = { 6 | 7 | -- the ID block, you can skip the fields marked 'optional ' 8 | -- 9 | id = { 10 | name = "UA Browser", 11 | description = "Browser hits ", -- optional 12 | }, 13 | 14 | 15 | -- countergroup block 16 | -- 17 | countergroup = { 18 | 19 | control = { 20 | guid = "{747F125F-2838-4A76-6D44-55974DE58F78}", 21 | name = "UA Browser", 22 | description = "Browser hits based on UA", 23 | bucketsize = 30, 24 | }, 25 | 26 | -- meters table 27 | -- id, type of meter, toppers to track, bottom-ers to track, Name, units, units-short 28 | -- 29 | meters = { 30 | { 0, T.K.vartype.COUNTER, 10, 10, "Hits", "hits", "hits" }, 31 | }, 32 | 33 | }, 34 | } 35 | -------------------------------------------------------------------------------- /lua/backend_scripts/ua-parser/cg_device.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- UA-Device counter group 3 | -- 4 | -- 5 | TrisulPlugin = { 6 | 7 | -- the ID block, you can skip the fields marked 'optional ' 8 | -- 9 | id = { 10 | name = "UA Device", 11 | }, 12 | 13 | 14 | -- countergroup block 15 | -- 16 | countergroup = { 17 | 18 | control = { 19 | guid = "{EB232F1A-05E6-45E7-1888-9AF224511E6D}", 20 | name = "UA Device", 21 | description = "Device hits based on UA", 22 | bucketsize = 30, 23 | }, 24 | 25 | meters = { 26 | { 0, T.K.vartype.COUNTER, 10, 10, "Hits", "hits", "hits" }, 27 | }, 28 | 29 | }, 30 | } 31 | -------------------------------------------------------------------------------- /lua/backend_scripts/ua-parser/cg_os.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- UA-OS counter group 3 | -- 4 | -- 5 | TrisulPlugin = { 6 | 7 | -- the ID block, you can skip the fields marked 'optional ' 8 | -- 9 | id = { 10 | name = "UA OS", 11 | description = "Browser hits ", -- optional 12 | }, 13 | 14 | 15 | -- countergroup block 16 | -- 17 | countergroup = { 18 | 19 | control = { 20 | guid = "{0F67F47E-A407-4047-2AF6-8E25FEC75C3A}", 21 | name = "UA OS", 22 | description = "OS hits based on UA", 23 | bucketsize = 30, 24 | }, 25 | 26 | meters = { 27 | { 0, T.K.vartype.COUNTER, 10, 10, "Hits", "hits", "hits" }, 28 | }, 29 | 30 | }, 31 | } 32 | -------------------------------------------------------------------------------- /lua/backend_scripts/ua-parser/ua-extractor.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- User-Agent 3 | -- 4 | -- TYPE: FRONTEND SCRIPT 5 | -- PURPOSE: Nothing much , just create User-Agents as Resources 6 | -- DESCRIPTION: 7 | -- 8 | TrisulPlugin = { 9 | 10 | 11 | -- 12 | id = { 13 | name = "Create UA-Resources", 14 | description = "use onattribute", 15 | }, 16 | 17 | 18 | -- reassembly_handler block 19 | -- 20 | reassembly_handler = { 21 | 22 | onattribute = function(engine, timestamp, flowkey, attr_name, attr_value) 23 | if attr_name == "User-Agent" then 24 | engine:add_resource("{ED5CA168-1E17-44E0-7ABD-65E5C2DFAD21}", 25 | flowkey:id(), 26 | attr_value); 27 | end 28 | end, 29 | 30 | }, 31 | 32 | } 33 | -------------------------------------------------------------------------------- /lua/backend_scripts/ua-parser/ua-resource.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- User-Agent resource 3 | -- 4 | -- TYPE: FRONTEND SCRIPT 5 | -- PURPOSE: Create a new resource group 6 | -- DESCRIPTION: You can create your own resource groups for your specific case 7 | -- 8 | -- 9 | TrisulPlugin = { 10 | 11 | id = { 12 | name = "UA-Resource-Group", 13 | }, 14 | 15 | -- resourcegroup block - defines a new User-Agent resource Group 16 | -- 17 | resourcegroup = { 18 | control = { 19 | guid = "{ED5CA168-1E17-44E0-7ABD-65E5C2DFAD21}", 20 | name = "HTTP User-Agent", 21 | description = "HTTP U-A resource" 22 | }, 23 | }, 24 | } 25 | -------------------------------------------------------------------------------- /lua/backend_scripts/unixsocket_api/README.md: -------------------------------------------------------------------------------- 1 | # Trisul Unix Socket API 2 | 3 | Allows external programs to insert metrics into the Trisul pipeline. 4 | You write [Trisul Engine:API](https://www.trisul.org/docs/lua/obj_engine.html) commands in a new-line separated format into a Unix Socket 5 | 6 | The lowest resolution with this method is 1-second. 7 | 8 | ## How to use 9 | 10 | When you [install](https://www.trisul.org/docs/lua/basics.html#installing_and_uninstalling) this script and restart Trisul-Probe you will find that it has created a new Unix Socket under 11 | 12 | ``` 13 | /usr/local/var/lib/trisul-probe/domain0/probe0/context0/run/api.sock.0 14 | ``` 15 | 16 | You can simply write commands to that socket with the following text syntax. Each parameter is separated by a "\n" 17 | 18 | ```` 19 | commandname<\n> 20 | argument1<\n> 21 | argument2<\n> 22 | ```` 23 | 24 | For example: to update a counter write this string. Note that 25 | 26 | ```` 27 | update_counter 28 | {4DF11F00-B726-4260-5F83-0D9891197B45} 29 | 192.168.29.8 30 | 0 31 | 10023 32 | ```` 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /lua/backend_scripts/unixsocket_api/rubyexample/apimon.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- Counter Group API Latency 3 | -- 4 | -- 5 | TrisulPlugin = { 6 | 7 | -- id block 8 | -- 9 | id = { 10 | name = "API Latency", 11 | description = "API Latency", 12 | }, 13 | 14 | countergroup = { 15 | 16 | control = { 17 | guid = "{9497A90C-86DF-44A5-439F-3B4092792728}", 18 | name = "API Latency", 19 | description = "API Response Times", 20 | bucketsize = 10, 21 | }, 22 | 23 | -- meters table 24 | -- id, type of meter, toppers to track, Name, units, units-short 25 | -- 26 | meters = { 27 | { 0, T.K.vartype.COUNTER, 20,20, "Hits", "Hits", "hits" }, 28 | { 1, T.K.vartype.AVERAGE, 20,20, "Avg", "Avg_us", "us" }, 29 | { 2, T.K.vartype.MAXIMUM, 20,20, "Max", "Max_us", "us" }, 30 | { 3, T.K.vartype.MINIMUM, 20,20, "Min", "Min_us", "us" }, 31 | }, 32 | 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /lua/backend_scripts/unixsocket_api/rubyexample/rbench.rb: -------------------------------------------------------------------------------- 1 | require 'net/http' 2 | require 'net/https' 3 | 4 | 5 | # set up unix socket 6 | apis = Socket.new(:UNIX, :DGRAM) # UNIX datagram socket 7 | s1_ai = Socket.sockaddr_un("/usr/local/var/lib/trisul-probe/domain0/probe0/context0/run/api.sock.0") 8 | apis.connect(s1_ai) 9 | 10 | uri = URI.parse("https://192.168.2.99") 11 | request = Net::HTTP.new(uri.host, uri.port) 12 | 13 | while true do 14 | t1=(Time.now.to_f*1000000).to_i 15 | response = request.get("/") 16 | t2=(Time.now.to_f*1000000).to_i 17 | 18 | us_latency = t2-t1 19 | 20 | apis.send("update_counter\n{9497A90C-86DF-44A5-439F-3B4092792728}\ntest_192.168.2.99=\n0\n1",0) 21 | 22 | apis.send("update_counter\n{9497A90C-86DF-44A5-439F-3B4092792728}\ntest_192.168.2.99=\n1\n#{us_latency}",0) 23 | 24 | apis.send("update_counter\n{9497A90C-86DF-44A5-439F-3B4092792728}\ntest_192.168.2.99=\n2\n#{us_latency}",0) 25 | 26 | apis.send("update_counter\n{9497A90C-86DF-44A5-439F-3B4092792728}\ntest_192.168.2.99=\n3\n#{us_latency}",0) 27 | 28 | puts(us_latency) 29 | 30 | end 31 | 32 | 33 | -------------------------------------------------------------------------------- /lua/backend_scripts/url-latency-monitor/README.md: -------------------------------------------------------------------------------- 1 | # URL Latency Monitor 2 | 3 | Measures per HTTP Request/Response latecny 4 | 5 | This adds a new counter group called *URL Service* with four metrics 6 | 7 | 1. hits - number of hits per URL 8 | 2. avg - avg latency in microseconds (us) 9 | 3. max - max latency (us) of all samples in window 10 | 4. min - min latency (us) of all samples in window 11 | 12 | 13 | -------------------------------------------------------------------------------- /lua/backend_scripts/url-latency-monitor/urlmon.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- URL-Monitor 3 | -- 4 | -- TYPE: BACKEND SCRIPT 5 | -- PURPOSE: Response Time per Service 6 | -- DESCRIPTION: for HTTP 7 | -- 8 | TrisulPlugin = { 9 | 10 | -- id block 11 | -- 12 | id = { 13 | name = "URL Mon", 14 | description = "URL Response Time", 15 | }, 16 | 17 | -- resource_monitor block 18 | -- 19 | resource_monitor = { 20 | 21 | resource_guid = '{4EF9DEB9-4332-4867-A667-6A30C5900E9E}', 22 | 23 | -- add 24 | onnewresource = function(engine, resource ) 25 | local res = resource:uri() 26 | local l,k,v = res:match("(%S+)%s(%S+)%s([^%?%s]+)") 27 | local usec = resource:label():match("usec:(%d+)") 28 | 29 | if usec then 30 | local key = l.."/"..k..v 31 | key=key:sub(1,50) 32 | local latency = tonumber(usec) 33 | engine:update_counter( "{C93B79D5-20A0-49D8-FA27-160B45D49C00}", key, 0, 1); 34 | engine:update_counter( "{C93B79D5-20A0-49D8-FA27-160B45D49C00}", key, 1, latency); 35 | engine:update_counter( "{C93B79D5-20A0-49D8-FA27-160B45D49C00}", key, 2, latency); 36 | engine:update_counter( "{C93B79D5-20A0-49D8-FA27-160B45D49C00}", key, 3, latency); 37 | end 38 | end, 39 | 40 | }, 41 | 42 | countergroup = { 43 | 44 | control = { 45 | guid = "{C93B79D5-20A0-49D8-FA27-160B45D49C00}", 46 | name = "URL Service ", 47 | description = "URL Service Response Times ", 48 | bucketsize = 60, 49 | }, 50 | 51 | -- meters table 52 | -- id, type of meter, toppers to track, Name, units, units-short 53 | -- 54 | meters = { 55 | { 0, T.K.vartype.COUNTER, 20,20, "Hits", "Hits", "hits" }, 56 | { 1, T.K.vartype.AVERAGE, 20,20, "Avg", "Avg_us", "us" }, 57 | { 2, T.K.vartype.MAXIMUM, 20,20, "Max", "Max_us", "us" }, 58 | { 3, T.K.vartype.MINIMUM, 20,20, "Min", "Min_us", "us" }, 59 | }, 60 | 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /lua/frontend_scripts/alerts/README.md: -------------------------------------------------------------------------------- 1 | Alerts 2 | ====== 3 | 4 | Sample scripts 5 | 6 | 1. socialalert.lua 7 | --------------- 8 | 9 | Look at TLS certificates for known social media sites and generate a full alert. These 10 | alerts use a private "sigid" range and integrate with Snort/Suricata streams. 11 | 12 | Demonstrates: T.re2(..) regex, add_full_alert(..) 13 | -------------------------------------------------------------------------------- /lua/frontend_scripts/buffer/README.md: -------------------------------------------------------------------------------- 1 | Buffer 2 | ====== 3 | 4 | LUA Scripts that demonstrate how you can deal with raw packet bytes. 5 | 6 | 7 | Trisul does not copy over the raw packets to from the C to the LUA side, 8 | instead it provides an abstraction called "Buffer" that allows you to 9 | retrieve the information you seek. 10 | 11 | 12 | 13 | tcphdr.lua 14 | ---------- 15 | 16 | Prints the TCP header to stdout. This demonstrates how you can extract numbers in 17 | network to host order and also how to extract bits such as the TCP flags. 18 | 19 | 20 | -------------------------------------------------------------------------------- /lua/frontend_scripts/buffer/tcphdr.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- tcphdr.lua 3 | -- 4 | -- Prints the TCP headers to stdout - does not actually meter anything 5 | -- 6 | -- Demonstrates how you can work with 7 | -- 1) The Buffer object 8 | -- 2) The Layer 9 | -- 3) The Packet 10 | -- 11 | TrisulPlugin = { 12 | 13 | id = { 14 | name = "TCPHDR ", 15 | description = "Demo - prints tcp header ", 16 | author = "Unleash", 17 | version_major = 1, 18 | version_minor = 0, 19 | }, 20 | 21 | simplecounter = { 22 | 23 | -- attach to the TCP protocol http://trisul.org/docs/ref/guid.html#protocols 24 | protocol_guid = "{77E462AB-2E42-42ec-9A58-C1A6821D6B31}", 25 | 26 | -- onpacket 27 | onpacket = function(engine,layer) 28 | 29 | local buff = layer:rawbytes() 30 | 31 | -- T.debugger({ engine = engine, layer = layer }) 32 | -- 33 | 34 | print("Source port = " .. buff:hval_16(0)) 35 | print("Dest port = " .. buff:hval_16(2)) 36 | print("Seq number = " .. buff:hval_32(4)) 37 | print("Ack number = " .. buff:hval_32(8)) 38 | 39 | local fval= buff:hval_16(12) 40 | 41 | -- how to extract a single bit ] 42 | -- since we are using LuaJIT we cant use the new bit32 library in Lua5.2 43 | -- 44 | print("RST flag = " .. tostring(T.util.testbit32(fval,2))) 45 | print("SYN flag = " .. tostring(T.util.testbit32(fval,1))) 46 | print("FIN flag = " .. tostring(T.util.testbit32(fval,0))) 47 | 48 | -- can also test for a single bit from the layer object directly 49 | -- for example RST is bit number 109 in the TCP header 50 | print("RST flag alt = " .. tostring(layer:testbit(109))) 51 | 52 | -- get 4 bit frame offset 53 | print("Frame offset words = " .. T.util.bitval32(layer:getbyte(12),7,4)) 54 | -- or from the buffer 55 | print("Frame offset words = " .. T.util.bitval32(buff:hval_8(12),7,4)) 56 | 57 | 58 | end, 59 | 60 | 61 | }, 62 | 63 | 64 | } 65 | 66 | -------------------------------------------------------------------------------- /lua/frontend_scripts/custom_alerts/README.md: -------------------------------------------------------------------------------- 1 | Create a new alert group 2 | ====== 3 | -------------------------------------------------------------------------------- /lua/frontend_scripts/custom_alerts/my-alerts.lua: -------------------------------------------------------------------------------- 1 | -- .lua 2 | -- 3 | -- my-alerts Testing a new alert group 4 | -- 5 | -- {74fe9532-edeb-421e-bd92-b5c62e7f2346} 6 | -- define_guid(<>, 0x74fe9532, 0xedeb, 0x421e, 0xbd, 0x92, 0xb5, 0xc6, 0x2e, 0x7f, 0x23, 0x46); 7 | -- 8 | -- 9 | TrisulPlugin = { 10 | 11 | id = { 12 | name = "Trading Alerts", 13 | description = "Custom alerts generated by our LUA code ", 14 | author = "Unleash", 15 | version_major = 1, 16 | version_minor = 0, 17 | }, 18 | 19 | alertgroup = { 20 | control = { 21 | guid = "{23794772-edeb-421e-bd92-b5c62e7f2346}", 22 | name = "Trading Alerts", 23 | description = "Alerts detected by our trading script ", 24 | }, 25 | 26 | }, 27 | 28 | 29 | } 30 | 31 | -------------------------------------------------------------------------------- /lua/frontend_scripts/custom_resources/README.md: -------------------------------------------------------------------------------- 1 | Create a new resource group 2 | ====== 3 | -------------------------------------------------------------------------------- /lua/frontend_scripts/custom_resources/pdf-files.lua: -------------------------------------------------------------------------------- 1 | -- .lua 2 | -- 3 | -- pdf-files Special Custom Resource 4 | -- 5 | -- 6 | -- 7 | TrisulPlugin = { 8 | 9 | id = { 10 | name = "PDF Files", 11 | description = "PDF File Resources ", 12 | author = "Unleash", 13 | version_major = 1, 14 | version_minor = 0, 15 | }, 16 | 17 | resourcegroup = { 18 | control = { 19 | guid = "{79237742-edeb-421e-bd92-b5c62e7f2346}", 20 | name = "PDF File Resource", 21 | description = "PDF File Custom Resource ", 22 | }, 23 | 24 | }, 25 | 26 | 27 | } 28 | 29 | -------------------------------------------------------------------------------- /lua/frontend_scripts/fileextract/README.md: -------------------------------------------------------------------------------- 1 | File Extraction Scripts 2 | =========== 3 | 4 | 5 | Brand new file extraction scripting framework that gives you 6 | the maximum flexibility in controlling what is extracted and 7 | how the contents are handled post extraction. 8 | 9 | 10 | 11 | 1. filex.lua 12 | -------------- 13 | 14 | Simple program prints the file details - all HTTP request/response headers and such 15 | 16 | 17 | -------------------------------------------------------------------------------- /lua/frontend_scripts/fileextract/filex_stream.lua: -------------------------------------------------------------------------------- 1 | -- .lua 2 | -- 3 | -- filex_stream 4 | -- File Extraction by lua using the onpayload(..) streaming interface 5 | -- this script operates on buffers, just appends them to a file 6 | -- 7 | TrisulPlugin = { 8 | 9 | id = { 10 | name = "file extraction ", 11 | description = "various aspects of file extraction ", 12 | author = "Unleash", 13 | version_major = 1, 14 | version_minor = 0, 15 | }, 16 | 17 | 18 | 19 | onload = function() 20 | os.execute('mkdir -p /tmp/kk') 21 | end, 22 | 23 | 24 | 25 | filex_monitor = { 26 | 27 | -- save all content to /tmp/kk 28 | -- 29 | onpayload_http = function ( engine, timestamp, flowkey, path, req_header, resp_header, dir , seekpos , buffer ) 30 | 31 | 32 | local fn = path:match("^.+/(.+)$") 33 | T.async:copybuffer( buffer, "/tmp/kk/"..fn) 34 | 35 | end, 36 | 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /lua/frontend_scripts/fileextract/filter.lua: -------------------------------------------------------------------------------- 1 | -- filter.lua 2 | -- 3 | -- Filter out common text based file types 4 | -- 5 | -- Demonstrates 6 | -- ------------ 7 | -- 1. Use of filter(..) to only save text/html content 8 | -- 9 | -- 2. We dont do anything else, so this script only impacts what files get 10 | -- the built in MD5 file hashing 11 | -- 12 | TrisulPlugin = { 13 | 14 | id = { 15 | name = "filter built in MD5 hashing", 16 | description = "Only allow some Content-Type to be extracted and MD5 generated ", 17 | }, 18 | 19 | 20 | 21 | -- Monitor attaches itself to file extraction module(s) 22 | -- 23 | filex_monitor = { 24 | 25 | -- 26 | -- filter : Check the response content type and decide 27 | -- If the response header matches the regex '(application|javascript)' 28 | -- then process the file and do its MD5, else skip 29 | -- 30 | filter = function( engine, timestamp, flowkey, header) 31 | 32 | if header:is_response() then 33 | if header:match_value("Content-Type", "(application|javascript)") then 34 | return true 35 | else 36 | return false 37 | end 38 | end 39 | -- always return true if header is a HTTP Request because you dont know the 40 | -- content type yet 41 | return true 42 | end, 43 | } 44 | 45 | } 46 | 47 | -------------------------------------------------------------------------------- /lua/frontend_scripts/fileextract/fx_largeimage.lua: -------------------------------------------------------------------------------- 1 | -- .lua 2 | -- 3 | -- filex 4 | -- File Extraction by lua 5 | -- 6 | -- 7 | TrisulPlugin = { 8 | 9 | id = { 10 | name = "file extraction ", 11 | description = "various aspects of file extraction ", 12 | author = "Unleash", 13 | version_major = 1, 14 | version_minor = 0, 15 | }, 16 | 17 | 18 | 19 | onload = function() 20 | print("LOADED : filex_largeimage.lua ") 21 | end, 22 | 23 | onunload = function() 24 | print("BYE: filex_largeimage.lua ") 25 | end, 26 | 27 | -- 28 | -- Monitor attaches itself to file extraction module(s) 29 | -- gets called at various stages, can control and add stats etc 30 | -- 31 | filex_monitor = { 32 | 33 | 34 | -- 35 | -- filter : decide if you want to reassemble this file or not.. 36 | -- 37 | filter = function( engine, timestamp, flowkey, header) 38 | if header:is_response() or header:is_method("post") then 39 | local ct = header:get_value("Content-Type") 40 | if ct and ct:match("jpeg") then 41 | T.log(">>>>> Saving JPG file for analysis "..ct) 42 | return true 43 | else 44 | return false 45 | end 46 | else 47 | -- request 48 | -- always return true 49 | return true 50 | end 51 | end, 52 | 53 | 54 | -- save all content to /tmp/kk 55 | -- 56 | onfile_http = function ( engine, timestamp, flowkey, path, req_header, resp_header, length ) 57 | local ct = resp_header:get_value("Content-Type") 58 | if ct and ct:match("jpeg") and length > 500000 then 59 | local fn = path:match("^.+/(.+)$") 60 | if fn then 61 | T.async:copy( path, "/tmp/filex/largejpg/"..fn) 62 | end 63 | end 64 | end, 65 | 66 | } 67 | 68 | } 69 | 70 | -------------------------------------------------------------------------------- /lua/frontend_scripts/fileextract/malware-cymru.lua: -------------------------------------------------------------------------------- 1 | -- .lua 2 | -- 3 | -- Malware lookup 4 | -- 5 | TrisulPlugin = { 6 | 7 | id = { 8 | name = "file extraction ", 9 | description = "various aspects of file extraction ", 10 | author = "Unleash", 11 | version_major = 1, 12 | version_minor = 0, 13 | }, 14 | 15 | 16 | -- 17 | filex_monitor = { 18 | 19 | 20 | -- save all content to /tmp/kk 21 | -- 22 | onfile_http = function ( engine, timestamp, flowkey, path, req_header, resp_header, length ) 23 | 24 | local h = io.popen("sha1sum "..path) 25 | local outputstr = h:read("*a") 26 | h:close() 27 | 28 | T.async:schedule( 29 | { 30 | data = outputstr, 31 | 32 | onexecute = function( indata) 33 | local hashval = indata:match("(%x+)") 34 | 35 | local cymru_command = "whois -h hash.cymru.com ".. hashval; 36 | local h = io.popen(cymru_command) 37 | local outputstr = h:read("*a") 38 | local h,last_seen,count = outputstr:match("(%x+)%s+(%d+)%s+(%d+)") 39 | 40 | if not h then 41 | return outputstr 42 | end 43 | 44 | 45 | end, 46 | 47 | onresult = function(engine, req, response) 48 | print("GOTTTT MALWARE ".. response) 49 | end 50 | } 51 | ) 52 | 53 | end, 54 | 55 | } 56 | 57 | } 58 | 59 | -------------------------------------------------------------------------------- /lua/frontend_scripts/fileextract/saveall.lua: -------------------------------------------------------------------------------- 1 | -- .lua 2 | -- 3 | -- saveall(.) 4 | -- Saves all files into /tmp/trisul_files , even the large ones that arrive in chunks 5 | -- 6 | -- 7 | TrisulPlugin = { 8 | 9 | id = { 10 | name = "saveall files", 11 | description = "saves all files into /tmp/fx_files", 12 | author = "Unleash", version_major = 1, version_minor = 0, 13 | }, 14 | 15 | 16 | 17 | onload = function() 18 | os.execute("mkdir -p /tmp/trisul_files") 19 | end, 20 | 21 | -- 22 | -- Save everything from the ramfs to /tmp/.. 23 | -- the 'is_chunk' is used to handle large files which are presented to this 24 | -- script in large (5MB or 10MB) chunks (see trisulProbeConfig.xml) 25 | -- 26 | filex_monitor = { 27 | 28 | 29 | -- save all content to /tmp/kk 30 | -- 31 | onfile_http = function ( engine, timestamp, flowkey, path, req_header, resp_header, length , is_chunk ) 32 | 33 | -- you can get 0 length for HTTP 304, etc - skip it 34 | if length == 0 then return; end 35 | 36 | if is_chunk then 37 | local fn = path:match("^.+/(.+)%.%d+.part$") 38 | -- just a chunk , concatenate with prev 39 | -- 40 | T.async:cat( path, "/tmp/trisul_files/"..fn) 41 | 42 | else 43 | -- full file 44 | -- 45 | local fn = path:match("^.+/(.+)$") 46 | 47 | T.async:copy( path, "/tmp/trisul_files/"..fn) 48 | 49 | end 50 | 51 | end, 52 | 53 | } 54 | 55 | } 56 | 57 | -------------------------------------------------------------------------------- /lua/frontend_scripts/heartbleed/tlsrec1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trisulnsm/trisul-scripts/bef85dea72016ef58a98246649c4cf7c4f3faca5/lua/frontend_scripts/heartbleed/tlsrec1.png -------------------------------------------------------------------------------- /lua/frontend_scripts/heartbleed/tlsrec2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trisulnsm/trisul-scripts/bef85dea72016ef58a98246649c4cf7c4f3faca5/lua/frontend_scripts/heartbleed/tlsrec2.png -------------------------------------------------------------------------------- /lua/frontend_scripts/httpserver/README.md: -------------------------------------------------------------------------------- 1 | HTTP Server 2 | ----------- 3 | 4 | This plugin demonstrates the use of a flow based counter. 5 | We tap into the HTTP-Header messages, then pull out the HTTP Server attribute 6 | and then meter based on the web server (apache/nginx..) etc. 7 | 8 | The example is purposely long winded, we extract the entire HTTP header into a LUA table. This is to demonstrate the techniques. You can also directly write a regex to only pull out the HTTP Server attribute. 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /lua/frontend_scripts/inputfilter/fortigate-log/README.md: -------------------------------------------------------------------------------- 1 | # Fortigate-Log : This input filter processes Fortigate logs on UDP 2 | 3 | 4 | ## Running 5 | 6 | Download the two lua files to `/usr/local/var/lib/trisul-probe/domain0/probe0/context0/config/local-lua` 7 | 8 | Then run the following command 9 | 10 | ``` 11 | trisul -demon /usr/local/etc/trisul-probe/domain0/probe0/context0/trisulProbeConfig.xml \ 12 | -mode lua \ 13 | -in /usr/local/var/lib/trisul-probe/domain0/probe0/context0/config/local-lua/fortigate-log.lua 14 | ``` 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /lua/frontend_scripts/inputfilter/lanl-cyber/flows.txt: -------------------------------------------------------------------------------- 1 | 1,9,C3090,N10471,C3420,N46,6,3,144 2 | 1,9,C3538,N2600,C3371,N46,6,3,144 3 | 2,0,C4316,N10199,C5030,443,6,2,92 4 | -------------------------------------------------------------------------------- /lua/frontend_scripts/inputfilter/lanl-cyber/la1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trisulnsm/trisul-scripts/bef85dea72016ef58a98246649c4cf7c4f3faca5/lua/frontend_scripts/inputfilter/lanl-cyber/la1.png -------------------------------------------------------------------------------- /lua/frontend_scripts/inputfilter/lanl-cyber/la2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trisulnsm/trisul-scripts/bef85dea72016ef58a98246649c4cf7c4f3faca5/lua/frontend_scripts/inputfilter/lanl-cyber/la2.png -------------------------------------------------------------------------------- /lua/frontend_scripts/inputfilter/lanl-cyber/la3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trisulnsm/trisul-scripts/bef85dea72016ef58a98246649c4cf7c4f3faca5/lua/frontend_scripts/inputfilter/lanl-cyber/la3.png -------------------------------------------------------------------------------- /lua/frontend_scripts/natsyslog/README.md: -------------------------------------------------------------------------------- 1 | # NAT Syslog 2 | 3 | This script attaches to UDP 514 Syslog and only stores flow records based on NAT syslog events 4 | -------------------------------------------------------------------------------- /lua/frontend_scripts/natsyslog/ipfix-protocol.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- IPFIX raw parser for NAT logs 3 | -- 4 | -- {F15F08A9-F3E0-4722-4D97-31CCF0743E4E} 5 | -- DEFINE_GUID(GUID_xxx,0xF15F08A9,0xF3E0,0x4722,0x4D,0x97,0x31,0xCC,0xF0,0x74,0x3E,0x4E); 6 | 7 | TrisulPlugin = { 8 | 9 | 10 | -- the ID block, you can skip the fields marked 'optional ' 11 | -- 12 | id = { 13 | name = "NAT-IPFIX", 14 | description = "IPFIX protocol default udp 1025", 15 | }, 16 | 17 | 18 | -- protocol_handler block 19 | -- 20 | protocol_handler = { 21 | 22 | -- new protocol for FLOWDIR 23 | control = { 24 | guid = "{F15F08A9-F3E0-4722-4D97-31CCF0743E4E}", 25 | name = "NAT-IPFIX", 26 | host_protocol_guid = '{14D7AB53-CC51-47e9-8814-9C06AAE60189}', 27 | host_protocol_ports = { 1025 } 28 | }, 29 | 30 | 31 | -- WHEN CALLED: when lower layer is constructed and 32 | -- return ( nEaten, nextProtID) 33 | parselayer = function(layer) 34 | return layer:layer_bytes(),nil 35 | end, 36 | 37 | 38 | }, 39 | } 40 | -------------------------------------------------------------------------------- /lua/frontend_scripts/natsyslog/raw-syslogs.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- new_resource_group.lua skeleton 3 | -- {7B431613-9291-49BF-F8D3-73578A445310} 4 | -- DEFINE_GUID(GUID_xxx,0x7B431613,0x9291,0x49BF,0xF8,0xD3,0x73,0x57,0x8A,0x44,0x53,0x10); 5 | -- 6 | TrisulPlugin = { 7 | 8 | id = { 9 | name = "Raw Syslogs", 10 | description = "raw syslogs ", 11 | }, 12 | 13 | 14 | -- resourcegroup block 15 | -- 16 | resourcegroup = { 17 | 18 | -- table control 19 | -- WHEN CALLED: specify details of your new resource group 20 | -- you can use 'trisulctl_probe testbench guid' to get a new GUID 21 | control = { 22 | guid = "{7B431613-9291-49BF-F8D3-73578A445310}", 23 | name = "Raw Syslogs", 24 | description = "Raw text syslogs", 25 | }, 26 | 27 | }, 28 | } 29 | -------------------------------------------------------------------------------- /lua/frontend_scripts/natsyslog/store-tagged-only.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- only store tagged flows 3 | -- 4 | TrisulPlugin = { 5 | 6 | id = { 7 | name = "Only tagged flows ", 8 | description = "Monitor IP flows and generated further metrics ", 9 | }, 10 | 11 | 12 | -- sg_monitor block 13 | sg_monitor = { 14 | 15 | session_guid = '{99A78737-4B41-4387-8F31-8077DB917336}', -- optional 16 | 17 | -- only store flows with tag 18 | flushfilter = function(engine, flow) 19 | local t = flow:tags() 20 | 21 | if #t > 0 then 22 | return true 23 | else 24 | return false 25 | end 26 | end, 27 | 28 | }, 29 | } 30 | 31 | -------------------------------------------------------------------------------- /lua/frontend_scripts/natsyslog/syslog-protocol.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- SYSLOG 3 | -- 4 | TrisulPlugin = { 5 | 6 | 7 | -- the ID block, you can skip the fields marked 'optional ' 8 | -- 9 | id = { 10 | name = "SYSLOG", 11 | description = "Syslog protocol udp 514", 12 | }, 13 | 14 | 15 | -- protocol_handler block 16 | -- 17 | protocol_handler = { 18 | 19 | -- new protocol for FLOWDIR 20 | control = { 21 | guid = "{4323003E-D060-440B-CA26-E146C0C7DB4E}", 22 | name = "SYSLOG", 23 | host_protocol_guid = '{14D7AB53-CC51-47e9-8814-9C06AAE60189}', 24 | host_protocol_ports = { 514 } 25 | }, 26 | 27 | 28 | -- WHEN CALLED: when lower layer is constructed and 29 | -- return ( nEaten, nextProtID) 30 | parselayer = function(layer) 31 | 32 | 33 | return layer:layer_bytes(),nil 34 | end, 35 | 36 | 37 | }, 38 | } 39 | -------------------------------------------------------------------------------- /lua/frontend_scripts/packetstore/README.md: -------------------------------------------------------------------------------- 1 | Fine control of PCAP storage 2 | ====== 3 | 4 | Control packet storage on a per-flow basis. 5 | 6 | 1. You get asked ONCE at start of a flow how you want to handle packet storage 7 | 2. Trisul applies the rule to all packets in that flow 8 | 3. Unlike *BPF* or hardware level filters - the packets are fully analyzed, they just arent stored. 9 | 10 | 11 | Streaming Budget 12 | ------ 13 | 14 | Since the _new flow_ rate is orders of magnitude lesser than the _packet rate_ you have some headroom for some computations. 15 | 16 | 17 | Some guidelines: 18 | 19 | 1. No network IO 20 | 2. Budget typically less than 1 sec per decision 21 | 3. That is usually plenty to lookup a table with millions of entries 22 | 4. Go to Dashboard > System Performance > Packet Drops to see how system is performing 23 | 24 | 25 | Samples 26 | ------ 27 | 28 | 1. skip_ssh.lua -- why waste 100s of GB of disk by storing SSH/SFTP traffic? this simple script skips that 29 | 30 | 2. Also see : [packet_storage documentation](http://trisul.org/docs/lua/packet_storage.html) 31 | -------------------------------------------------------------------------------- /lua/frontend_scripts/packetstore/nopcap_tls.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- packet_storage.lua skeleton 3 | -- 4 | -- TYPE: FRONTEND SCRIPT 5 | -- PURPOSE: Dont store SSL traffic on Port 443 6 | -- DESCRIPTION: No point storing SSL traffic on Port 443, but store non-SSL traffic 7 | -- since a lot of applications are now using Port 443. Use the 8 | -- filter_payload(..) method to check the start of Flow Bytes for TLS prints 9 | -- 10 | -- 11 | -- 12 | -- 13 | -- local dbg=require'debugger' 14 | TrisulPlugin = { 15 | 16 | id = { 17 | name = "No PCAP TLS", description = "No PCAP TLS ", author = "Unleash", 18 | }, 19 | 20 | -- packet_storage block; 21 | -- 22 | packet_storage = { 23 | 24 | -- WHEN CALLED: the first reassembled chunk in each direction is seen 25 | -- use this if you need to see the actual first few bytes of the flow 26 | -- before you decide you want to store the flow or not 27 | filter_payload = function(engine, timestamp, flowkey, direction, seekpos, buff) 28 | 29 | -- we arent handling non Port 443, return -1 for (no opinion) 30 | if flowkey:id():find("p-01BB") == nil then return -1 end 31 | 32 | -- if TLS/SSL fingerprint 33 | local hs_type = buff:hval_8(0) 34 | local tls_version_major = buff:hval_8(1) 35 | local tls_version_minor = buff:hval_8(2) 36 | 37 | 38 | if hs_type == 22 and tls_version_major ==3 and tls_version_minor < 4 then 39 | -- 40 | -- looks like TLS , ignore 41 | -- 42 | return 0 43 | 44 | elseif flowkey:protocol()=='11' and buff:tostring(9,4) == "Q035" then 45 | -- looks like Google Chrome's QUIC , ignore 46 | return 0 47 | else 48 | -- 49 | -- return -1 , no opinion. use the default policy 50 | -- 51 | print("Non TLS or QUIC traffic on Port 443="..flowkey:id()) 52 | return -1 53 | end 54 | end, 55 | 56 | }, 57 | } 58 | -------------------------------------------------------------------------------- /lua/frontend_scripts/packetstore/skip-ssh.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- skip_ssh.lua 3 | -- 4 | -- [FASTPATH] 5 | -- 6 | -- Do not store SSH (Port 22) packets ( a near total waste of your disk $$$ in NSM applications) 7 | -- 8 | TrisulPlugin = { 9 | 10 | id = { 11 | name = "packet_storage", 12 | description = "how to control on flow level packet storage ", 13 | }, 14 | 15 | 16 | packet_storage = { 17 | 18 | -- look at flow tuples and decide 19 | -- return 0 - 6 20 | -- 21 | filter = function(engine, time, flow) 22 | 23 | if flow:portz_readable() == "22" then 24 | -- print("blocking this.."..flow:id() ) 25 | return 0 -- return 0 to say -> 'I vote "Dont Store Packets for this Flow" 26 | else 27 | return -1 -- return -1 to say -> 'I have no opinion, do what you normally would do' 28 | end 29 | end, 30 | }, 31 | 32 | } 33 | -------------------------------------------------------------------------------- /lua/frontend_scripts/packetstore/skip-youtube/README.md: -------------------------------------------------------------------------------- 1 | skip_youtube.lua 2 | ================ 3 | 4 | In the Network Security Monitoring (NSM) paradigm the ability to store and access raw packets is a central feature. 5 | 6 | However not all organizations have the budget to adopt a 'store every bit' posture due to 7 | the prohibitive cost. Fortunately you can adopt a smart storage strategy that can get you close to 8 | 99% of the benefits of the 'store every bit' strategy at maybe 20-30% of the cost. 9 | 10 | This script uses the *packet_storage* LUA script type in Trisul to implement the following policy. 11 | 12 | 1. All Netflix, Youtube, and Twitter packets arent stored based on Domain name patterns 13 | 2. In many environments, this could be a 50-60% savings right away 14 | 3. You can extend this framework to cull trusted high volume and/or encrypted flows to suit your enterprise 15 | 16 | How this works 17 | --------------- 18 | 19 | 1. The Passive DNS extractor [2] is a pre-requisite. It builds a LevelDB database containing real time IP->Domain mapping 20 | 2. skip_youtube.lua uses the *filter* method to check each new flow _(not each packet)_ against the allowed list and determines the storage policy as 'discard' or 'allow' 21 | 22 | 23 | Installation 24 | ------------ 25 | 26 | > You need to install LevelDB on the probe first 27 | > Ubuntu: `apt-get install libleveldb1` 28 | > CentOS: `yum install leveldb` 29 | 30 | 31 | 1. Install the "Passive DNS" script first 32 | 2. Download the skip_youtube.lua and trisul_leveldb.lua into the Local LUA directory on the probe. Use `trisulctl_probe list lua default@probe0` to view the directory name 33 | 3. Restart the probe. Voila! 34 | 35 | References 36 | ---------- 37 | 38 | 1. LUA *packet_storage* API reference : https://www.trisul.org/docs/lua/packet_storage.html 39 | 2. Passive DNS extractor : https://github.com/trisulnsm/apps/blob/master/analyzers/passive-dns/README.md 40 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /lua/frontend_scripts/pingtunnel/ping-alert.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trisulnsm/trisul-scripts/bef85dea72016ef58a98246649c4cf7c4f3faca5/lua/frontend_scripts/pingtunnel/ping-alert.png -------------------------------------------------------------------------------- /lua/frontend_scripts/protocol_tree/README.md: -------------------------------------------------------------------------------- 1 | Protocol Tree Maker 2 | =================== 3 | 4 | Pop this script into Trisul to create a protocol tree of your network traffic. 5 | 6 | 7 | -------------------------------------------------------------------------------- /lua/frontend_scripts/reassembly/README.md: -------------------------------------------------------------------------------- 1 | # packet reassembly 2 | 3 | (Frontend packet pipeline Trisul script) 4 | 5 | Scripts in this and subdirectories demonstrate how you can plug your LUA scripts into the TCP packet reassembly framework. 6 | 7 | Open each lua file to see what it does 8 | 9 | -------------------------------------------------------------------------------- /lua/frontend_scripts/reassembly/flow_attributes.lua: -------------------------------------------------------------------------------- 1 | -- flow-attributes.lus 2 | -- 3 | -- Prints HTTP Hosts - a flow attribute discovered by the Trisul Reassembly Pipeline 4 | -- 5 | -- 6 | TrisulPlugin = { 7 | 8 | 9 | id = { 10 | 11 | name = "flowattrib", 12 | description = "prints HTTP Host attributes ", 13 | }, 14 | 15 | onload = function() 16 | 17 | 18 | end, 19 | 20 | 21 | reassembly_handler = { 22 | 23 | 24 | 25 | -- Attributes are discovered by Trisul Reassembly, you can 26 | -- see entire list of attributes in the LUA Docs. Here we 27 | -- just print the HTTP-Host values 28 | 29 | onattribute = function(engine, time, flow, attr_type, attr_value ) 30 | 31 | if attr_type == "Host" then 32 | print("Attribute "..attr_type.." = "..attr_value) 33 | end 34 | end, 35 | }, 36 | 37 | } 38 | -------------------------------------------------------------------------------- /lua/frontend_scripts/reassembly/haSSH/hassh-counters.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- ssh_counters 3 | -- 4 | -- TYPE: FRONTEND SCRIPT 5 | -- PURPOSE: For SSH monitor 6 | -- DESCRIPTION: POC of SSH client and server print 7 | -- 8 | TrisulPlugin = { 9 | 10 | id = { 11 | name = "SSH hashes - Hassh", 12 | }, 13 | 14 | countergroup = { 15 | 16 | control = { 17 | guid = "{E49AA7D0-3DC8-46AC-E278-5DD07B298F0A}", 18 | name = "HaSSH Prints", 19 | description = "HaSSH Prints", 20 | bucketsize = 60, 21 | }, 22 | 23 | -- meters table 24 | -- id, type of meter, toppers to track, Name, units, units-short 25 | -- 26 | meters = { 27 | { 0, T.K.vartype.COUNTER, 20, 20, "SSH Client Types", "Client Hits", "hits" }, 28 | { 1, T.K.vartype.COUNTER, 20, 20, "SSH Server Types", "Server Hits", "hits" }, 29 | }, 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /lua/frontend_scripts/reassembly/haSSH/hassh1.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trisulnsm/trisul-scripts/bef85dea72016ef58a98246649c4cf7c4f3faca5/lua/frontend_scripts/reassembly/haSSH/hassh1.gif -------------------------------------------------------------------------------- /lua/frontend_scripts/reassembly/haSSH/md5ffi.lua: -------------------------------------------------------------------------------- 1 | -- Setup LUAJIT2.1 FFI into libcrypto.so for MD5 2 | local ffi=require('ffi') 3 | 4 | local status,C 5 | for _,lib in ipairs( {'libcrypto.so.1.0.2k', 'libcrypto.so.1.0.0'} ) 6 | do 7 | status, C = pcall(function() return ffi.load(lib) end) 8 | if status then break end 9 | end 10 | if not status then error "Cant load FFI libcrypto version " end 11 | 12 | -- local dbg=require'debugger' 13 | ffi.cdef[[ 14 | typedef struct MD5state_st 15 | { 16 | unsigned int A,B,C,D; 17 | unsigned int Nl,Nh; 18 | unsigned int data[16]; 19 | unsigned int num; 20 | } MD5_CTX; 21 | int MD5_Init(MD5_CTX *c); 22 | int MD5_Update(MD5_CTX *c, const void *data, size_t len); 23 | int MD5_Final(unsigned char *md, MD5_CTX *c); 24 | ]] 25 | 26 | -- ffi based MD5 27 | function md5sum( input) 28 | local hashresults = ffi.new("uint8_t[16]") 29 | ctx = ffi.new'MD5_CTX' 30 | C.MD5_Init(ctx) 31 | C.MD5_Update(ctx,input,#input) 32 | C.MD5_Final(hashresults,ctx) 33 | return T.util.bin2hex(ffi.string(hashresults,16)) 34 | end 35 | 36 | -------------------------------------------------------------------------------- /lua/frontend_scripts/reassembly/haSSH/ssh-spy.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- ssh-spy 3 | -- 4 | -- 1. Port Independent SSH-detector 5 | -- 2. Connects to Trisul TCP reassembly 6 | -- 3. Uses PDURecord helper library to interface wth ssh-dissect 7 | -- 8 | 9 | local PDURecord = require'pdurecord' 10 | local SSHDissector = require'ssh-dissect' 11 | 12 | TrisulPlugin = { 13 | 14 | id = { name = "ssh-spy", description = "SSH KE/Crypto/HMAC tracker", }, 15 | 16 | onload = function() 17 | T.Pimpl = {} -- on non standard ports 18 | end, 19 | 20 | -- reassembly_handler block 21 | reassembly_handler = { 22 | 23 | -- run the PDU streamer , which will callback into the dissector 24 | onpayload = function(engine, timestamp, flowkey, direction, seekpos, buffer) 25 | 26 | local ctl = T.Pimpl[flowkey:id()] 27 | if not ctl then 28 | if seekpos==0 and buffer:tostring():find("^SSH%-2%.0") == 1 then 29 | print("New haSSH Analyzer attached to a PDURecord f="..flowkey:id() ) 30 | local ssh1, ssh2 = SSHDissector.new_pair() 31 | local ins = PDURecord.new(flowkey:id(), ssh1) 32 | local outs = PDURecord.new(flowkey:id(), ssh2) 33 | ctl = { [0]= ins, [1]= outs } 34 | T.Pimpl[flowkey:id()] = ctl 35 | else 36 | engine:disable_reassembly(flowkey:id()) 37 | return 38 | end 39 | end 40 | 41 | local pdur = ctl[direction] 42 | pdur.engine=engine 43 | pdur.timestamp=timestamp 44 | pdur.flowid = flowkey:id() 45 | pdur:push_chunk(seekpos, buffer:tostring()) 46 | end, 47 | 48 | -- 49 | onterminateflow = function(engine, timestamp, flowkey) 50 | -- print("Terminating SSH Analyzer attached to a PDURecord f="..flowkey:id() ) 51 | T.Pimpl[flowkey:id()] = nil 52 | end, 53 | 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /lua/frontend_scripts/reassembly/ja3/prints/README.md: -------------------------------------------------------------------------------- 1 | ja3fingerprint.json 2 | ==================== 3 | 4 | > ## Moved 5 | > **JA3 Prints Database moved to new repository** [trisulnsm/ja3prints](https://github.com/trisulnsm/ja3prints) 6 | > Please submit any pull requests there. Thanks ! 7 | 8 | A database of JA3 Fingerprints in JSON format. 9 | 10 | The original sources of these prints can be found as comments. If you come across prints not seen in the list, please submit a pull request. 11 | 12 | For more about JA3 : https://github.com/salesforce/ja3 13 | 14 | 15 | Changes 16 | ======== 17 | 18 | ```` 19 | Mar 1 2018 55 Malware Prints thanks to JunPritsker from malware-traffic-analysis PCAPS 20 | Jan 8 2018 Converted and added about 160 prints from John Althouse 21 | 22 | ```` 23 | 24 | 25 | 26 | Other files in this directory 27 | ------------------------------ 28 | 29 | 30 | There are some other utility scripts in this directory to convert from various different formats into the JSON format. 31 | 32 | 33 | toja3.rb 34 | --------- 35 | 36 | Quick script to generate a ja3_hash from this awesome fingerprint DB from https://github.com/LeeBrotherston/tls-fingerprinting/blob/master/fingerprints/fingerprints.json 37 | 38 | To run 39 | 40 | ```` 41 | ruby toja3.rb fingerprints.json > ja3_fingerprints.json 42 | ```` 43 | 44 | 45 | to_fingerprints.rb 46 | ------------------- 47 | 48 | Reverse of toja3.rb. 49 | 50 | 51 | get_ja3.rb 52 | ------------ 53 | 54 | Ruby Trisul Remote Protocol (TRP) script to automatically correlate unkown ja3 prints from apache webserver logs. 55 | 56 | 57 | 58 | csv_toja3.rb 59 | -------------- 60 | 61 | Convert CSV to JSON. 62 | -------------------------------------------------------------------------------- /lua/frontend_scripts/reassembly/ja3/prints/csv_toja3.rb: -------------------------------------------------------------------------------- 1 | require 'json' 2 | require 'csv' 3 | 4 | raise "Usage : csv_toja3 " unless ARGV.length==1 5 | 6 | CSV_ORDER = { 7 | ja3_hash:0, 8 | description:1 9 | } 10 | 11 | 12 | rows = CSV.foreach( ARGV[0], {headers:false,skip_lines:/^#/ }) do |row| 13 | out = { 14 | :desc => row[1].gsub(/"/,''), 15 | :ja3_hash => row[0], 16 | :ja3_string => '' 17 | } 18 | puts out.to_json 19 | end 20 | 21 | -------------------------------------------------------------------------------- /lua/frontend_scripts/reassembly/ja3/prints/tofingerprints.rb: -------------------------------------------------------------------------------- 1 | require 'json' 2 | 3 | # 4 | # Converts JA3 format to Fingerprints found on https://github.com/LeeBrotherston/tls-fingerprinting 5 | # skips fields that are not used by JA3 such as signature algorithms 6 | # 7 | raise "Usage : tofingerprints " unless ARGV.length==1 8 | 9 | File.foreach(ARGV.shift) do |line| 10 | next if line.chomp("\n").length==0 11 | out = {} 12 | 13 | ja3 = JSON.parse(line) 14 | ja3_str=ja3['ja3_str'].split(",") 15 | 16 | out["id"]=0 17 | out["desc"]=ja3['desc'] 18 | out["record_tls_version"]="0x0301" 19 | out["tls_version"]="0x#{ja3_str[0].to_i.to_s(16).upcase.rjust(4,'0')}" 20 | out["ciphersuite_length"]="0x#{(ja3_str[1].split("-").length*2).to_i.to_s(16).upcase.rjust(4,'0')}" 21 | {1=>"ciphersuite",2=>"extensions",3=>"e_curves"}.each_pair do |idx,str| 22 | ja3_str[idx]=ja3_str[idx] || "" 23 | out[str]=ja3_str[idx].split("-").collect do | suite | 24 | "0x#{suite.to_i.to_s(16).upcase.rjust(4,'0')}" 25 | end.join(" ") 26 | end 27 | out["ec_point_fmt"]="0x#{ja3_str[4].to_i.to_s(16).upcase.rjust(2,'0')}" 28 | puts out.to_json 29 | end 30 | -------------------------------------------------------------------------------- /lua/frontend_scripts/reassembly/ja3/prints/toja3.rb: -------------------------------------------------------------------------------- 1 | require 'json' 2 | require 'digest/md5' 3 | 4 | raise "Usage : toja3 " unless ARGV.length==2 5 | 6 | 7 | JA3_FIELD_ORDER = %w(record_tls_version ciphersuite extensions e_curves ec_point_fmt) 8 | 9 | File.foreach(ARGV.shift) do |line| 10 | 11 | j = JSON.parse(line) 12 | 13 | 14 | jarr=JA3_FIELD_ORDER.collect do |f| 15 | v = j[f] || "" 16 | v.split(/\s+/).collect(&:hex).join('-') 17 | end 18 | 19 | ja3_str = jarr.join(',') 20 | out = { :desc => j['desc'], :ja3_hash => Digest::MD5.hexdigest(ja3_str), :ja3_str => ja3_str } 21 | 22 | puts out.to_json 23 | end 24 | -------------------------------------------------------------------------------- /lua/frontend_scripts/reassembly/reass_filter.lua: -------------------------------------------------------------------------------- 1 | -- reass_filter.lua 2 | -- shows how you can use the "filter" method to 3 | -- control which flows you want to reassemble 4 | -- 5 | TrisulPlugin = { 6 | 7 | id = { 8 | name = "reass_filter", 9 | description = "Only interested in reassembly of particular flows ", 10 | }, 11 | 12 | 13 | reassembly_handler = { 14 | 15 | -- only interested in reassembling flows involving this IP address 16 | -- by using the filter method you save a CPU cycles and memory on the 17 | -- Trisul frontend pipeline 18 | -- in this example - we only want to reassembly from/to one IP address 19 | -- 20 | -- read the docs : return true = yes, we are interested in this one 21 | -- 22 | filter = function(engine, time, flow) 23 | if flow:ipa_readable() == "209.216.249.58" or 24 | flow:ipz_readable() == "209.216.249.58" then 25 | return true 26 | else 27 | return false 28 | end 29 | end, 30 | 31 | -- a new flow has started 32 | -- due to the filter, only flows involving that IP generate this callback 33 | -- we just print the IP 34 | onnewflow = function(engine, time, flow) 35 | local fstr = string.format("%2s %-15s %-6s %-15s %-6s %20s \n", 36 | flow:protocol(), 37 | flow:ipa_readable(), flow:porta_readable(), 38 | flow:ipz_readable(), flow:portz_readable(), 39 | os.date("%c",time) 40 | ); 41 | print( "SCRIPT2 : new flow" .. fstr) 42 | end, 43 | 44 | -- do some interestig things with the reassembled payload 45 | -- that are streaming through this function 46 | onpayload = function(engine, time, flow, dir, seekpos, buff ) 47 | print( flow:id()) 48 | end, 49 | 50 | -- some clean up here 51 | onterminateflow = function(engine, time, flow) 52 | print( "SCRIPT2 : terminated flow" .. flow:id()) 53 | end, 54 | 55 | 56 | }, 57 | 58 | } 59 | -------------------------------------------------------------------------------- /lua/frontend_scripts/reassembly/save_payloads/README.md: -------------------------------------------------------------------------------- 1 | savetcp.lua 2 | =========== 3 | 4 | Saves reassembled TCP Payloads into separate files. 5 | 6 | 7 | How to run 8 | ---------- 9 | 10 | Save the lua file into [one of the directories](https://www.trisul.org/docs/lua/basics.html#installing_and_uninstalling) Trisul will search for Lua scripts. 11 | 12 | ### 1. Run over a PCAP file 13 | 14 | We make sure we create the special development context called `debug0` 15 | 16 | ````bash 17 | 18 | trisulctl_probe create context debug0 19 | 20 | cp savetcp.lua /usr/local/var/lib/trisul-probe/domain0/probe0/context_debug0/config/local-lua 21 | 22 | trisulctl_probe testbench run /home/mgill/pcaps/upload.tcpd 23 | 24 | ```` 25 | 26 | Then you get the payload files in `/tmp` 27 | 28 | 29 | ````bash 30 | 31 | mgill@mgill14$ ls -1 /tmp/ 32 | /tmp/payloads_06A:C0.A8.01.0B:p-8050_A9.2C.A2.C0:p-01BB_0 33 | /tmp/payloads_06A:C0.A8.01.0B:p-8050_A9.2C.A2.C0:p-01BB_1 34 | /tmp/payloads_06A:C0.A8.01.0B:p-8074_A9.2C.A2.C0:p-01BB_0 35 | /tmp/payloads_06A:C0.A8.01.0B:p-8074_A9.2C.A2.C0:p-01BB_1 36 | /tmp/payloads_06A:C0.A8.01.0B:p-820D_D8.3A.C4.2E:p-01BB_0 37 | /tmp/payloads_06A:C0.A8.01.0B:p-820D_D8.3A.C4.2E:p-01BB_1 38 | 39 | ```` 40 | 41 | * _0 contains payloads in IN direction 42 | * _1 for OUT direction 43 | 44 | 45 | 46 | 47 | ### Run over live traffic 48 | 49 | Copy to the file to a live Trisul context. All Trisul deployments have a context called `context0`. Use that 50 | 51 | 52 | ````bash 53 | cp savetcp.lua /usr/local/var/lib/trisul-probe/domain0/probe0/context0/config/local-lua 54 | ```` 55 | 56 | Then just restart Trisul probe 57 | 58 | ```` 59 | trisulctl_probe restart context default@probe0 60 | 61 | ```` 62 | 63 | 64 | 65 | -------------------------------------------------------------------------------- /lua/frontend_scripts/reassembly/ssh-analyzer/README.md: -------------------------------------------------------------------------------- 1 | ssh passive analysis 2 | ==================== 3 | 4 | 5 | Uses the techniques described in [Traffic Analysis of Secure Shell ](https://trisul.org/blog/analysing-ssh/post.html) to 6 | 7 | 1. detect successful logins 8 | 2. detect keystrokes after a successful login 9 | 3. detect SSH Tunnels, forward or reverse 10 | 11 | 12 | The files are 13 | - ssh_dissect.lua -- SSH protocol analyzer 14 | - ssh-spy.lua -- connects the ssh_dissect.lua into Trisul TCP Reassembly 15 | - ssh-alert-group.lua -- a new alert group to house the alerts 16 | 17 | 18 | The scripts uses the `PDURecord` and `SweepBuffer` helpers from the [BitMaul stream to PDU library](https://github.com/trisulnsm/trisul-scripts/tree/master/lua/bitmaul) 19 | 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /lua/frontend_scripts/reassembly/ssh-analyzer/ssh-spy.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- ssh-spy 3 | -- 4 | -- 1. Port Independent SSH-detector 5 | -- 2. Connects to Trisul TCP reassembly 6 | -- 3. Uses PDURecord helper library to interface wth ssh_dissect 7 | -- 8 | 9 | local PDURecord = require'pdurecord' 10 | local SSHDissector = require'ssh_dissect' 11 | 12 | TrisulPlugin = { 13 | 14 | id = { name = "ssh-spy", description = "SSH KE/Crypto/HMAC tracker", }, 15 | 16 | onload = function() 17 | T.Pimpl = {} -- on non standard ports 18 | end, 19 | 20 | -- reassembly_handler block 21 | reassembly_handler = { 22 | 23 | -- run the PDU streamer , which will callback into the dissector 24 | onpayload = function(engine, timestamp, flowkey, direction, seekpos, buffer) 25 | 26 | local ctl = T.Pimpl[flowkey:id()] 27 | if not ctl then 28 | if seekpos==0 and buffer:tostring():find("^SSH%-2%.0") == 1 then 29 | print("New SSH Analyzer attached to a PDURecord f="..flowkey:id() ) 30 | local ssh1, ssh2 = SSHDissector.new_pair() 31 | local ins = PDURecord.new(flowkey:id(), ssh1) 32 | local outs = PDURecord.new(flowkey:id(), ssh2) 33 | ctl = { [0]= ins, [1]= outs } 34 | T.Pimpl[flowkey:id()] = ctl 35 | else 36 | engine:disable_reassembly(flowkey:id()) 37 | return 38 | end 39 | end 40 | 41 | local pdur = ctl[direction] 42 | pdur.engine=engine 43 | pdur.timestamp=timestamp 44 | pdur:push_chunk(seekpos, buffer:tostring()) 45 | end, 46 | 47 | -- 48 | onterminateflow = function(engine, timestamp, flowkey) 49 | -- print("Terminating SSH Analyzer attached to a PDURecord f="..flowkey:id() ) 50 | T.Pimpl[flowkey:id()] = nil 51 | end, 52 | 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /lua/frontend_scripts/reassembly/ssh-analyzer/ssh_alert_group.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- SSH Related Alerts 3 | -- 4 | -- we alert when 5 | -- 1. non-ETM ciphers are used 6 | -- 2. on successful login 7 | -- 3. on successful login with keystroke 8 | -- 3. ssh v1 is used 9 | -- 4. on keystroke on port forward SSH 10 | -- 11 | -- this file just creates the new alert group, the actual alerting 12 | -- happens in ssh_dissect.lua 13 | -- 14 | TrisulPlugin = { 15 | 16 | id = { 17 | name = "SSH Alerts", 18 | description = "On ssh events", 19 | author = "Trisul", 20 | }, 21 | 22 | 23 | alertgroup = { 24 | 25 | -- WHEN CALLED: specify details of your new alert group 26 | -- you can use 'trisulctl_probe testbench guid' to get a new GUID 27 | control = { 28 | guid = "{E713ED84-F2D9-4469-148C-00C119992926}", 29 | name = "SSH-Alerts", 30 | description = "SSH shell login events", 31 | }, 32 | 33 | }, 34 | } 35 | -------------------------------------------------------------------------------- /lua/frontend_scripts/reassembly/tls-sni/README.md: -------------------------------------------------------------------------------- 1 | Server Name Indication based metrics 2 | ==================================== 3 | 4 | 1. A new counter group called "SNI" 5 | 1. meter 0 : bandwidth per hostname 6 | 2. meter 1 : flows/hits per hostname 7 | 8 | 2. A new resource group called "SNI" contains IP->SNI hostname mapping 9 | 10 | 11 | ** Requires the BitMaul library ** 12 | -------------------------------------------------------------------------------- /lua/frontend_scripts/reassembly/tls-version-count/README.md: -------------------------------------------------------------------------------- 1 | # TLS Version, Certificate Algorithm metrics 2 | 3 | 4 | -------------------------------------------------------------------------------- /lua/frontend_scripts/reassembly/tls-version-count/counter_pk_algos.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- Counter : Public Key Algorithms seen 3 | -- 4 | TrisulPlugin = { 5 | 6 | -- the ID block, you can skip the fields marked 'optional ' 7 | -- 8 | id = { 9 | name = "TLS Public Key Algo", 10 | description = "Meter cert public key algorithms", -- optional 11 | }, 12 | 13 | -- 14 | countergroup = { 15 | 16 | -- control table 17 | -- specify details of your new counter group you can use 18 | -- 'trisulctl_probe testbench guid' to get a new GUID 19 | control = { 20 | guid = "{88F603AE-4519-4E3D-E1C8-D1882E398724}", 21 | name = "TLS Public Key Algo", 22 | description = "Counts cert pk algos", 23 | bucketsize = 30, 24 | }, 25 | 26 | -- meters table 27 | -- id, type of meter, toppers to track, bottom-ers to track, Name, units, units-short 28 | -- 29 | meters = { 30 | { 0, T.K.vartype.COUNTER, 10, 0, "Certs", "certs", "Certs" }, 31 | }, 32 | }, 33 | } 34 | -------------------------------------------------------------------------------- /lua/frontend_scripts/reassembly/tls-version-count/counter_sig_algos.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- Counter : Public Key Algorithms seen 3 | -- 4 | TrisulPlugin = { 5 | 6 | -- the ID block, you can skip the fields marked 'optional ' 7 | -- 8 | id = { 9 | name = "TLS Sig Algo", 10 | description = "Meter cert signature algorithms", -- optional 11 | }, 12 | 13 | -- 14 | countergroup = { 15 | 16 | -- control table 17 | -- specify details of your new counter group you can use 18 | -- 'trisulctl_probe testbench guid' to get a new GUID 19 | control = { 20 | guid = "{C90640F6-ACD1-4BE5-92FF-A417DC6A987A}", 21 | name = "TLS Sig Algo", 22 | description = "Counts cert signature algos", 23 | bucketsize = 30, 24 | }, 25 | 26 | -- meters table 27 | -- id, type of meter, toppers to track, bottom-ers to track, Name, units, units-short 28 | -- 29 | meters = { 30 | { 0, T.K.vartype.COUNTER, 10, 0, "Certs", "certs", "Certs" }, 31 | }, 32 | }, 33 | } 34 | -------------------------------------------------------------------------------- /lua/frontend_scripts/simplecounter/README.md: -------------------------------------------------------------------------------- 1 | Counters 2 | ======== 3 | 4 | This directory contains scripts related to metering. 5 | 6 | 7 | 1. rstcounter.lua 8 | -------------- 9 | 10 | A new counter group for hosts with two meters (total volume + RST count) 11 | -------------------------------------------------------------------------------- /lua/hub_scripts/radius-aaa/README.md: -------------------------------------------------------------------------------- 1 | # Radius Parsers 2 | 3 | 4 | This directory contains RADIUS dump parsers in LUA for use with Trisul IPDR AAAING process. 5 | 6 | 7 | See `man trisul_aaaing` 8 | 9 | 10 | ## Purpose 11 | 12 | The purpose of this repository is to make available common parsers for various RADIUS AAA formats. 13 | 14 | The AAAING process can use a LUA file to parse each line in any format. The lua files in this directory 15 | support various formats. 16 | 17 | 18 | Modify the dump filename pattern as per your choice. 19 | 20 | 21 | ## LUA Script Docs 22 | 23 | The purpose of the AAA LUA script 24 | 3. Specify a file prefix which would trigger this lua script 25 | 1. Extract a timestamp from a file name 26 | 2. Parse a single line in the AAA dump file and return a LUA table 27 | 28 | 29 | 30 | The structure is shown below 31 | 32 | ``` 33 | 34 | -- return a prefix that triggers this script 35 | function getfileprefix() 36 | return 'net1aaa*'; 37 | end 38 | 39 | 40 | -- From the file name return a tv_sec (unix epoch) 41 | function timestampfromfilename(fn) 42 | local ts = fn:match("(%d%d%d%d%d%d%d%d%d%d)") 43 | return tonumber(ts) 44 | end 45 | 46 | 47 | 48 | -- return a table { privateip, timefrom, timeto, user, subscriberid, fulline, nasip } 49 | -- radacctid,acctsessionid,acctuniqueid,customer_id,nasipaddress,nasportid, 50 | -- acctstarttime,acctupdatetime,acctstoptime,acctsessiontime, 51 | -- callingstationid,framed_ipv_4_address,framed_ipv_6_address,delegated_ipv6_prefix 52 | 53 | function parseline(theline) 54 | 55 | 56 | return { 57 | customer_id, 58 | framedipv4, 59 | tounix(acctstarttime), 60 | tounix(acctendtime), 61 | subscriber_id, 62 | theline , 63 | nasip 64 | } 65 | 66 | end 67 | 68 | ``` 69 | 70 | 71 | See the samples 72 | 73 | 74 | -------------------------------------------------------------------------------- /lua/libs/README.md: -------------------------------------------------------------------------------- 1 | Libraries 2 | ========= 3 | 4 | Libraries we use in Trisul Network Analytics. 5 | 6 | Mostly LuaJIT based. 7 | 8 | 9 | ## Libraries 10 | 11 | 1. tris_leveldb : LevelDB wrapper. 12 | 2. ipprefixdb: Large scale and fast IP subnet database used for Geo lookups 13 | 3. geodb: Compilers to free CSV files released by popular GeoIP providers 14 | 15 | 16 | -------------------------------------------------------------------------------- /lua/libs/geodb/README.md: -------------------------------------------------------------------------------- 1 | GeoDB - compilers for various Geo Databases 2 | =========================================== 3 | 4 | These compilers convert IP subnet to Geo mappings from a CSV format into a LevelDB backend. 5 | 6 | > *Purpose* You could use the API's that come with these databases. Those who desire a common format for storing IP Subnet information will find this useful. 7 | 8 | 9 | The following compilers are available 10 | ---------------- 11 | 12 | 1. `compile_geolite2` : MaxMind GeoLite2 from https://dev.maxmind.com/geoip/geoip2/geolite2/ 13 | 2. `compile_ip2location` : IP2Location LITE : from https://lite.ip2location.com/ 14 | 15 | ## Running the compilers 16 | 17 | Here are some rules 18 | 19 | 1. All CSV files must be UNZIPped 20 | 2. Put all the CSV files in a single directory 21 | 22 | ### IP2Location 23 | 24 | To compile the IP2Location lists in directory `ip2loc` into a LevelDB prefix database `ip2loc.level` 25 | 26 | ````lua 27 | luajit compile_ip2loc.lua /home/demo/directory/with/IP2Location/CSV/Files ip2loc.level 28 | ```` 29 | 30 | ### GeoLite2 31 | 32 | To compile the GeoLite2 lists in directory `geolite2` into a LevelDB prefix database `ip2loc.level` 33 | 34 | ````lua 35 | luajit compile_ip2loc.lua /home/demo/directory/with/IP2Location/CSV/Files ip2loc.level 36 | ```` 37 | 38 | 39 | ## Testing 40 | 41 | In the `helpers` directory here, you can find a `querytool.lua` that you can use to check any IP in any of the databases 42 | 43 | `Usage : querytool ` 44 | 45 | 46 | Showing ASN query 47 | 48 | ````bash 49 | 50 | cd helpers 51 | luajit querytool.lua ../my.level ASN 45.118.180.88 52 | 53 | ```` 54 | Showing query for CITYCODE 55 | 56 | ````bash 57 | luajit querytool.lua ../my.level CITYCODE 4143637 58 | CITYCODE 59 | By Raw 4143637 60 | ZZZZ4143637 US_DE_Middletown 61 | ```` 62 | 63 | -------------------------------------------------------------------------------- /lua/libs/geodb/helpers/ip6.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- IPv6 to Trisul Key format 3 | -- 4 | 5 | local ffi=require'ffi' 6 | local dbg=require'debugger' 7 | 8 | ffi.cdef [[ 9 | typedef uint32_t socklen_t; 10 | int inet_pton(int af, const char *src, void *dst); 11 | const char *inet_ntop(int af, const void *src, char *dst, socklen_t size); 12 | static const int AF_INET6=10; 13 | ]] 14 | 15 | IP6 = {} 16 | 17 | function IP6.ip6_to_bin(ip6) 18 | local binip6 = ffi.new(' char [16]') 19 | ffi.C.inet_pton(ffi.C.AF_INET6, ip6, binip6); 20 | return binip6 21 | end 22 | 23 | function IP6.bin2hex(binarr,len) 24 | local h = {} 25 | for i = 1 , len do 26 | h[#h+1]=string.format("%02X",bit.band(binarr[i-1],0xff)) 27 | end 28 | return table.concat(h) 29 | end 30 | 31 | function IP6.ip6_to_key(ip6) 32 | return IP6.bin2hex(IP6.ip6_to_bin(ip6),16) 33 | 34 | end 35 | 36 | -- 2001:1900:5:2:2::2ae0/125 37 | -- return start/end 38 | function IP6.ip6_cidr(ip6, cidr) 39 | 40 | -- cidr masks 41 | local bitmask = ffi.new(' char [16]') 42 | local startoct = math.floor(cidr/8) 43 | local maskbits=cidr-startoct*8 44 | local mask=math.pow(2,8-maskbits)-1 45 | for i = 1 , startoct do 46 | bitmask[i-1]=tonumber(0) 47 | end 48 | for i = startoct+1, 16 do 49 | bitmask[i-1]=tonumber(mask) 50 | mask=0xff 51 | end 52 | 53 | 54 | local ip6num = IP6.ip6_to_bin(ip6) 55 | 56 | for i=0, 15 do 57 | ip6num[i]=bit.bor(bitmask[i],ip6num[i]) 58 | end 59 | 60 | return IP6.ip6_to_key(ip6,16), IP6.bin2hex(ip6num,16) 61 | end 62 | 63 | function IP6.tests() 64 | print( IP6.ip6_to_key("fe80::a60:6eff:fed9:b6bd")) 65 | local f,l = IP6.ip6_cidr("2001:1900:5:2:2::2ae0",125) 66 | print(f.." to "..l) 67 | end 68 | 69 | return IP6 70 | 71 | -------------------------------------------------------------------------------- /lua/libs/geodb/helpers/querytool.lua: -------------------------------------------------------------------------------- 1 | local ipdb=require'ipprefixdb' 2 | 3 | 4 | if #arg ==2 then 5 | dbpath = arg[1] 6 | dbname = nil 7 | ipaddr = arg[2] 8 | elseif #arg ==3 then 9 | dbpath = arg[1] 10 | dbname = arg[2] 11 | ipaddr = arg[3] 12 | else 13 | print("Usage : querytool leveldb-database type ip-address") 14 | return 15 | end 16 | 17 | local db1=ipdb:new() 18 | db1:open(dbpath,true) 19 | 20 | print(dbname) 21 | if dbname then 22 | db1:set_databasename(dbname) 23 | end 24 | 25 | print("By IP ".. ipaddr) 26 | print(db1:get_dotted_ip(ipaddr)) 27 | 28 | print("By Raw ".. ipaddr) 29 | print(db1:getraw(ipaddr)) 30 | 31 | db1:close() 32 | 33 | 34 | -------------------------------------------------------------------------------- /lua/libs/ipprefixdb/README.md: -------------------------------------------------------------------------------- 1 | ipprefix db 2 | ================== 3 | 4 | A library that does stores and queries very large IP ranges using LevelDB backend. 5 | 6 | Can be used for applications in routing, network security. 7 | 8 | Features 9 | ---- 10 | 11 | 1. Fast 12 | 2. Easy 13 | 3. Low memory use (considering 2GB limit in LuaJIT) 14 | 4. Designed to support millions of prefixes 15 | 5. Supports multiple databases 16 | 17 | Pre-requisities 18 | ----- 19 | 20 | You must have LevelDB installed on your system , we're looking for `libleveldb.so` somewhere in your library search path. 21 | 22 | ## Usage 23 | 24 | ````lua 25 | local IPPrefixDB=require'ipprefixdb' 26 | 27 | -- create and open 28 | local db1 = IPPrefixDB.new() 29 | db1:open("/tmp/ipdb.level") 30 | 31 | -- store 32 | 33 | db1:put("192.168.0.0/16", "192 network private") 34 | db1:put("192.168.4.0/8", "Video servers 4/8") 35 | db1:put(16802560,16802815, "Using IP 32 bit numbers ") 36 | db1:put("C0.A8.01.01","C0.A8.01.FF", "Using trisul keys 1.1 ") 37 | db1:put("192.168.4.18","192.168.4.22", "Using dotted IP without even a proper subnet ") 38 | 39 | 40 | -- get 41 | print ( db1:get("192.168.4.19") ) 42 | print ( db1:get("18.82.8.82") ) 43 | print ( db1:get("18.82.8.82") ) 44 | print ( db1:get("A8.83.8F.FA") ) 45 | print ( db1:get(8834882) ) 46 | 47 | 48 | -- dump 49 | db1:dump() 50 | 51 | -- closing 52 | db1:close() 53 | 54 | ```` 55 | 56 | ## Query tool 57 | 58 | Use the `querytool.lua` script to query the database 59 | 60 | 61 | Usage 62 | ````lua 63 | luajit querytools.lua [optional-database-prefix] 64 | ```` 65 | 66 | -------------------------------------------------------------------------------- /lua/libs/ipprefixdb/helpers/querytool.lua: -------------------------------------------------------------------------------- 1 | local ipdb=require'ipprefixdb' 2 | 3 | if #arg ==2 then 4 | dbpath = arg[1] 5 | dbname = nil 6 | ipaddr = arg[2] 7 | elseif #arg ==3 then 8 | dbpath = arg[1] 9 | dbname = arg[2] 10 | ipaddr = arg[3] 11 | 12 | else 13 | print("Usage : querytool leveldb-database type ip-address") 14 | return 15 | end 16 | 17 | local db1=ipdb:new() 18 | db1:open(dbpath,true) 19 | 20 | print(dbname) 21 | if dbname then 22 | db1:set_databasename(dbname) 23 | end 24 | 25 | 26 | print("For IP ".. ipaddr) 27 | print(db1:get_dotted_ip(ipaddr)) 28 | 29 | db1:close() 30 | 31 | 32 | -------------------------------------------------------------------------------- /lua/libs/ipprefixdb/test.lua: -------------------------------------------------------------------------------- 1 | local IPPrefixDB=require'ipprefixdb' 2 | 3 | 4 | -- create and open 5 | local db1 = IPPrefixDB.new() 6 | db1:open("/tmp/ipdb.level") 7 | 8 | 9 | -- store 10 | 11 | db1:put_cidr("192.168.0.0/16", "Entire 192 network private") 12 | db1:put_cidr("192.168.4.0/24", "Video servers 192.168.4.0/24") 13 | db1:put_dotted_ip("192.168.4.18","192.168.4.22", "4.18 to 4.22 special ") 14 | 15 | db1:put_ipv6_cidr("2001:550:c00::/38",6252001) 16 | db1:put_ipv6_cidr("2001:470:b:c42::/63",1814991) 17 | db1:put_ipv6_cidr("2001:978:2:39::5:800/117",2921044) 18 | db1:put_ipv6_cidr("2001:2030:0:1c:6129:fc61:4a70:fc91/128",6252001) 19 | db1:put_ipv6_cidr("2001:2030:0:1d:b986::/79",6255148) 20 | 21 | 22 | 23 | db1:dump() 24 | 25 | print ("\n192.168.4.18=") 26 | print (db1:get_dotted_ip("192.168.4.18") ) 27 | 28 | -- Test there are 3 nested ranges 29 | print ("\n192.168.4.22=") 30 | print (db1:get_dotted_ip("192.168.4.22") ) 31 | 32 | print ("\n192.168.4.21=") 33 | print (db1:get_dotted_ip("192.168.4.21") ) 34 | 35 | print ("\n192.168.4.15=") 36 | print (db1:get_dotted_ip("192.168.4.15") ) 37 | 38 | print ("\n192.168.4.23=") 39 | print (db1:get_dotted_ip("192.168.4.23") ) 40 | 41 | print ("\n192.168.4.18=") 42 | print (db1:get_dotted_ip("192.168.4.18") ) 43 | 44 | print ("\n192.168.4.17=") 45 | print (db1:get_dotted_ip("192.168.4.17") ) 46 | 47 | print ("\n192.168.2.11=") 48 | print (db1:get_dotted_ip("192.168.2.11") ) 49 | 50 | print ("\n192.168.5.11=") 51 | print (db1:get_dotted_ip("192.168.5.11") ) 52 | 53 | print ("\n45.68.15.1=") 54 | print (db1:get_dotted_ip("45.68.15.1") ) 55 | 56 | print ("\n241.168.200.199=") 57 | print (db1:get_dotted_ip("\n241.168.200.199=")) 58 | 59 | 60 | -- dump 61 | -- db1:dump() 62 | 63 | -- closing 64 | db1:close() 65 | 66 | 67 | -------------------------------------------------------------------------------- /lua/skeletons/README.md: -------------------------------------------------------------------------------- 1 | Skeleton selector 2 | =========== 3 | 4 | 5 | Trisul LUA API documentation at https://trisul.org/docs/lua 6 | 7 | 8 | What do I want to do ? Listen to .. |Use this skeleton 9 | ---|--- 10 | every packet at a given protocol layer | simple_counter.lua 11 | each TCP reassembled segment | reassembly_handler.lua 12 | each HTTP URI, header, TLS certificate, etc | reassembly_handler.lua 13 | read a custom PCAP file or Flow file as input to Trisul | input_filter.lua 14 | listen to alerts from custom sources and feed into Trisul pipeline | input_filter.lua 15 | a custom network protocol not supported by Trisul | protocol_handler.lua 16 | control PCAP storage on a per-flow basis | packet_storage.lua 17 | HTTP file extraction | filex_monitor.lua 18 | create a new counter group | new_counter_group.lua 19 | create a new alert group | new_alert_group.lua 20 | create a new resource group | new_resource_group.lua 21 | when a streaming window is opened, closed | engine_monitor.lua 22 | each new alert . Eg IDS alert, Flow Tracker, etc | alert_monitor.lua 23 | each new resource. Eg DNS, SSL Cert, HTTP URI, etc| resource_monitor.lua 24 | each new Full Text Document. Full DNS , SSL, HTTP Headers | fts_monitor.lua 25 | when a counter (metric) is updated, | cg_monitor.lua 26 | when a topper list is flushed to storage | cg_monitor.lua 27 | when a new key is first seen in a counter group| cg_monitor.lua 28 | a new flow is seen | sg_monitor.lua 29 | when flows are flushed to storage| sg_monitor.lua 30 | create your own flow tracker | flow_tracker.lua | 31 | 32 | Common facilities available to all backend scripts 33 | 34 | 1. `onmetronome` to get called every second (approx) 35 | 2. `flushfilter` to decide is a particular object gets flushed to database or not 36 | 3. `beginflush` and `endflush` to indicate the closing of a streaming Time Window 37 | 38 | 39 | -------------------------------------------------------------------------------- /lua/skeletons/input_filter.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- input_filter.lua skeleton 3 | -- 4 | -- TYPE: FRONTEND SCRIPT 5 | -- PURPOSE: Input filter to drive the Trisul pipeline 6 | -- DESCRIPTION: Custom input for packets, flows, or alerts 7 | -- 8 | -- 9 | -- 10 | -- 11 | TrisulPlugin = { 12 | 13 | 14 | -- the ID block, you can skip the fields marked 'optional ' 15 | -- 16 | id = { 17 | name = "CSV netflow input", 18 | description = "read flow records from CSV", -- optional 19 | author = "Unleash", -- optional 20 | version_major = 1, -- optional 21 | version_minor = 0, -- optional 22 | }, 23 | 24 | -- COMMON FUNCTIONS: onload, onunload, onmessage 25 | -- 26 | -- WHEN CALLED : your LUA script is loaded into Trisul 27 | onload = function() 28 | -- your code 29 | end, 30 | 31 | -- WHEN CALLED : your LUA script is unloaded / detached from Trisul 32 | onunload = function() 33 | -- your code 34 | end, 35 | 36 | -- any messages you want to handle for state management 37 | message_subscriptions = {}, 38 | 39 | -- WHEN CALLED: when another plugin sends you a message 40 | onmessage = function(msgid, msg) 41 | 42 | end, 43 | 44 | 45 | 46 | 47 | -- input_filter block 48 | -- 49 | inputfilter = { 50 | 51 | 52 | 53 | -- WHEN CALLED: when Trisul platform wants a new packet or flow 54 | -- step block : to handle packets and flows 55 | -- read the next line from the file and do engine:updateXXX(..) to add metrics 56 | step = function(packet, engine) 57 | -- your lua code here 58 | end, 59 | 60 | 61 | 62 | -- WHEN CALLED: when Trisul platform wants a new packet or flows 63 | -- step_alert block : to feed alerts into the pipeline 64 | -- need to return a table { } with alert information; see the docs 65 | step_alert = function() 66 | -- your lua code here 67 | -- return a table { } 68 | 69 | end, 70 | 71 | 72 | 73 | }, 74 | } 75 | -------------------------------------------------------------------------------- /lua/skeletons/message_monitor.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- message_monitor.lua skeleton 3 | -- 4 | -- TYPE: FRONTEND SCRIPT 5 | -- PURPOSE: message monitor 6 | -- DESCRIPTION: MMON listens to TMS frontend messages 7 | -- 8 | TrisulPlugin = { 9 | 10 | 11 | -- the ID block, you can skip the fields marked 'optional ' 12 | -- 13 | id = { 14 | name = "MessageMonitor", 15 | description = "mmonitor listens to messages ", 16 | }, 17 | 18 | 19 | 20 | -- common functions onload, onunload, onmessage().. 21 | 22 | -- WHEN CALLED : your LUA script is loaded into Trisul 23 | onload = function() 24 | -- your code 25 | end, 26 | 27 | -- WHEN CALLED : your LUA script is unloaded / detached from Trisul 28 | onunload = function() 29 | -- your code 30 | end, 31 | 32 | -- any messages you want to handle for state management 33 | message_subscriptions = {}, 34 | 35 | -- WHEN CALLED: when another plugin sends you a message 36 | onmessage = function(msgid, msg) 37 | -- your code 38 | end, 39 | 40 | 41 | -- messagemonitor block 42 | -- 43 | messagemonitor = { 44 | 45 | 46 | -- when a new flow metric is seen 47 | -- used with TCP reassembly 48 | onflowmetric = function(engine,flowid,meter,value) 49 | 50 | end, 51 | 52 | -- when a new NetFlow record is seen 53 | -- use this to tap into NetFlow records and add counters within context of those records 54 | onnewflowrecord = function(engine, flowid, bytes_az, bytes_za, packets_az, packets_za) 55 | 56 | end, 57 | 58 | 59 | }, 60 | } 61 | -------------------------------------------------------------------------------- /lua/skeletons/new_alert_group.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- new_alert_group.lua skeleton 3 | -- 4 | -- TYPE: FRONTEND SCRIPT 5 | -- PURPOSE: Create a new alert group 6 | -- DESCRIPTION: You can create your own alert groups for your specific case 7 | -- 8 | -- 9 | -- 10 | -- 11 | TrisulPlugin = { 12 | 13 | 14 | -- the ID block, you can skip the fields marked 'optional ' 15 | -- 16 | id = { 17 | name = "My IOC hits", 18 | description = "Alerts when my private IOCs match ", -- optional 19 | author = "Unleash", -- optional 20 | version_major = 1, -- optional 21 | version_minor = 0, -- optional 22 | }, 23 | 24 | -- COMMON FUNCTIONS: onload, onunload, onmessage 25 | -- 26 | -- WHEN CALLED : your LUA script is loaded into Trisul 27 | onload = function() 28 | -- your code 29 | end, 30 | 31 | -- WHEN CALLED : your LUA script is unloaded / detached from Trisul 32 | onunload = function() 33 | -- your code 34 | end, 35 | 36 | -- any messages you want to handle for state management 37 | message_subscriptions = {}, 38 | 39 | -- WHEN CALLED: when another plugin sends you a message 40 | onmessage = function(msgid, msg) 41 | 42 | end, 43 | 44 | 45 | -- alertgroup block 46 | -- 47 | alertgroup = { 48 | 49 | -- table control 50 | -- WHEN CALLED: specify details of your new alert group 51 | -- you can use 'trisulctl_probe testbench guid' to get a new GUID 52 | control = { 53 | guid = "{a973e25d-4434-4f0a-9656-9d2c0247eaf8}", 54 | name = "My IOC Hit", 55 | description = "When my IOC hits ", 56 | }, 57 | 58 | }, 59 | } 60 | -------------------------------------------------------------------------------- /lua/skeletons/new_resource_group.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- new_resource_group.lua skeleton 3 | -- 4 | -- TYPE: FRONTEND SCRIPT 5 | -- PURPOSE: Create a new resource group 6 | -- DESCRIPTION: You can create your own resource groups for your specific case 7 | -- 8 | -- 9 | -- 10 | -- 11 | TrisulPlugin = { 12 | 13 | 14 | -- the ID block, you can skip the fields marked 'optional ' 15 | -- 16 | id = { 17 | name = "SHA256 Hashes", 18 | description = "logs SHA hash Resources ", -- optional 19 | author = "Unleash", -- optional 20 | version_major = 1, -- optional 21 | version_minor = 0, -- optional 22 | }, 23 | 24 | -- COMMON FUNCTIONS: onload, onunload, onmessage 25 | -- 26 | -- WHEN CALLED : your LUA script is loaded into Trisul 27 | onload = function() 28 | -- your code 29 | end, 30 | 31 | -- WHEN CALLED : your LUA script is unloaded / detached from Trisul 32 | onunload = function() 33 | -- your code 34 | end, 35 | 36 | -- any messages you want to handle for state management 37 | message_subscriptions = {}, 38 | 39 | -- WHEN CALLED: when another plugin sends you a message 40 | onmessage = function(msgid, msg) 41 | 42 | end, 43 | 44 | 45 | -- resourcegroup block 46 | -- 47 | resourcegroup = { 48 | 49 | -- table control 50 | -- WHEN CALLED: specify details of your new resource group 51 | -- you can use 'trisulctl_probe testbench guid' to get a new GUID 52 | control = { 53 | guid = "{a973e25d-4434-4f0a-9656-9d2c0247eaf8}", 54 | name = "SHA256 file hashes", 55 | description = "File hash resources ", 56 | }, 57 | 58 | }, 59 | } 60 | -------------------------------------------------------------------------------- /lua/skeletons/protocol_handler.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- protocol_handler.lua skeleton 3 | -- 4 | -- TYPE: FRONTEND SCRIPT 5 | -- PURPOSE: protocol handler, 6 | -- DESCRIPTION: dissects a protocol at a given layer, return number of bytes 'eaten' and then 7 | -- the next layer protocol 8 | -- 9 | TrisulPlugin = { 10 | 11 | 12 | -- the ID block, you can skip the fields marked 'optional ' 13 | -- 14 | id = { 15 | name = "FlowDirect", 16 | description = "Flow director based on router ip", -- optional 17 | }, 18 | 19 | 20 | -- protocol_handler block 21 | -- 22 | protocol_handler = { 23 | 24 | -- new protocol for FLOWDIR 25 | control = { 26 | guid = "{0CED6B98-0D90-475C-D2D7-06A8E9E64B7C}", -- new protocol GUID, use tp testbench guid to create 27 | name = "FLOWDIRECT", -- new protocol name 28 | 29 | -- these two are optional 30 | host_protocol_guid = '{14D7AB53-CC51-47e9-8814-9C06AAE60189}', -- GUID for UDP 31 | host_protocol_ports = { 67,68 } -- we want UDP ports 67,68 32 | }, 33 | 34 | 35 | -- WHEN CALLED: when lower layer is constructed and 36 | -- return ( nEaten, nextProtID) 37 | parselayer = function(layer) 38 | 39 | -- return nEaten, nextProtocolGUID 40 | -- if you have no idea about next protocol, return nothing 41 | 42 | end, 43 | 44 | 45 | }, 46 | } 47 | -------------------------------------------------------------------------------- /lua/skeletons/simple_counter.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- simple_counter.lua skeleton 3 | -- 4 | -- TYPE: FRONTEND SCRIPT 5 | -- PURPOSE: handle packets and update metrics 6 | -- DESCRIPTION: Use this to monitor raw network traffic and update metrics 7 | -- 8 | -- 9 | -- 10 | -- 11 | TrisulPlugin = { 12 | 13 | 14 | -- the ID block, you can skip the fields marked 'optional ' 15 | -- 16 | id = { 17 | name = "DNS packet monitor", 18 | description = "Listen to DNS packets, parse, and updates ", -- optional 19 | author = "Unleash", -- optional 20 | version_major = 1, -- optional 21 | version_minor = 0, -- optional 22 | }, 23 | 24 | -- COMMON FUNCTIONS: onload, onunload, onmessage 25 | -- 26 | -- WHEN CALLED : your LUA script is loaded into Trisul 27 | onload = function() 28 | -- your code 29 | end, 30 | 31 | -- WHEN CALLED : your LUA script is unloaded / detached from Trisul 32 | onunload = function() 33 | -- your code 34 | end, 35 | 36 | -- any messages you want to handle for state management 37 | message_subscriptions = {}, 38 | 39 | -- WHEN CALLED: when another plugin sends you a message 40 | onmessage = function(msgid, msg) 41 | 42 | end, 43 | 44 | 45 | 46 | 47 | -- simple_counter block 48 | -- 49 | simplecounter = { 50 | 51 | -- Required field : which protocol (layer) do you wish to attach to 52 | -- as admin > Profile > Trisul Protocols for a list 53 | protocol_guid = "{0A2C724B-5B9F-4ba6-9C97-B05080558574}", 54 | 55 | 56 | -- WHEN CALLED: when the Trisul platform detects a packet at the protocol_guid layer 57 | -- above. In this case, every DNS packet 58 | -- 59 | onpacket = function(engine,layer) 60 | -- your code here 61 | -- typically access the raw bytes and use engine:methods(..) to add metrics 62 | end, 63 | 64 | 65 | }, 66 | } 67 | -------------------------------------------------------------------------------- /lua/techniques/mrumap/test.lua: -------------------------------------------------------------------------------- 1 | local mrumap=require'mrumap' 2 | 3 | local tst = mrumap.new() 4 | print(tst:get("hi")) 5 | 6 | tst:put('oldest',1) 7 | tst:put('secondoldest',1) 8 | tst:put('oldestbutaccessedlater',1) 9 | 10 | tst:put('deletedlater',1) 11 | -- 12 | local function randstr() 13 | local chars={} 14 | for i=1,10 do 15 | table.insert(chars,string.char(math.random(97, 122))) 16 | end 17 | return table.concat(chars,"") 18 | end 19 | 20 | -- insert about 21 | for i = 1, 25 do 22 | tst:put(randstr(),1) 23 | end 24 | 25 | tst:get("oldestbutaccessedlater") 26 | 27 | -- print in MRU order 28 | for k in tst:mru_iter() do 29 | print(k) 30 | end 31 | print("size="..tst:size()) 32 | 33 | -- pop the LRU item 34 | tst:pop_back() 35 | 36 | for k in tst:mru_iter() do 37 | print(k) 38 | end 39 | 40 | print("size="..tst:size()) 41 | 42 | -- delete 43 | tst:delete("deletedlater") 44 | 45 | print("size="..tst:size()) 46 | 47 | -- roll back all to zero 48 | while tst:pop_lru() do 49 | --print("size="..tst:size()) 50 | end 51 | 52 | print("size="..tst:size()) 53 | 54 | print("capacity="..tst:capacity()) 55 | 56 | -------------------------------------------------------------------------------- /lua/techniques/re2http.lua: -------------------------------------------------------------------------------- 1 | -- re2http.lua 2 | -- 3 | -- Use RE2 regex to capture select HTTP headers and log them 4 | -- 5 | -- Demonstrates 6 | -- 1. Use re2:partial_match_cN(..) , match with capture 7 | -- 8 | TrisulPlugin = { 9 | 10 | id = { 11 | name = "HTTP Parse Sample", 12 | description = "Use REGEX re2 to parse ", 13 | author = "Unleash", 14 | version_major = 1, 15 | version_minor = 0, 16 | }, 17 | 18 | 19 | 20 | onload = function () 21 | 22 | P = TrisulPlugin 23 | 24 | -- 25 | -- we log the date, user-agent, host, referrer 26 | -- 27 | P.regexes = { 28 | T.re2("User-Agent\\s*:\\s*(.*)\r\n"), 29 | T.re2("Host\\s*:\\s*(.*)\r\n"), 30 | T.re2("Referer\\s*:\\s*(.*)\r\n") 31 | } 32 | math.randomseed(os.time()) 33 | 34 | 35 | P.outfile = io.open("/tmp/httpheaders-"..math.random(1000,2000)..".log","w") 36 | 37 | end, 38 | 39 | 40 | onunload = function() 41 | 42 | P.outfile.close() 43 | 44 | end, 45 | 46 | 47 | flowmonitor = { 48 | 49 | onflowattribute = function(engine,flow,timestamp, 50 | attribute_name, attribute_value) 51 | 52 | if attribute_name == "HTTP-Header" then 53 | 54 | local val = attribute_value:tostring() 55 | 56 | -- 57 | -- write to output 58 | -- 59 | P.outfile:write(os.date("%c",timestamp)) 60 | P.outfile:write("\t") 61 | for i,v in ipairs( P.regexes) do 62 | local status, match = v:partial_match_c1(val) 63 | if status then 64 | P.outfile:write(match) 65 | P.outfile:write("\t") 66 | end 67 | end 68 | P.outfile:write("\n") 69 | 70 | end 71 | 72 | end, 73 | 74 | }, 75 | 76 | } 77 | 78 | -------------------------------------------------------------------------------- /lua/techniques/tagflow.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- tagclen.lua 3 | -- 4 | -- Tags user agent with old java verson 5 | -- 6 | -- 7 | TrisulPlugin = { 8 | 9 | id = { 10 | name = "Tag flows ", 11 | description = "Tagging user agent ", 12 | author = "Unleash", 13 | version_major = 1, 14 | version_minor = 0, 15 | }, 16 | 17 | 18 | flowmonitor = { 19 | 20 | onflowattribute = function(engine,flow, 21 | timestamp, 22 | aname, avalue) 23 | 24 | if aname == "User-Agent" then 25 | if avalue:tostring():find("Mozilla/[123]") then 26 | engine:tag_flow( flow:id(), "oldmoz") 27 | end 28 | end 29 | 30 | end, 31 | }, 32 | } 33 | 34 | -------------------------------------------------------------------------------- /lua/tutorial/README.md: -------------------------------------------------------------------------------- 1 | LUA Scripting Tutorials 2 | ----------------------- 3 | 4 | These scripts are to be used along with the "Getting Started" LUA Tutorials on the 5 | documentaton page at http://trisul.org/lua/ page 6 | 7 | 8 | hello.lua Hello World with Trisul and LUA 9 | ------------------------------------------ 10 | 11 | (Part of Tutorial 1) 12 | 13 | This script should be the very first LUA script that you write. It demonstrates the following 14 | 15 | - Basic skeleton structure of a LUA script 16 | - onload and onunload functions 17 | - Where to place the LUA scripts 18 | - Running a PCAP file and printing to console and log file 19 | 20 | This tiny script is a great starting point. 21 | 22 | 23 | pktlen.lua Packet Length Metering 24 | ---------------------------------- 25 | 26 | (Part of Tutorial 2) 27 | 28 | 29 | A real useful metrics script. This script looks at streaming network traffic and 30 | computes a packet length distribution. The traffic is classified into 31 | 32 | Packet lengths 33 | 1500 34 | 1000-1500 35 | 500-1000 36 | 200-500 37 | 100-200 38 | 0-100 39 | 40 | 41 | THis demonstrates 42 | 43 | - Creating a new counter group called "Packet Length" 44 | - Using Keys - the string "1000-1500" is a key used to identify metrics for this bucket 45 | - Using LUA Objects "Layer" "Packet" and "Buffer" 46 | - Viewing the results in web trisul 47 | 48 | 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /lua/tutorial/tutorial1/README.md: -------------------------------------------------------------------------------- 1 | LUA Scripting : Tutorial 1 2 | ========================== 3 | 4 | This directory contains content for [Trisul LUA Scripting Tutorial 1 "Hello World")[https://trisul.org/docs/lua/tutorial1.html) 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /lua/tutorial/tutorial1/hello.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- Basic working script, just prints hello 3 | -- 4 | TrisulPlugin = { 5 | 6 | id = { 7 | name = "Hello World", 8 | description = "Nothing much ", 9 | author = "Unleash", 10 | version_major = 1, 11 | version_minor = 0, 12 | }, 13 | 14 | onload = function() 15 | print("Onload - hello world "); 16 | T.host:log(T.K.loglevel.INFO, "Hello world now in log file "); 17 | end, 18 | 19 | 20 | onunload = function () 21 | print("Onunload - bye "); 22 | end, 23 | 24 | } 25 | 26 | -------------------------------------------------------------------------------- /lua/tutorial/tutorial1/tutorial.pcap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trisulnsm/trisul-scripts/bef85dea72016ef58a98246649c4cf7c4f3faca5/lua/tutorial/tutorial1/tutorial.pcap -------------------------------------------------------------------------------- /lua/tutorial/tutorial2/16minutes.pcap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trisulnsm/trisul-scripts/bef85dea72016ef58a98246649c4cf7c4f3faca5/lua/tutorial/tutorial2/16minutes.pcap -------------------------------------------------------------------------------- /lua/tutorial/tutorial2/README.md: -------------------------------------------------------------------------------- 1 | LUA Scripting : Tutorial 2 2 | ========================== 3 | 4 | This directory contains content for [Trisul LUA Scripting Tutorial 2 "Packet Length Counter")[https://trisul.org/docs/lua/tutorial2.html) 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /lua/tutorial/tutorial3/README.md: -------------------------------------------------------------------------------- 1 | Tutorial 3 : TCP Based Protocol Analysis. A Basic DNP3 analyzer 2 | ======================================= 3 | 4 | 5 | In this tutorial we show how you can write a basic TCP Stream Based protocol analyzer. 6 | We show 7 | 8 | 1. DNP3 basic analysing using the SweepBuf module 9 | 2. How to work with packet buffer, we show how DNP3 checksum can be verified 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /lua/tutorial/tutorial4/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | Prep : Create a new context in which you can play 4 | ==================================================== 5 | 6 | 7 | ```` 8 | 9 | trisulctl_probe create context test1 10 | 11 | 12 | ```` 13 | 14 | Now add in the helper macros 15 | 16 | 17 | 18 | Step 1: How to hook into Trisul Resources 19 | ================= 20 | 21 | ```` 22 | 23 | DOCKER:localhost:root$ trisulctl_probe testbench run /trisulroot/upload_misc.tcpd 24 | .. 25 | Replacing image with 26 | /usr/local/bin/trisul -nodemon /usr/local/etc/trisul-probe/domain0/probe0/context_debug0/trisulProbeConfig.xml -mode offline -in /trisulroot/upload_misc.tcpd 27 | 28 | GET toolbar.google.com /buttons/feeds/topbuttons/?hl=en&sd=com HTTP/1.1 29 | GET www.google.com /tools/toolbar/service/version4?&version=4.0.1601.4978&os=big&hl=en&tbbrand=GGLD&sd=com&osver=5.1&ossp=2.0&browser=6.0.2900.2180&rlz=&needc=3 HTTP/1.1 30 | GET www.google.com /url?sa=p&pref=tb&pval=2&q=http%3A%2F%2Fwww.google.com%2Ftools%2Ftoolbar%2Fservice%2Fnoupdate%3F HTTP/1.1 31 | GET www.google.com /tools/toolbar/service/noupdate? HTTP/1.1 32 | GET www.dnswatch.info / HTTP/1.1 33 | GET www.dnswatch.info /js/lookup.js HTTP/1.1 34 | GET pagead2.googlesyndication.com /pagead/show_ads.js HTTP/1.1 35 | GET www.dnswatch.info /images/dnswatch-logo.gif HTTP/1.1 36 | 37 | 38 | ```` 39 | 40 | 41 | 42 | Step 2: Generate an alert and push it into Trisul 43 | 44 | 45 | 46 | --- 47 | 48 | 49 | 50 | Step 3 : How to read in an CSV Intel file 51 | 52 | Download helper csv.lua 53 | Load URL -> full line map in memory table 54 | 55 | 56 | 57 | 58 | 59 | -------------------------------------------------------------------------------- /lua/tutorial/tutorial4/step1.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- resource_monitor.lua skeleton 3 | -- 4 | -- step1 : Just prints URL Resources 5 | -- 6 | -- TYPE: BACKEND SCRIPT 7 | -- PURPOSE: handle Resources extracted by Trisul 8 | -- DESCRIPTION: Trisul platform extracts resources (files,hashes,dns,ssl certs, etc) 9 | -- they stream through the backend pipeline. Here is where you handle them 10 | -- 11 | TrisulPlugin = { 12 | 13 | -- id block 14 | -- 15 | id = { 16 | name = "STEP1", 17 | description = "Print HTTP URL Resources", 18 | }, 19 | 20 | 21 | -- resource_monitor block 22 | -- 23 | resource_monitor = { 24 | 25 | -- which resource group do you want to monitor 26 | -- the following GUID represents HTTP URL resources, 27 | -- to see list of Resource GUIDs 28 | -- https://www.trisul.org/docs/ref/guid.html#resource_groups 29 | -- or Login as admin : view resources 30 | -- 31 | resource_guid = '{4EF9DEB9-4332-4867-A667-6A30C5900E9E}', 32 | 33 | -- WHEN CALLED : a new resource is seen (immediately) 34 | -- the resource is a LUA Object 35 | -- see https://www.trisul.org/docs/lua/resource_monitor.html#resource 36 | onnewresource = function(engine, resource ) 37 | 38 | -- print(resource:uri()) 39 | 40 | end, 41 | 42 | } 43 | 44 | 45 | } 46 | -------------------------------------------------------------------------------- /trp/README.md: -------------------------------------------------------------------------------- 1 | Trisul Scripts 2 | ================ 3 | 4 | 5 | Repository of small but powerful network and security monitoring scripts for use with Trisul Network Analytics (trisul.org) 6 | 7 | 8 | 9 | How to run these samples ? 10 | ------------------------- 11 | 12 | You need to have : 13 | 14 | - A running instance of Trisul 15 | - Git clone this repo, if you havent already `git clone https://github.com/trisulnsm/trisul-scripts.git ` 16 | - Go to the `trisul-scripts/helloworld` directory 17 | - Run the hello.rb script ( ruby hello.rb 192.168.1.222 ) replace that IP with your Trisul's IP. The password for the private key file is `client` 18 | 19 | ## Getting started 20 | Complete instructions are in the "Step by step guide" http://trisul.org/docs/trp/trpgemsteps.html 21 | 22 | 23 | The NSM strategy followed by Trisul is 24 | * Big picture and zoom out with Trisul 25 | * Content, protocol and zoom in with Unsniff 26 | 27 | ### 1. Network Security Monitoring using Trisul 28 | 29 | Search traffic, flows, alerts, resource, and packets. Analyze patterns 30 | isolate flows and pull out raw packets in PCAP format. This is a fast 31 | analysis designed to narrow down the info required for further processing. 32 | 33 | You will be using Ruby along with the Trisul Remote Protocol API (http://trisul.org/docs/trp/ ) 34 | 35 | ### 2. Content analysis with Unsniff 36 | 37 | Once you have the packets, you can use Unsniff Network Analyzers scripting 38 | abilities to extract content, look into protocol fields, search and filter for 39 | specific things. This is a deep forensics stage designed to carve out 40 | files and other content. 41 | 42 | 43 | Ruby with Unsniff Network Analyzer API ( http://www.unleashnetworks.com/unsniffwiki/docs/doku.php?id=start ) 44 | 45 | *Note* Scripts using the Unsniff API run only on Windows systems. 46 | 47 | -------------------------------------------------------------------------------- /trp/apt1-detect/dnss.txt: -------------------------------------------------------------------------------- 1 | kona.kontera.com 2 | 3 | -------------------------------------------------------------------------------- /trp/apt1-detect/ip-ranges.txt: -------------------------------------------------------------------------------- 1 | 192.168.1.33 2 | 209.216.1.1-209.216.1.128 3 | 57.7.13.0-57.7.16.255 4 | -------------------------------------------------------------------------------- /trp/apt1-detect/md5s.txt: -------------------------------------------------------------------------------- 1 | 18edd24a1cd4496578c50fe2efd1a9a8 2 | 0d539981335c18acacd9a0c2c7ca8f0d 3 | ed9472ba7d8ca2920e8e93aab38e5aa4 4 | d5753af0d384857ca34bf8b54c5eb417 5 | 627c405e3d4969e57f48dd09289aa29d 6 | -------------------------------------------------------------------------------- /trp/apt1-detect/search_fqdn.rb: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/ruby 2 | # Trisul Remote Protocol TRP Demo script 3 | # 4 | # Runs a set of FQDNs past all names known by Trisul 5 | # 6 | # Example 7 | # ruby search_fqdn.rb ipc:///usr/local/var/lib/trisul/CONTEXT0/run/trp_0 dns_list.txt 8 | # 9 | require 'trisulrp' 10 | 11 | USAGE = "Usage : search_fqdn.rb ZMQ_ENDPOINT NAMES-FILE\n"\ 12 | "Examples: 1) ruby search_fqdn.rb tcp://localhost:5555 dns_list.txt \n "\ 13 | " 2) ruby search_fqdn.rb ipc:///usr/local/var/lib/trisul/CONTEXT0/run/trp_0 dns_list.txt" 14 | 15 | 16 | # usage 17 | unless ARGV.size==2 18 | abort USAGE 19 | end 20 | 21 | 22 | # zmq connection point 23 | zmq_endpt = ARGV[0] 24 | # get all time .. 25 | tmarr = TrisulRP::Protocol.get_available_time(zmq_endpt) 26 | 27 | # send resource group request 28 | # we want resource group RG_DNS identified by GUID {D1E2..} see docs 29 | req = TrisulRP::Protocol.mk_request( 30 | TRP::Message::Command::RESOURCE_GROUP_REQUEST, 31 | :resource_group => RG_DNS, 32 | :time_interval => mk_time_interval(tmarr), 33 | :uri_list => File.read(ARGV[1]) 34 | .split("\n") 35 | .collect {|a| a.strip} ) 36 | 37 | # print resource ids, 38 | get_response_zmq(zmq_endpt,req) do |resp| 39 | puts "Found #{resp.resources.size} matches" 40 | resp.resources.each do | res | 41 | puts "Resource #{res.slice_id}:#{res.resource_id} " 42 | end 43 | end 44 | 45 | -------------------------------------------------------------------------------- /trp/apt1-detect/search_keyspace.rb: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/ruby 2 | # Trisul Remote Protocol TRP Demo script 3 | # 4 | # Search for matches in key space (IP Ranges in this case) 5 | # 6 | # Example 7 | # ruby search_keyspace.rb ipc:///usr/local/var/lib/trisul/CONTEXT0/run/trp_0 keyspaces.txt 8 | # 9 | # keyspaces.txt is in the format 10 | # 19.88.100.0-19.88.103.0 11 | # or just (for exact match) 12 | # 19.88.100.121 13 | # 14 | require 'trisulrp' 15 | 16 | USAGE = "Usage : search_keyspace.rb ZMQ_ENDPOINT NAMES-FILE\n"\ 17 | "Examples: 1) ruby search_keyspace.rb tcp://localhost:5555 ip-ranges.txt \n "\ 18 | " 2) ruby search_keyspace.rb ipc:///usr/local/var/lib/trisul/CONTEXT0/run/trp_0 ip-ranges.txt" 19 | 20 | 21 | # usage 22 | unless ARGV.size==2 23 | abort USAGE 24 | end 25 | 26 | # zmq end point 27 | zmq_endpt = ARGV[0] 28 | 29 | # get all time..then for this demo script crop to latest 1 day, 30 | # in production loop for each day .. 31 | tmarr = TrisulRP::Protocol.get_available_time(zmq_endpt) 32 | tmarr[0] = tmarr[1]-86400 33 | # read in keyspaces 34 | spaces=File.readlines(ARGV[1]).collect do |l| 35 | p = l.chomp.split("-") 36 | if p.size==2 37 | TRP::KeySpaceRequest::KeySpace.new( 38 | :from => make_key(p[0]), :to => make_key(p[1])) 39 | elsif p.size==1 40 | TRP::KeySpaceRequest::KeySpace.new( 41 | :from => make_key(p[0]), :to => make_key(p[0])) 42 | end 43 | end 44 | 45 | 46 | # send keyspace request 47 | # we want keys in CG_HOSTS 48 | req = TrisulRP::Protocol.mk_request(TRP::Message::Command::KEYSPACE_REQUEST, 49 | :counter_group => CG_HOST, 50 | :time_interval => mk_time_interval(tmarr), 51 | :spaces => spaces) 52 | 53 | 54 | # print hits 55 | get_response_zmq(zmq_endpt,req) do |resp| 56 | puts "Found #{resp.hits.size} matches" 57 | resp.hits.each do | res | 58 | puts "Hit Key #{res} " 59 | end 60 | end 61 | 62 | -------------------------------------------------------------------------------- /trp/apt1-detect/search_md5.rb: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/ruby 2 | # Trisul Remote Protocol TRP Demo script 3 | # 4 | # Search *ALL* HTTP objects for matching MD5 5 | # in last 24 hours. Reconstructs TCP/Decompresses/Dechunks 6 | # while hashing. 7 | # 8 | # Example 9 | # ruby search_md5.rb ipc:///usr/local/var/lib/trisul/CONTEXT0/run/trp_0 md5s.txt 10 | # 11 | # Will print out matching flows, you can extract the packets from each of 12 | # those flows using FilteredDatagramRequest(..) 13 | # 14 | require 'trisulrp' 15 | 16 | USAGE = "Usage : search_md5.rb ZMQ_ENDPOINT NAMES-FILE\n"\ 17 | "Examples: 1) ruby search_md5.rb tcp://localhost:5555 md5s.txt \n "\ 18 | " 2) ruby search_md5.rb ipc:///usr/local/var/lib/trisul/CONTEXT0/run/trp_0 md5s.txt" 19 | 20 | 21 | # usage 22 | unless ARGV.size==2 23 | abort USAGE 24 | end 25 | 26 | 27 | # zmq end point 28 | zmq_endpt = ARGV[0] 29 | 30 | # get 24 hours latest time window 31 | tmarr = TrisulRP::Protocol.get_available_time(zmq_endpt) 32 | tmarr[0] = tmarr[1] - 24*3600 33 | 34 | 35 | # send grep request (GrepRequest) 36 | # read md5s from file given as input 37 | req = TrisulRP::Protocol.mk_request(TRP::Message::Command::GREP_REQUEST, 38 | :time_interval => mk_time_interval(tmarr), 39 | :md5list => File.read(ARGV[1]) 40 | .split("\n") 41 | .collect {|a| a.strip} ) 42 | 43 | # print matching flows if any 44 | get_response_zmq(zmq_endpt,req) do |resp| 45 | puts "Found #{resp.sessions.size} matches" 46 | resp.sessions.each_with_index do | sess, idx | 47 | puts "Flow #{sess.slice_id}:#{sess.session_id} #{resp.hints[idx]} " 48 | end 49 | end 50 | 51 | -------------------------------------------------------------------------------- /trp/apt1-detect/search_text.rb: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/ruby 2 | # Trisul Remote Protocol TRP Demo script 3 | # 4 | # Search all flows (incl HTTP) for a text pattern 5 | # 6 | # Example 7 | # ruby search_text.rb ipc:///usr/local/var/lib/trisul/CONTEXT0/run/trp_0 \"hello this is a test \" 8 | 9 | # 10 | # 11 | require 'trisulrp' 12 | 13 | USAGE = "Usage : search_text.rb ZMQ_ENDPOINT \"hello this is a test\"\n"\ 14 | "Examples: 1) ruby search_text.rb tcp://localhost:5555 \"hello this is a test\"\n "\ 15 | " 2) ruby search_text.rb ipc:///usr/local/var/lib/trisul/CONTEXT0/run/trp_0 \"hello this is a test\"" 16 | 17 | # usage 18 | unless ARGV.size==2 19 | abort USAGE 20 | end 21 | 22 | # zmq end point 23 | zmq_endpt = ARGV[0] 24 | 25 | # get 24 hours latest time window 26 | tmarr = TrisulRP::Protocol.get_available_time(zmq_endpt) 27 | tmarr[0] = tmarr[1] - 24*3600 28 | 29 | 30 | 31 | # send grep request (GrepRequest) 32 | # read md5s from file given as input 33 | req = TrisulRP::Protocol.mk_request(TRP::Message::Command::GREP_REQUEST, 34 | :time_interval => mk_time_interval(tmarr), 35 | :pattern => ARGV[2] ) 36 | 37 | # print matching flows if any 38 | get_response_zmq(zmq_endpt,req) do |resp| 39 | puts "Found #{resp.sessions.size} matches" 40 | resp.sessions.each_with_index do | sess, idx | 41 | puts "Flow #{sess.slice_id}:#{sess.session_id} #{resp.hints[idx]} " 42 | end 43 | end 44 | 45 | -------------------------------------------------------------------------------- /trp/cert-extract/README.md: -------------------------------------------------------------------------------- 1 | Print certificate chain 2 | ======================= 3 | 4 | 5 | Print the SSL certificate chain of all HTTPS connections from a particular host. 6 | 7 | Sample run 8 | 9 | ```` 10 | C:\Users\Vivek\Documents\devbo\us\certxtrp>ruby csx.rb demo2.trisul.org 12001 192.168.1.105 https 11 | Enter PEM pass phrase: 12 | 13 | Certificate chain for 65.55.184.155 to 192.168.1.105 14 | www.update.microsoft.com (Microsoft) 15 | Microsoft Secure Server Authority () 16 | Microsoft Secure Server Authority () 17 | Microsoft Internet Authority () 18 | Microsoft Internet Authority () 19 | GTE CyberTrust Global Root (GTE Corporation) 20 | 21 | Certificate chain for 65.55.184.27 to 192.168.1.105 22 | www.update.microsoft.com (Microsoft) 23 | Microsoft Secure Server Authority () 24 | Microsoft Secure Server Authority () 25 | Microsoft Internet Authority () 26 | Microsoft Internet Authority () 27 | GTE CyberTrust Global Root (GTE Corporation) 28 | 29 | Certificate chain for 198.232.168.144 to 192.168.1.105 30 | registration2.services.openoffice.org (Sun Microsystems, Inc) 31 | Sun Microsystems Inc SSL CA (Sun Microsystems Inc) 32 | Sun Microsystems Inc SSL CA (Sun Microsystems Inc) 33 | (VeriSign, Inc.) 34 | (VeriSign, Inc.) 35 | (VeriSign, Inc.) 36 | 37 | ```` 38 | -------------------------------------------------------------------------------- /trp/cginfo/README.md: -------------------------------------------------------------------------------- 1 | Print counter group info 2 | ======================== 3 | 4 | 5 | Trisul metering infrastructure is based on counter groups. 6 | Each counter group contains a number of "meters" 7 | Each counter group monitors a number of "keys" each of them 8 | having the same set of "meters". 9 | 10 | The COUNTER_GROUP_INFO TRP request can be used to get details of 11 | any counter group via scripting. 12 | 13 | 14 | There are two scripts here. 15 | 16 | 1. cginfo.rb : Print details of a SINGLE counter group 17 | 2. cginfoall.rb : Details of ALL counter groups on Trisul 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /trp/cginfo/cginfo.rb: -------------------------------------------------------------------------------- 1 | # Trisul Remote Protocol TRP Demo script 2 | # 3 | # Counter Group Info 4 | # 5 | # Prints information about all supported couner groups on a trisul instance 6 | # 7 | require 'trisulrp' 8 | 9 | 10 | USAGE = "Usage: cginfo.rb ZMQ_ENDPOINT CGGUID\n" \ 11 | "Example: 1) ruby cginfo.rb ipc:///usr/local/var/lib/trisul-hub/domain0/hub0/context0/run/trp_0 {C51B48D4-7876-479E-B0D9-BD9EFF03CE2E}\n"\ 12 | " 2) ruby cginfoall.rb tcp://localhost:5555 {C51B48D4-7876-479E-B0D9-BD9EFF03CE2E}" 13 | 14 | # usage 15 | unless ARGV.size==2 16 | abort USAGE 17 | end 18 | 19 | zmq_endpt = ARGV[0] 20 | target_guid = ARGV[1] 21 | req =TrisulRP::Protocol.mk_request(TRP::Message::Command::COUNTER_GROUP_INFO_REQUEST, 22 | :counter_group => target_guid ) 23 | 24 | # print a single counter group info 25 | get_response_zmq(zmq_endpt,req) do |resp| 26 | resp.group_details.each do |group_detail| 27 | p "Start Time = #{Time.at(group_detail.time_interval.from.tv_sec)}" 28 | p "End time = #{Time.at(group_detail.time_interval.to.tv_sec)}" 29 | p "Bucket Size = #{group_detail.bucket_size}" 30 | p "Name = " + group_detail.name 31 | p "GUID = " + group_detail.guid 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /trp/cginfo/cginfoall.rb: -------------------------------------------------------------------------------- 1 | # Trisul Remote Protocol TRP Demo script 2 | # 3 | # Counter Group Info 2 4 | # 5 | # 6 | # Prints info about all counter groups 7 | # 8 | require 'rubygems' if RUBY_VERSION < '1.9' 9 | require 'trisulrp' 10 | 11 | 12 | require 'trisulrp' 13 | 14 | 15 | USAGE = "Usage: cginfoall.rb ZMQ_ENDPOINT\n" \ 16 | "Example: 1) ruby cginfoall.rb ipc:///usr/local/var/lib/trisul-hub/domain0/hub0/context0/run/trp_0\n"\ 17 | " 2) ruby cginfoall.rb tcp://localhost:5555" 18 | 19 | # usage 20 | unless ARGV.size==1 21 | abort USAGE 22 | end 23 | 24 | #zmq end point 25 | zmq_endpt = ARGV[0] 26 | 27 | # print info about all counter groups, 28 | # note that we have not specified the counter_group guid in the request 29 | # So all the counter groups are retrieved. 30 | req =mk_request(TRP::Message::Command::COUNTER_GROUP_INFO_REQUEST) 31 | 32 | get_response_zmq(zmq_endpt,req) do |resp| 33 | resp.group_details.each do |group_detail| 34 | print group_detail.name.ljust(25) 35 | print group_detail.guid 36 | print " #{Time.at(group_detail.time_interval.from.tv_sec)} " 37 | print " #{Time.at(group_detail.time_interval.to.tv_sec)} " 38 | print " #{group_detail.bucket_size} " 39 | print "\n" 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /trp/elephantflows/ftracker.rb: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/ruby 2 | # Trisul Remote Protocol TRP Demo script 3 | # 4 | # Elephant Flows - flows that transfer huge amounts of data 5 | # - These flows are tracked by Trisul as Tracker #0 6 | # 7 | # 8 | # 9 | require 'trisulrp' 10 | 11 | # Check arguments 12 | 13 | USAGE = "Usage : ftracker.rb ZMQ_ENDPOINT trackerid\n"\ 14 | "Example : 1) ruby ftracker.rb ipc:///usr/local/var/lib/trisul/CONTEXT0/run/trp_0 1\n"\ 15 | " 2) ruby ftracker.rb tcp://localhost:5555 1" 16 | 17 | #usage 18 | unless ARGV.size ==2 19 | abort USAGE 20 | end 21 | 22 | #zmq end point 23 | zmq_endpt = ARGV[0] 24 | 25 | # get all time..then for this demo script crop to latest 1 day, 26 | tmarr = TrisulRP::Protocol.get_available_time(zmq_endpt) 27 | tmarr[0] = tmarr[1]-86400 28 | 29 | # arguments 30 | trackerid = ARGV[1] 31 | 32 | # send keyspace request 33 | req = TrisulRP::Protocol.mk_request( 34 | TRP::Message::Command::SESSION_TRACKER_REQUEST, 35 | :tracker_id => trackerid.to_i, 36 | :time_interval => mk_time_interval(tmarr)) 37 | 38 | 39 | # print matching flows using the print_session_details helper 40 | get_response_zmq(zmq_endpt,req) do |resp| 41 | print_session_details_header() 42 | resp.sessions.each do |s| 43 | print_session_details(s) 44 | end 45 | end 46 | 47 | -------------------------------------------------------------------------------- /trp/flows/README.md: -------------------------------------------------------------------------------- 1 | Working with flows 2 | ================== 3 | 4 | Various types of flow lookups. 5 | 6 | ## query-flow-file_zmq.rb 7 | Save the flows in output file 8 | * Usage ruby query-flow-file_zmq.rb ipc:///usr/local/var/lib/trisul/CONTEXT0/run/trp_0 any_ip=C0.A8.01.01 9 | 10 | ## query-flow_zmq.rb 11 | Print the flow details in your terminal 12 | * Usage ruby query-flow_zmq.rb ipc:///usr/local/var/lib/trisul/CONTEXT0/run/trp_0 any_ip=C0.A8.01.01 13 | 14 | 15 | 16 | ## flows_2.rb 17 | 18 | Demonstrates searching flows by 2 parameters. 19 | 20 | * Usage of KEY_SESSION_ACTIVITY_REQUEST 21 | * How to convert hostnames/ips/appnames into keys 22 | * How to print flow details 23 | 24 | -------------------------------------------------------------------------------- /trp/flows/pcap-for-flowid.rb: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/ruby 2 | # Trisul Remote Protocol TRP Demo script 3 | # 4 | # Given a flow id in slice:flowid format, get the pcap 5 | # 6 | # Usage 7 | # ruby pcapforflow ZMQ_ENDPOINT sliceid:flowid 8 | # 9 | require 'trisulrp' 10 | 11 | 12 | USAGE = "Usage: pcap-for-flowid.rb ZMQ_ENDPOINT sliceid:flowid \n" \ 13 | "Example: 1) ruby pcap-for-flowid.rb ipc:///usr/local/var/lib/trisul/CONTEXT0/run/trp_0 2:23232\n"\ 14 | " 2) ruby pcap-for-flowid.rb tcp://localhost:5555 2:1111,1:23232" 15 | 16 | # usage 17 | unless ARGV.size==2 18 | abort USAGE 19 | end 20 | 21 | 22 | #ZMQ connection end point 23 | zmq_endpt= ARGV.shift 24 | bysession = TRP::FilteredDatagramRequest::BySession.new( 25 | :session_ids => ARGV.shift.split(',').collect do | sid | 26 | sessid = sid.split(':').map(&:to_i) 27 | TRP::SessionID.new({ :slice_id => sessid[0], :session_id => sessid[1]}) 28 | end 29 | ) 30 | 31 | 32 | req = TrisulRP::Protocol.mk_request( 33 | TRP::Message::Command::FILTERED_DATAGRAMS_REQUEST, 34 | :session => bysession 35 | ) 36 | 37 | get_response_zmq(zmq_endpt,req) do |fdr| 38 | File.open("#{fdr.sha1}.pcap","wb") do |f| 39 | f.write(fdr.contents) 40 | end 41 | print "Saved to #{fdr.sha1}.pcap\n" 42 | end 43 | 44 | -------------------------------------------------------------------------------- /trp/flows/pcap-for-resourceid.rb: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/ruby 2 | # Trisul Remote Protocol TRP Demo script 3 | # 4 | # 5 | # 6 | # Usage 7 | # ruby pcap-for-resourceid ZMQ_ENDPOINT sliceid:resid 8 | # 9 | require 'trisulrp' 10 | 11 | USAGE = "Usage: pcap-for-resourceid.rb ZMQ_ENDPOINT sliceid:flowid \n" \ 12 | "Example: 1) ruby pcap-for-resourceid.rb ipc:///usr/local/var/lib/trisul/CONTEXT0/run/trp_0 2:23232\n"\ 13 | " 2) ruby pcap-for-resourceid.rb tcp://localhost:5555 2:1111,1:23232" 14 | 15 | # usage 16 | unless ARGV.size==2 17 | abort USAGE 18 | end 19 | 20 | 21 | #ZMQ connection end point 22 | zmq_endpt= ARGV.shift 23 | 24 | by = TRP::FilteredDatagramRequest::ByResource.new( 25 | :resource_group => "{4EF9DEB9-4332-4867-A667-6A30C5900E9E}", 26 | :resource_ids => ARGV.shift.split(',').collect do | sid | 27 | sessid = sid.split(':').map(&:to_i) 28 | TRP::ResourceID.new({ :slice_id => sessid[0], :resource_id => sessid[1]}) 29 | end 30 | ) 31 | 32 | 33 | req = TrisulRP::Protocol.mk_request( 34 | TRP::Message::Command::FILTERED_DATAGRAMS_REQUEST, 35 | :resource => by 36 | ) 37 | 38 | get_response_zmq(zmq_endpt,req) do |fdr| 39 | File.open("#{fdr.sha1}.pcap","wb") do |f| 40 | f.write(fdr.contents) 41 | end 42 | print "Saved to #{fdr.sha1}.pcap\n" 43 | end 44 | 45 | -------------------------------------------------------------------------------- /trp/flows/query-by-flowid.rb: -------------------------------------------------------------------------------- 1 | 2 | #!/usr/local/bin/ruby 3 | # Trisul Remote Protocol TRP Demo script 4 | # 5 | # Query by flow id , every flow in Trisul has a unique id 6 | # of the form slice:session. Eg 1:999 7 | # 8 | # Example 9 | # ruby query-by-flowid ZMQ_ENDPOINT id 10 | # 11 | require 'trisulrp' 12 | 13 | USAGE = "Usage: query-by-flowid.rb ZMQ_ENDPOINT sliceid:flowid \n" \ 14 | "Example: 1) ruby query-by-flowid.rb ipc:///usr/local/var/lib/trisul/CONTEXT0/run/trp_0 2:23232\n"\ 15 | " 2) ruby query-by-flowid.rb tcp://localhost:5555 2:1111" 16 | 17 | # usage 18 | unless ARGV.size==2 19 | abort USAGE 20 | end 21 | 22 | 23 | #ZMQ connection end point 24 | zmq_endpt= ARGV[0] 25 | 26 | # arguments convert to a SessionID object 27 | flowid = ARGV[1].split(':').collect{|a|a.to_i} 28 | sessid = TRP::SessionID.new( {:slice_id=>flowid[0], 29 | :session_id=>flowid[1]} ) 30 | 31 | 32 | # send keyspace request 33 | req = TrisulRP::Protocol.mk_request( 34 | TRP::Message::Command::SESSION_ITEM_REQUEST, 35 | :session_ids => [sessid] ) 36 | 37 | 38 | # print matching flows using the print_session_details helper 39 | get_response_zmq(zmq_endpt,req) do |resp| 40 | resp.sessions.each do |item| 41 | print "#{item.session_id.slice_id}:#{item.session_id.session_id} " 42 | print "#{Time.at(item.time_interval.from.tv_sec)} " 43 | print "#{item.time_interval.to.tv_sec-item.time_interval.from.tv_sec} ".rjust(8) 44 | print "#{item.protocol.key}".ljust(8) 45 | print "#{item.key1A.key}".ljust(28) 46 | print "#{item.key2A.key}".ljust(11) 47 | print "#{item.key1Z.key}".ljust(28) 48 | print "#{item.key2Z.key}".ljust(11) 49 | print "#{item.az_bytes}".rjust(10) 50 | print "#{item.za_bytes}".rjust(10) 51 | print "#{item.az_packets}".rjust(10) 52 | print "#{item.za_packets}".rjust(10) 53 | print "\n" 54 | end 55 | end 56 | 57 | -------------------------------------------------------------------------------- /trp/flows/query-flow-file_zmq.rb: -------------------------------------------------------------------------------- 1 | 2 | #!/usr/local/bin/ruby 3 | # 4 | # SAME AS query-flow_zmq.rb but save flows to file 5 | # Example 6 | # ruby query-flow-file_zmq zmq:endpoint 7 | # 8 | require 'trisulrp' 9 | 10 | USAGE = "Usage: query-flow-file_zmq.rb ZMQ_ENDPOINT key=value \n" \ 11 | "Example: 1) ruby query-flow-file_zmq.rb ipc:///usr/local/var/lib/trisul/CONTEXT0/run/trp_0 source_ip=C0.A8.01.01\n"\ 12 | " 2) ruby query-flow-file_zmq.rb tcp://localhost:5555 source_ip=C0.A8.01.01" 13 | 14 | # usage 15 | unless ARGV.length>=2 16 | abort USAGE 17 | end 18 | 19 | 20 | #ZMQ connection end point 21 | zmq_endpt= ARGV.shift 22 | 23 | 24 | # process arguments 25 | qhash = ARGV.inject({}) do |acc,i| 26 | qparts = i.split("=") 27 | acc.store( qparts[0].to_sym, qparts[1]) 28 | acc 29 | end 30 | 31 | # get 24 hours latest time window 32 | tmarr = TrisulRP::Protocol.get_available_time(zmq_endpt) 33 | tmarr[0] = tmarr[1] - 24*3600 34 | 35 | # send keyspace request 36 | outputfile=File.join("/tmp","Flows_#{rand(1000000000)}.csv") 37 | req = TrisulRP::Protocol.mk_request( 38 | TRP::Message::Command::QUERY_SESSIONS_REQUEST, 39 | qhash.merge( { 40 | :time_interval => mk_time_interval(tmarr), 41 | :resolve_keys => true, 42 | :outputpath=>outputfile 43 | }) 44 | ) 45 | 46 | 47 | # print matching flows using the print_session_details helper 48 | get_response_zmq(zmq_endpt,req) do |resp| 49 | p "Output file : #{outputfile}" 50 | end 51 | 52 | -------------------------------------------------------------------------------- /trp/flows/query_pdp.rb: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/ruby 2 | # 3 | # query pdp via zmq interface 4 | # 5 | require 'trisulrp' 6 | 7 | # Check arguments 8 | raise %q{ 9 | 10 | 11 | query-pdp.rb - Query pdp by any params 12 | 13 | Usage 14 | query-pdp.rb trisul-ip trp-port 15 | 16 | 17 | x=imsi,rai,.. any of the 9 keys` 18 | 19 | 20 | Example 21 | ruby query-pdp.rb 192.168.1.22 12001 ipa=C0.A8.01.01 22 | 23 | } unless ARGV.length>=2 24 | 25 | 26 | # open a connection to Trisul server from command line args 27 | conn = ARGV.shift 28 | 29 | # process arguments 30 | qhash = ARGV.inject({}) do |acc,i| 31 | qparts = i.split("=") 32 | acc.store( qparts[0].to_sym, qparts[1]) 33 | acc 34 | end 35 | 36 | # get 24 hours latest time window 37 | tmarr = TrisulRP::Protocol.get_available_time(conn) 38 | tmarr[0] = tmarr[1] - 24*3600 39 | 40 | # send keyspace request 41 | req = TrisulRP::Protocol.mk_request( 42 | TRP::Message::Command::QUERY_PDP_REQUEST, 43 | qhash.merge( { 44 | :time_interval => mk_time_interval(tmarr), 45 | :maxitems => 10 46 | }) 47 | ) 48 | 49 | 50 | # print matching flows using the print_session_details helper 51 | get_response_zmq(conn,req) do |resp| 52 | resp.sessions.each do |item| 53 | print "#{item.session_id.slice_id}:#{item.session_id.session_id} " 54 | print "#{Time.at(item.time_interval.from.tv_sec)} " 55 | print "#{item.time_interval.to.tv_sec-item.time_interval.from.tv_sec} ".rjust(8) 56 | print "#{item.msisdn}".ljust(14) 57 | print "#{item.mccmnc}".ljust(10) 58 | print "#{item.ipa}".ljust(14) 59 | print "#{item.imei}".ljust(18) 60 | print "#{item.imsi}".ljust(18) 61 | print "#{item.apn}".ljust(11) 62 | print "#{item.rai}".ljust(11) 63 | print "#{item.rat}".ljust(11) 64 | print "#{item.uli}".ljust(14) 65 | print "[#{item.cause}]".ljust(4) 66 | print "#{item.trace}" 67 | print "\n" 68 | end 69 | end 70 | 71 | -------------------------------------------------------------------------------- /trp/fts/README.md: -------------------------------------------------------------------------------- 1 | Search for verb 2 | -------------------------------------------------------------------------------- /trp/getpcap/daypcaps.rb: -------------------------------------------------------------------------------- 1 | # Trisul Remote Protocol TRP Demo script 2 | # 3 | # get all pcaps in a given month, one file per day .. 4 | # save on server 5 | # 6 | require 'trisulrp' 7 | require 'date' 8 | 9 | USAGE = "Usage: daypcaps.rb ZMQ_ENDPOINT\n" \ 10 | "Example: 1) ruby daypcaps.rb ipc:///usr/local/var/lib/trisul/CONTEXT0/run/trp_0\n"\ 11 | " 2) ruby daypcaps.rb tcp://localhost:5555" 12 | 13 | # usage 14 | unless ARGV.length==1 15 | abort USAGE 16 | end 17 | 18 | 19 | zmq_endpt = ARGV[0] 20 | 21 | print "Enter month (YYYY-MM) : " 22 | fd = STDIN.readline.split('-') 23 | dstart = Date.new(*fd.map(&:to_i)) 24 | 25 | (dstart..dstart.next_month).each do|day| 26 | print "\nProcessing Date = #{day.to_s}\n" 27 | 28 | tint=TRP::TimeInterval.new ( { 29 | :from => TRP::Timestamp.new(:tv_sec => day.to_time.tv_sec ), 30 | :to => TRP::Timestamp.new(:tv_sec => day.next_day.to_time.tv_sec) 31 | } ) 32 | 33 | 34 | req = TrisulRP::Protocol.mk_request( 35 | TRP::Message::Command::FILTERED_DATAGRAMS_REQUEST, 36 | :disposition => TRP::PcapDisposition::SAVE_ON_SERVER, 37 | :filter_expression => 38 | TRP::FilteredDatagramRequest::ByFilterExpr.new( 39 | :time_interval => tint, 40 | :filter_expression => "{9F5AD3A9-C74D-46D8-A8A8-DCDD773730BA}=0800" 41 | ) 42 | ) 43 | 44 | 45 | # get the response and save pcap 46 | # 47 | get_response_zmq(zmq_endpt,req) do |fdr| 48 | print "Finished Date = #{day.to_s}\n" 49 | print "Number of bytes = #{fdr.num_bytes}\n" 50 | print "Number of pkts = #{fdr.num_datagrams}\n" 51 | print "Hash = #{fdr.sha1}\n" 52 | print "Saved pcap file on server = #{fdr.path}\n" 53 | end 54 | 55 | 56 | end 57 | 58 | -------------------------------------------------------------------------------- /trp/getpcap/getpackets2.rb: -------------------------------------------------------------------------------- 1 | # Trisul Remote Protocol TRP Demo script 2 | # 3 | # 4 | # Save all packets between two days to a PCAP file on the server 5 | # 6 | # == This version allows you to enter a from date and to date 7 | # and output filename 8 | # 9 | require 'trisulrp' 10 | 11 | USAGE = "Usage: getpackets2.rb ZMQ_ENDPOINT \n" \ 12 | "Example: 1) ruby getpackets2.rb ipc:///usr/local/var/lib/trisul/CONTEXT0/run/trp_0\n"\ 13 | " 2) ruby getpackets2.rb tcp://localhost:5555" 14 | 15 | # usage 16 | unless ARGV.length==1 17 | abort USAGE 18 | end 19 | 20 | #zeromq end point 21 | zmq_endpt = ARGV[0] 22 | 23 | print "Enter FROM date (YYYY-MM-DD) : " 24 | fd = STDIN.readline.split('-') 25 | print "Enter TO date (YYYY-MM-DD) : " 26 | td = STDIN.readline.split('-') 27 | tint=TRP::TimeInterval.new ( { 28 | :from => TRP::Timestamp.new(:tv_sec => Time.new(*fd).tv_sec ), 29 | :to => TRP::Timestamp.new(:tv_sec => Time.new(*td).tv_sec ) 30 | } ) 31 | 32 | 33 | # create the PCAP request for all IP packets (Ethertype = 0300) 34 | # to get ip packets use filter expression 35 | # "{9F5AD3A9-C74D-46D8-A8A8-DCDD773730BA}=0800" 36 | # ^-- this means link layer countergroup = 0x0800 37 | # 38 | req = TrisulRP::Protocol.mk_request( 39 | TRP::Message::Command::FILTERED_DATAGRAMS_REQUEST, 40 | :disposition => TRP::PcapDisposition::SAVE_ON_SERVER, 41 | :filter_expression => 42 | TRP::FilteredDatagramRequest::ByFilterExpr.new( 43 | :time_interval => tint, 44 | :filter_expression => "{9F5AD3A9-C74D-46D8-A8A8-DCDD773730BA}=0800" 45 | ) 46 | ) 47 | 48 | 49 | # get the response and save pcap 50 | # 51 | get_response_zmq(zmq_endpt,req) do |fdr| 52 | print "Number of bytes = #{fdr.num_bytes}\n" 53 | print "Number of pkts = #{fdr.num_datagrams}\n" 54 | print "Hash = #{fdr.sha1}\n" 55 | print "Saved pcap file on server = #{fdr.path}\n" 56 | end 57 | 58 | 59 | -------------------------------------------------------------------------------- /trp/getvolume/README.md: -------------------------------------------------------------------------------- 1 | Get volume of any counter item 2 | ============================== 3 | 4 | How much HTTP data was transferred in last 1 week ? 5 | This script will help you answer volume based questions like the above 6 | 7 | 8 | This script demonstrates the following 9 | 10 | 1. How to find out the bucket size (resolution) of counter groups 11 | 2. How to retrieve and calculate total volume 12 | 13 | 14 | The following run prints HTTP volumes in past 24 hours. 15 | 16 | Note that HTTP is identified by the counter group {C-51...} and key p-0050 17 | 18 | ``` 19 | $ ruby ../trisul-scripts/getvolume/getvolume.rb 192.168.1.22 12001 {C51B48D4-7876-479E-B0D9-BD9EFF03CE2E} p-0050 20 | Enter PEM pass phrase: 21 | Volume of Meter 0 = 157639740 bytes 22 | Volume of Meter 1 = 840 bytes 23 | Volume of Meter 2 = 144789000 bytes 24 | Volume of Meter 3 = 12846390 bytes 25 | Volume of Meter 4 = 183090 bytes 26 | Volume of Meter 5 = 0 bytes 27 | 28 | ``` 29 | -------------------------------------------------------------------------------- /trp/getvolume/get_total_volume.rb: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/ruby 2 | # Trisul Remote Protocol TRP Demo script 3 | # 4 | # Same as getvolume.rb but via ZMQ interface 5 | # 6 | # Example 7 | # ruby get_total_volume.rb tcp://localhost:5555 8 | # 9 | # 10 | require 'trisulrp' 11 | require '../helpers/model_utils.rb' 12 | 13 | USAGE = "Usage: getvolume.rb ZMQ_ENDPT CGGUID CGKEY \n" \ 14 | "Example: 1) ruby get_total_volume.rb tcp://localhost:5555 \n"\ 15 | " 2) ruby get_total_volume.rb ipc:///usr/local/var/lib/trisul-hub/domain0/hub0/context0/run/trp_0" 16 | 17 | # usage 18 | unless ARGV.size==1 19 | abort USAGE 20 | end 21 | 22 | zmq_endpt = ARGV[0] 23 | target_guid = "{393B5EBC-AB41-4387-8F31-8077DB917336}" 24 | target_key = "TOTALBW" 25 | 26 | 27 | # get 24 hours latest time window 28 | tmarr = TrisulRP::Protocol.get_available_time(zmq_endpt) 29 | tmarr[0] = tmarr[1] - 24*3600 30 | bucket_size = 60 31 | 32 | # find out the bucket size in seconds 33 | req = mk_request(TRP::Message::Command::COUNTER_GROUP_INFO_REQUEST, 34 | :counter_group => target_guid ) 35 | resp = TrisulRP::Protocol.get_response_zmq(zmq_endpt,req) 36 | target_bucket_size = resp.group_details[0].bucket_size 37 | 38 | # send request for http ( cg=APPS, key=p-0050) 39 | # notice use of volumes_only => 1 40 | req = TrisulRP::Protocol.mk_request(TRP::Message::Command::COUNTER_ITEM_REQUEST, 41 | :counter_group=> target_guid , 42 | :key=> target_key , 43 | :volumes_only=>1, 44 | :time_interval => mk_time_interval(tmarr) ) 45 | 46 | # print volume for each meter 47 | get_response_zmq(zmq_endpt,req) do |resp| 48 | p "Total Volume : #{ModelUtils::fmt_volume(resp[:totals].values.first*bucket_size,'B')}" 49 | end 50 | 51 | -------------------------------------------------------------------------------- /trp/getvolume/getvolume_zmq.rb: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/ruby 2 | # Trisul Remote Protocol TRP Demo script 3 | # 4 | # Same as getvolume.rb but via ZMQ interface 5 | # 6 | # Example 7 | # ruby getvolume_zmq.rb tcp://localhost:5555 {C51B48D4-7876-479E-B0D9-BD9EFF03CE2E} p-0050 8 | # 9 | # 10 | require 'trisulrp' 11 | 12 | USAGE = "Usage: getvolume.rb ZMQ_ENDPT CGGUID CGKEY \n" \ 13 | "Example: 1) ruby getvolume.rb tcp://localhost:5555 {C51B48D4-7876-479E-B0D9-BD9EFF03CE2E} p-0050\n"\ 14 | " 2) ruby getvolume.rb ipc:///usr/local/var/lib/trisul/CONTEXT0/run/trp_0 {C51B48D4-7876-479E-B0D9-BD9EFF03CE2E} p-0050" 15 | 16 | # usage 17 | unless ARGV.size==3 18 | abort USAGE 19 | end 20 | 21 | zmq_endpt = ARGV[0] 22 | target_guid = ARGV[1] 23 | target_key = ARGV[2] 24 | 25 | 26 | # get 24 hours latest time window 27 | tmarr = TrisulRP::Protocol.get_available_time(zmq_endpt) 28 | tmarr[0] = tmarr[1] - 24*3600 29 | 30 | # find out the bucket size in seconds 31 | req = mk_request(TRP::Message::Command::COUNTER_GROUP_INFO_REQUEST, 32 | :counter_group => target_guid ) 33 | resp = TrisulRP::Protocol.get_response_zmq(zmq_endpt,req) 34 | target_bucket_size = resp.group_details[0].bucket_size 35 | 36 | # send request for http ( cg=APPS, key=p-0050) 37 | # notice use of volumes_only => 1 38 | req = TrisulRP::Protocol.mk_request(TRP::Message::Command::COUNTER_ITEM_REQUEST, 39 | :counter_group=> target_guid , 40 | :key=> target_key , 41 | :time_interval => mk_time_interval(tmarr) ) 42 | 43 | # print volume for each meter 44 | get_response_zmq(zmq_endpt,req) do |resp| 45 | resp.stats.meters[0].values.each do |val| 46 | p "#{Time.at(val.ts.tv_sec)} = #{val.val}" 47 | end 48 | end 49 | 50 | -------------------------------------------------------------------------------- /trp/helloworld/README.md: -------------------------------------------------------------------------------- 1 | Simplest script - hello world 2 | ============================= 3 | 4 | Sends a simple HelloRequest to the TRP Server and prints the output. 5 | [TRP Documentation](https://trisul.org/docs/ref/trpproto.html) 6 | 7 | 8 | ## How to get the server endpoint? 9 | 10 | Connect to the Trisul Domain from the cli and type the following 11 | 12 | the **endpoints_query** port is the server port. 13 | 14 | ```` 15 | trisul_hub:unpl-seco-16-prod(domain0)> show config default@hub0 16 | 17 | node hub0 18 | context_name default 19 | endpoints_flush tcp://192.168.2.99:13000 20 | endpoints_flush tcp://192.168.2.99:13001 21 | endpoints_query tcp://192.168.2.99:13004 22 | endpoints_pub tcp://192.168.2.99:13002 23 | endpoints_pub tcp://192.168.2.99:13003 24 | 25 | layer probe 26 | 0 probe0 27 | 1 probeWEST 28 | 3 probeEAST00 29 | -------------------------------------------------------------- 30 | trisul_hub:unpl-seco-16-prod(domain0)> 31 | 32 | ```` 33 | 34 | 35 | How to run 36 | ---------- 37 | 38 | 39 | hello.rb 40 | 41 | ```` 42 | ruby hello.rb tcp://192.168.1.222:13004 43 | 44 | ```` 45 | 46 | 47 | -------------------------------------------------------------------------------- /trp/helloworld/hello.rb: -------------------------------------------------------------------------------- 1 | # Trisul Remote Protocol TRP Demo script 2 | # 3 | # helloworld - connect to a Trisul sensor and print sensor ID 4 | # 5 | # Usage ruby hello.rb 6 | # 7 | require 'trisulrp' 8 | 9 | USAGE = "Usage: hello.rb ZMQ_ENDPT \n" \ 10 | "Example: 1) ruby hello.rb tcp://localhost:5555 \n"\ 11 | " 2) ruby hello.rb ipc:///usr/local/var/lib/trisul/CONTEXT0/run/trp_0" 12 | 13 | # usage 14 | unless ARGV.size==1 15 | abort USAGE 16 | end 17 | zmq_endpt = ARGV[0] 18 | 19 | req = mk_request(TRP::Message::Command::HELLO_REQUEST, 20 | :station_id => "MyAutomationProg") 21 | 22 | get_response_zmq(zmq_endpt,req) do |resp| 23 | p resp.station_id 24 | p resp.station_id_request 25 | p resp.message 26 | p resp.local_timestamp 27 | end 28 | 29 | -------------------------------------------------------------------------------- /trp/hourlystats/README.md: -------------------------------------------------------------------------------- 1 | Hourly Statistics of any item 2 | ================================== 3 | 4 | 5 | This script allows you to draw a neat hourly chart of traffic of any item. 6 | 7 | - hourlystats.rb : gets the raw timeseries data (30s) and adds it up 8 | - hourlystats2.rb : uses the volumes_only flag to avoid getting raw timeseries 9 | 10 | 11 | ``` 12 | [vivek@localhost trp]$ ruby hourlystats.rb 192.168.1.22 12001 {C51B48D4-7876-479E-B0D9-BD9EFF03CE2E} p-0050 0 13 | Enter PEM pass phrase: 14 | Day/Hour| 03:00| 06:00| 09:00| 12:00| 15:00| 18:00| 21:00| 00:00| 15 | 2013-01-24| 0| 0| 0| 22293120| 65219070| 15995910| 0| 0| 16 | 2013-01-23| 0| 0| 0| 0| 0| 0| 0| 0| 17 | 2013-01-22| 0| 0| 0| 52982520| 63958710| 94162590| 29339940| 0| 18 | 2013-01-21| 0| 0| 0| 0| 0| 0| 0| 0| 19 | 201 20 | 21 | ``` 22 | 23 | You need to know the GUID, Meter No, and Key of any item. 24 | 25 | 26 | GUID and meters are available from Customize -> Meters 27 | 28 | The Key is available from the key dashboard. Search for any item in the search box and get the key. 29 | 30 | 31 | Some common keys 32 | ---------------- 33 | 34 | To get HTTP Total Volume 35 | ``` 36 | [vivek@localhost trp]$ ruby hourlystats.rb trisul-ip 12001 {C51B48D4-7876-479E-B0D9-BD9EFF03CE2E} p-0050 0 37 | ``` 38 | 39 | To get HTTP Into Your Network (use meter id 2 instead of 0: the last parameter). See Customize > Counters > View Meters 40 | 41 | ``` 42 | [vivek@localhost trp]$ ruby hourlystats.rb trisul-ip 12001 {C51B48D4-7876-479E-B0D9-BD9EFF03CE2E} p-0050 1 43 | ``` 44 | 45 | Traffic transmitted by IP 192.168.1.8. Search and the key is C0.A8.01.08 and Guids from View Meters 46 | 47 | ``` 48 | [vivek@localhost trp]$ ruby hourlystats.rb trisul-ip 12001 {4CD742B1-C1CA-4708-BE78-0FCA2EB01A86} C0.A8.01.08 1 49 | ``` 50 | -------------------------------------------------------------------------------- /trp/ids/README.md: -------------------------------------------------------------------------------- 1 | Working with IDS Alerts 2 | ======================== 3 | 4 | Scripts dealing with IDS alerts, flows, and packets 5 | 6 | 7 | ## save_pcap.rb 8 | 9 | save_pcap.rb demonstrates how you can automate gathering together 10 | context around an alert. This script can be run daily and will 11 | gather all packets in all flows that generated a Severity (Priority) 1 12 | alert into a single PCAP file. 13 | 14 | ### Techinques used 15 | 16 | * Use QUERY_SESSIONS to retrieve flows by tag 17 | * Working with session_id 18 | * Retrieving packets for multiple flows 19 | * Naming the PCAP file as the SHA1 of contents 20 | 21 | Sample run 22 | 23 | ```` 24 | 25 | [nsmeast@localhost helloworld]$ ruby save-pcap.rb 127.0.0.1 12001 26 | Enter PEM pass phrase: 27 | Saved to 5ed6ed34b431a3c253112fa7cefaaa93d6172ef3.pcap 28 | 29 | 30 | 31 | ```` 32 | 33 | -------------------------------------------------------------------------------- /trp/ioc-sweeper/README.md: -------------------------------------------------------------------------------- 1 | Sweeping past traffic against OpenIOC Intel 2 | ================================================ 3 | 4 | Trisul 3.0 sample script to demonstrate how you 5 | can consume an OpenIOC format feed and sweep past 6 | traffic for matches. 7 | 8 | 9 | 10 | The ruby script iocsweep.rb 11 | ------------------------------------ 12 | 13 | You also need to install Nokogiri to process the OpenIOC XML file 14 | 15 | ```` 16 | gem install nokogiri 17 | ```` 18 | 19 | 20 | Locating Indicators of a particular type 21 | ---------------------------------------- 22 | 23 | A little bit of XML XPath magic lets you get a handle on the various 24 | indicators. For example 25 | 26 | ```` 27 | 28 | # gets an array of IndicatorItems of type PortItem/remoteIP 29 | doc.xpath("//xmlns:IndicatorItem/xmlns:Context[@search='PortItem/remoteIP']") 30 | 31 | # easy to extract the IPs contained inside the indicator.. 32 | doc.xpath("//xmlns:IndicatorItem/xmlns:Context[@search='PortItem/remoteIP']") 33 | .collect do |a| 34 | a.at_xpath("//xmlns:Content").text 35 | end 36 | 37 | ```` 38 | 39 | -------------------------------------------------------------------------------- /trp/keyspace/README.md: -------------------------------------------------------------------------------- 1 | Active keys in a range 2 | ====================== 3 | 4 | The KEYSPACE_REQUEST command is used to retrieve the active keys in a range. 5 | 6 | This request can be used to retrieve all IPs seen in the 10.x.x.x private IP range. 7 | 8 | This directory has two scripts 9 | 10 | 1. active_keys.rb - Print all active keys in Trisul Key Format 11 | 12 | 13 | ### TRP Messages used 14 | 15 | The scripts uses the following TRP Messages 16 | 17 | 1. [KeySpace](http://trisul.org/docs/ref/trpprotomessages.html#keyspace) - to retrieve keys in a range 18 | 19 | 20 | ### Sample run 21 | 22 | 23 | The following run displays the all Ports seen in the range p-0000 to p-1000. In Trisul key format 24 | this equate port 0 to port 4096 25 | 26 | Note that "Apps" counter group has the GUID {C51..} and the keys are in Trisul Key Format. 27 | 28 | 29 | ``` 30 | [vivek@longdog trp]$ ruby active_keys.rb tcp://192.168.1.22:12001 {C51B48D4-7876-479E-B0D9-BD9EFF03CE2E} p-0000 p-1000 31 | Found 15 matches 32 | Hit Key p-000D 33 | Hit Key p-0016 34 | Hit Key p-0035 35 | Hit Key p-0043 36 | Hit Key p-0050 37 | Hit Key p-007B 38 | Hit Key p-0089 39 | Hit Key p-008A 40 | Hit Key p-01BB 41 | Hit Key p-034B 42 | Hit Key p-03E1 43 | Hit Key p-076C 44 | Hit Key p-078F 45 | Hit Key p-0BB8 46 | Hit Key p-0C3B 47 | 48 | ```` 49 | 50 | 51 | -------------------------------------------------------------------------------- /trp/keyspace/active_keys.rb: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/ruby 2 | # Trisul Remote Protocol TRP Demo script 3 | # 4 | # Print all active keys in given range and time interval 5 | # 6 | # 7 | require 'trisulrp' 8 | 9 | # Check arguments 10 | #active_keys.rb - keys active in a range. Use to print all hosts seen in subnet etc 11 | 12 | USAGE = "Usage : ruby active_keys.rb ZMQ_ENDPOINT CGGUID KFROM KTO \n"\ 13 | "Example : 1) ruby active_keys.rb tcp://localhost:5555 {C51B48D4-7876-479E-B0D9-BD9EFF03CE2E} p-0000 p-FFFF\n"\ 14 | " 2) ruby active_keys.rb ipc:///usr/local/var/lib/trisul/CONTEXT0/run/trp_0 {C51B48D4-7876-479E-B0D9-BD9EFF03CE2E} p-0000 p-FFFF" 15 | 16 | 17 | unless ARGV.size == 4 18 | abort USAGE 19 | end 20 | 21 | #Get ZeroMQ end point 22 | zmq_endpt = ARGV[0] 23 | 24 | # get 24 hours latest time window 25 | tmarr = TrisulRP::Protocol.get_available_time(zmq_endpt) 26 | tmarr[0] = tmarr[1] - 3*86400 27 | 28 | # arguments 29 | cgguid = ARGV[1] 30 | from_key = ARGV[2] 31 | to_key = ARGV[3] 32 | 33 | # space message 34 | space = TRP::KeySpaceRequest::KeySpace.new( :from_key => TRP::KeyT.new( :key => from_key) , 35 | :to_key => TRP::KeyT.new( :key=>to_key) ) 36 | 37 | # send keyspace request 38 | req = TrisulRP::Protocol.mk_request( 39 | TRP::Message::Command::KEYSPACE_REQUEST, 40 | :counter_group => cgguid , 41 | :time_interval => mk_time_interval(tmarr), 42 | :spaces => [space] , 43 | :maxitems => 500 ) 44 | 45 | 46 | # print hits 47 | get_response_zmq(zmq_endpt,req) do |resp| 48 | puts "Found #{resp.hits.size} matches" 49 | resp.hits.each do | res | 50 | puts "Hit Key #{res.key} " 51 | end 52 | end 53 | 54 | 55 | -------------------------------------------------------------------------------- /trp/python/README.md: -------------------------------------------------------------------------------- 1 | # TRP samples for python 2 | 3 | How to use Python to query Trisul data using TRP (Trisul Remote Protocol). 4 | https://www.trisul.org/docs/trp/index.html 5 | 6 | TRP is a query/response API that uses the following technologies 7 | 1. ZeroMQ for communication 8 | 2. Google Protocol Buffers for the API messages 9 | 10 | 11 | ### Prerequisites 12 | 13 | 1. Download trp.proto from https://raw.githubusercontent.com/trisulnsm/trisul-scripts/master/trp/trp.proto 14 | `curl -O https://raw.githubusercontent.com/trisulnsm/trisul-scripts/master/trp/trp.proto` 15 | 16 | 17 | 2. Then run the following steps to create a run environment 18 | 19 | ```sh 20 | $ protoc trp.proto --python_out=. 21 | $ sudo apt-get install python-pip 22 | $ sudo pip install protobuf 23 | $ sudo apt-get install python-zmq 24 | ``` 25 | 26 | ## Running the samples 27 | 28 | To run a sample just type. 29 | 30 | ```sh 31 | $ python cginfo.py ipc:///usr/local/var/lib/trisul-hub/domain0/hub0/context0/run/trp_0 32 | ``` 33 | 34 | The string `ipc:///usr/local/var/lib/trisul-hub/domain0/hub0/context0/run/trp_0` is the ZMQ connection string where the Trisul TRP server is running on. To find out the connection string the default context type. 35 | 36 | ```sh 37 | trisulctl_hub show config default 38 | ``` 39 | 40 | 41 | Another example. 42 | 43 | 44 | ```sh 45 | $ python counter_group_topper_request.py ipc:///usr/local/var/lib/trisul-hub/domain0/hub0/context0/run/trp_0 {889900CC-0063-11A5-8380-FEBDBABBDBEA} 0 46 | ``` 47 | -------------------------------------------------------------------------------- /trp/python/cginfo.py: -------------------------------------------------------------------------------- 1 | ''' 2 | # CG Info 3 | # Prints all counter groups 4 | ''' 5 | 6 | import sys 7 | import datetime 8 | import trp_pb2 9 | import zmq 10 | 11 | def usage(): 12 | print "python ", sys.argv[0],"" 13 | print "python ", sys.argv[0],"ipc:///usr/local/var/lib/trisul-hub/domain0/hub0/context0/run/trp_0" 14 | 15 | 16 | if len(sys.argv) != 2: 17 | usage() 18 | sys.exit(2) 19 | 20 | 21 | # helper 22 | def get_response(zmq_endpoint,req): 23 | #zmq send 24 | context = zmq.Context() 25 | socket = context.socket(zmq.REQ) 26 | socket.connect(zmq_endpoint) 27 | socket.send(req.SerializeToString()) 28 | 29 | #zmq receive 30 | data=socket.recv() 31 | resp = unwrap_response(data) 32 | return resp 33 | socket.close 34 | 35 | # helper 36 | def unwrap_response(data): 37 | resp = trp_pb2.Message() 38 | resp.ParseFromString(data) 39 | for x in resp.DESCRIPTOR.enum_types: 40 | name = x.values_by_number.get(int(resp.trp_command)).name 41 | return { 42 | 'COUNTER_GROUP_INFO_RESPONSE':resp.counter_group_info_response 43 | }.get(name,resp) 44 | 45 | ################# 46 | # actual code starts here 47 | # to retrieve all counter groups send an empty COUNTER_GROUP_INFO_REQUEST 48 | 49 | req = trp_pb2.Message() 50 | req.trp_command=req.COUNTER_GROUP_INFO_REQUEST 51 | resp = get_response(sys.argv[1],req) 52 | 53 | # display the information 54 | for cg in resp.group_details: 55 | print cg.guid + "," + cg.name 56 | 57 | 58 | -------------------------------------------------------------------------------- /trp/resources/sweep_intel.rb: -------------------------------------------------------------------------------- 1 | # 2 | # sweep_intel : Search the INTEL resource group for a match 3 | # 4 | require 'trisulrp' 5 | 6 | USAGE = "Usage: sweep-intel.rb ZMQ_ENDPT time-from time-to indicator \n" \ 7 | 8 | # usage 9 | abort USAGE unless ARGV.size==4 10 | 11 | zmq_endpt = ARGV[0] 12 | time_from_str = ARGV[1] 13 | time_to_str = ARGV[2] 14 | pattern = ARGV[3] 15 | 16 | time_interval=[ Time.parse(time_from_str), 17 | Time.parse(time_to_str) ] 18 | 19 | p time_interval 20 | 21 | req = TrisulRP::Protocol.mk_request( 22 | TRP::Message::Command::QUERY_RESOURCES_REQUEST, 23 | :resource_group => "{EE1C9F46-0542-4A7E-4C6A-55E2C4689419}", 24 | :time_interval => time_interval, 25 | :regex_uri => pattern ) 26 | 27 | # print hits 28 | get_response_zmq(zmq_endpt,req) do |resp| 29 | resp.resources.each do |res| 30 | req2 = TrisulRP::Protocol.mk_request( 31 | TRP::Message::Command::QUERY_RESOURCES_REQUEST, 32 | :resource_group => "{EE1C9F46-0542-4A7E-4C6A-55E2C4689419}", 33 | :idlist => res.resource_id ) 34 | get_response_zmq(zmq_endpt,req2) do |resp2| 35 | oneres = resp2.resources[0] 36 | print("#{oneres.probe_id} #{oneres.resource_id} #{oneres.userlabel}\n") 37 | end 38 | end 39 | end 40 | 41 | -------------------------------------------------------------------------------- /trp/stats/cistats.rb: -------------------------------------------------------------------------------- 1 | # Trisul Remote Protocol TRP Demo script 2 | # 3 | # Counter item stats 4 | # 5 | # All stats for all time for a particular counter item (press Ctrl+C to stop ) 6 | # 7 | # ruby cistats.rb tcp://192.168.1.8:5555 {C51B48D4-7876-479E-B0D9-BD9EFF03CE2E} p-0050 8 | # ruby cistats.rb tcp://192.168.1.8:5555 {393B5EBC-AB41-4387-8F31-8077DB917336} TOTALBW 9 | # 10 | # 11 | require 'trisulrp' 12 | 13 | # Check arguments 14 | USAGE = "cistats.rb - Dump all stats \n"\ 15 | "Usage : cistats.rb trisul-zmq-endpt cgguid key\n"\ 16 | "Example : 1) ruby cistats.rb tcp://192.168.1.8:5555 {393B5EBC-AB41-4387-8F31-8077DB917336} TOTALBW\n"\ 17 | " 2) ruby cistats.rb ipc:///usr/local/var/lib/trisul/CONTEXT0/run/trp_0 {393B5EBC-AB41-4387-8F31-8077DB917336} TOTALBW" 18 | unless ARGV.length==3 19 | abort USAGE 20 | end 21 | 22 | # parameters 23 | zmq_endpt = ARGV.shift 24 | target_guid = ARGV.shift 25 | target_key = ARGV.shift 26 | 27 | # get entire time window 28 | tmarr= TrisulRP::Protocol.get_available_time(zmq_endpt) 29 | 30 | # toppers 31 | req =TrisulRP::Protocol.mk_request(TRP::Message::Command::COUNTER_ITEM_REQUEST, 32 | :counter_group => target_guid, 33 | :key => target_key, 34 | :time_interval => mk_time_interval(tmarr) ) 35 | 36 | TrisulRP::Protocol.get_response_zmq(zmq_endpt,req) do |resp| 37 | print "Counter Group = #{resp.counter_group}\n" 38 | print "Key = #{resp.key.key} #{resp.key.label} \n" 39 | resp.stats.each do |stat| 40 | print "#{Time.at(stat.ts_tv_sec).to_s} " 41 | stat.values.each do |val| 42 | print "#{val} " 43 | end 44 | print "\n" 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /trp/toppers/topper_trend.rb: -------------------------------------------------------------------------------- 1 | # Trisul Remote Protocol TRP Demo script 2 | # 3 | # Topper Trend 4 | # Traffic trends for topper items 5 | # 6 | # ruby topper_trend.rb zmq-endpt {C51B48D4-7876-479E-B0D9-BD9EFF03CE2E} 1 7 | # 8 | require 'trisulrp' 9 | 10 | # Check arguments 11 | 12 | USAGE = " topper_trend.rb - Retrieve toppers for any counter and stat for last 1 day \n"\ 13 | "Usage : topper_trend.rb zmq-entpt cgguid meter-id\n"\ 14 | "Example : 1) ruby topper_trend.rb tcp://localhost:5555 {C51B48D4-7876-479E-B0D9-BD9EFF03CE2E} 0\n"\ 15 | " 2) ruby topper_trend.rb ipc:///usr/local/var/lib/trisul/CONTEXT0/run/trp_0 {C51B48D4-7876-479E-B0D9-BD9EFF03CE2E} 0" 16 | 17 | unless ARGV.length==3 18 | abort USAGE 19 | end 20 | 21 | 22 | zendpt = ARGV.shift 23 | 24 | # parameters 25 | target_guid = ARGV.shift 26 | target_meter = ARGV.shift 27 | 28 | # last 24 hours 29 | tmarr= TrisulRP::Protocol.get_available_time(zendpt) 30 | tmarr[0] = tmarr[1] - 24*3600 31 | 32 | # get keys 33 | req =TrisulRP::Protocol.mk_request( 34 | TRP::Message::Command::TOPPER_TREND_REQUEST, 35 | :counter_group => target_guid, 36 | :meter => target_meter.to_i, 37 | :time_interval => mk_time_interval(tmarr)) 38 | 39 | TrisulRP::Protocol.get_response_zmq(zendpt,req) do |resp| 40 | print "Counter Group = #{resp.counter_group}\n" 41 | print "Meter = #{resp.meter}\n" 42 | print "Numkeys = #{resp.keytrends.size}\n" 43 | resp.keytrends.each do |kt| 44 | print "key = #{kt.key}\n" 45 | m = kt.meters[0] 46 | print "meter = #{m.meter}\n" 47 | m.values.each do |v| 48 | print "ts = #{v.ts.tv_sec}=#{v.val}\n" 49 | end 50 | 51 | 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /trp/toppers/toppers_zmq.rb: -------------------------------------------------------------------------------- 1 | # Trisul Remote Protocol TRP Demo script 2 | # 3 | # Same as toppers.rb but uses ZMQ transport 4 | # 5 | # Prints topper for the particular counter group for any meter in specified time range 6 | # 7 | # ruby topper_zmq.rb 192.168.1.45 12001 {C51B48D4-7876-479E-B0D9-BD9EFF03CE2E} 8 | # 9 | require 'trisulrp' 10 | USAGE = " toppers_zmq.rb - Retrieve toppers for any counter and stat \n"\ 11 | "Usage : toppers_zmq.rb trisul-zmq-endpt cgguid meter-id\n"\ 12 | "Example : 1) ruby toppers_zmq.rb tcp://localhost:5555 {C51B48D4-7876-479E-B0D9-BD9EFF03CE2E} 0\n"\ 13 | " 2) ruby toppers_zmq.rb ipc:///usr/local/var/lib/trisul/CONTEXT0/run/trp_0 {C51B48D4-7876-479E-B0D9-BD9EFF03CE2E} 0" 14 | 15 | unless ARGV.length==3 16 | abort USAGE 17 | end 18 | 19 | 20 | zmq_endpt = ARGV.shift 21 | target_guid = ARGV.shift 22 | target_meter = ARGV.shift 23 | 24 | # get topper bucket size - multiply metric by that 25 | #req = mk_request(TRP::Message::Command::COUNTER_GROUP_INFO_REQUEST, 26 | # :counter_group => target_guid ) 27 | #resp = TrisulRP::Protocol.get_response_zmq(zmq_endpt,req) 28 | #target_bucket_size = resp.group_details[0].topper_bucket_size 29 | #print "bucket size = #{target_bucket_size}" 30 | 31 | 32 | # toppers 33 | req =TrisulRP::Protocol.mk_request(TRP::Message::Command::COUNTER_GROUP_REQUEST, 34 | :counter_group => target_guid, 35 | :meter => target_meter.to_i, 36 | :time_interval => mk_time_interval([Time.at(0),Time.at(9999999999)])) 37 | 38 | TrisulRP::Protocol.get_response_zmq(zmq_endpt,req) do |resp| 39 | print "Counter Group = #{resp.counter_group}\n" 40 | print "Meter = #{resp.meter}\n" 41 | resp.keys.each do |key| 42 | total_bytes = key.metric * 300 43 | print "Key = #{key.key} Label = #{key.label} Metric= #{total_bytes}\n" 44 | end 45 | end 46 | -------------------------------------------------------------------------------- /trp/youtube/README.md: -------------------------------------------------------------------------------- 1 | Search and pull out Youtube videos 2 | ================================== 3 | 4 | 5 | This script will save all YouTube videos seen by a Trisul sensor. 6 | 7 | Written in Ruby with the trisulrp (http://trisul.org/docs/trp/) gem. 8 | 9 | There are two scripts 10 | 11 | #### 1. youtube_vids.rb Get PCAP of Youtube videos 12 | Connect to a Trisul Sensor, search for video streams, download PCAPs 13 | of these video streams. 14 | 15 | 16 | #### 2. youtube_titles.rb Dump the video file and rename with Title 17 | Load the PCAP into Unsniff and dump the playable video file 18 | Use Unsniff to get the 'HTTP referer' for each video stream, 19 | Get the referrer page PCAP 20 | Dump the referrer HTML using Unsniff API 21 | Load the referred HTML using Nokogiri and search for the 22 | Change the name of the video file to the "title" 23 | 24 | 25 | Demonstrates the following 26 | -------------------------- 27 | 28 | - Using the Trisul Remote Protocol (TRP) 29 | - How to search for URLs matching a certain pattern 30 | - Extract PCAPs for matching URLs 31 | - Using the Unsniff Network Analyzer API to parse HTTP 32 | - Dumping Videos in WEBM and FLV format using Unsniff API 33 | 34 | --------------------------------------------------------------------------------