├── .gitattributes ├── docs-img ├── mpl-ts.png ├── tele-bot.png ├── grafana-1.png ├── grafana-2.png ├── grafana-3.png ├── grafana-4.png ├── grafana-as.png ├── grafana-pr.png ├── grafana-tr.png ├── grafana-tr1.png ├── grafana-tr2.png ├── mpl-ts-ips.png ├── gnuplot-day-i.png ├── mpl-donut-as.png ├── gnuplot-day-ip.png ├── gnuplot-day-prot.png ├── grafana-pie-calc.png ├── mpl-donut-prot.png ├── pg-data-source.png ├── grafana-class-http.png ├── grafana-class-port.png ├── grafana-class-size.png ├── grafana-geoip-city.png ├── grafana-geoip-country.png ├── grafana-geoip-state.png └── v2502--grafana-sample.png ├── .travis.yml ├── .gitmodules ├── devices.conf ├── xe-debug.h ├── lxc ├── xenoeye.service ├── mo │ ├── mynet-ingress │ │ ├── icmp │ │ │ └── mo.conf │ │ ├── tcp │ │ │ ├── mo.conf │ │ │ └── syn │ │ │ │ └── mo.conf │ │ ├── udp │ │ │ ├── mo.conf │ │ │ ├── dns-amp │ │ │ │ └── mo.conf │ │ │ └── ntp-amp │ │ │ │ └── mo.conf │ │ ├── frag │ │ │ └── mo.conf │ │ └── mo.conf │ ├── mynet-ingress6 │ │ ├── icmp │ │ │ └── mo.conf │ │ ├── tcp │ │ │ ├── mo.conf │ │ │ └── syn │ │ │ │ └── mo.conf │ │ ├── udp │ │ │ ├── mo.conf │ │ │ ├── dns-amp │ │ │ │ └── mo.conf │ │ │ └── ntp-amp │ │ │ │ └── mo.conf │ │ ├── frag │ │ │ └── mo.conf │ │ └── mo.conf │ ├── mynet-egress │ │ └── mo.conf │ └── mynet-egress6 │ │ └── mo.conf └── grafana │ ├── Routers.json │ └── Routers6.json ├── filter-ag.def ├── scripts ├── fill-db.sh ├── telegram-bot │ ├── on-start.sh │ ├── on-stop.sh │ ├── xe-tele-bot.py │ └── xe-tele-bot-aio3.py ├── xe-dbexport-pg.sh ├── xe-dbexport-ch.sh ├── mkchart-gnuplot.sh ├── mkchart-matplotlib-ts.py ├── mkchart-prot-mpl.py ├── mkchart-matplotlib-donut-prot.py ├── mkchart-matplotlib-donut-as.py └── mkchart-matplotlib-ts-ip.py ├── sflow.h ├── iplist.h ├── flow-debug.h ├── devices.h ├── .github └── workflows │ └── c.yml ├── .gitignore ├── tests └── test_filters.c ├── monit-objects-common.h ├── LICENSE ├── netflow-templates.h ├── xenoeye.conf ├── flow-info.h ├── configure.ac ├── geoip.h ├── filter.def ├── utils.c ├── Makefile.am ├── scapture.c ├── xenoeye.h ├── utils.h ├── xe-sni.h ├── monit-objects-mavg-limfile.c ├── netflow.h ├── netflow.def ├── xe-dns.h ├── sflow.c ├── geoip.c ├── netflow-templates.c ├── rawparse.h ├── filter-lexer.c ├── ip-btrie.h ├── iplist.c ├── xegeoq.c ├── xesflow.c ├── devices.c ├── monit-objects-mavg-dump.c ├── monit-objects.h ├── pcapture.c └── monit-objects-mavg-under.c /.gitattributes: -------------------------------------------------------------------------------- 1 | *.c linguist-language=C 2 | -------------------------------------------------------------------------------- /docs-img/mpl-ts.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmxdev/xenoeye/HEAD/docs-img/mpl-ts.png -------------------------------------------------------------------------------- /docs-img/tele-bot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmxdev/xenoeye/HEAD/docs-img/tele-bot.png -------------------------------------------------------------------------------- /docs-img/grafana-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmxdev/xenoeye/HEAD/docs-img/grafana-1.png -------------------------------------------------------------------------------- /docs-img/grafana-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmxdev/xenoeye/HEAD/docs-img/grafana-2.png -------------------------------------------------------------------------------- /docs-img/grafana-3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmxdev/xenoeye/HEAD/docs-img/grafana-3.png -------------------------------------------------------------------------------- /docs-img/grafana-4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmxdev/xenoeye/HEAD/docs-img/grafana-4.png -------------------------------------------------------------------------------- /docs-img/grafana-as.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmxdev/xenoeye/HEAD/docs-img/grafana-as.png -------------------------------------------------------------------------------- /docs-img/grafana-pr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmxdev/xenoeye/HEAD/docs-img/grafana-pr.png -------------------------------------------------------------------------------- /docs-img/grafana-tr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmxdev/xenoeye/HEAD/docs-img/grafana-tr.png -------------------------------------------------------------------------------- /docs-img/grafana-tr1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmxdev/xenoeye/HEAD/docs-img/grafana-tr1.png -------------------------------------------------------------------------------- /docs-img/grafana-tr2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmxdev/xenoeye/HEAD/docs-img/grafana-tr2.png -------------------------------------------------------------------------------- /docs-img/mpl-ts-ips.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmxdev/xenoeye/HEAD/docs-img/mpl-ts-ips.png -------------------------------------------------------------------------------- /docs-img/gnuplot-day-i.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmxdev/xenoeye/HEAD/docs-img/gnuplot-day-i.png -------------------------------------------------------------------------------- /docs-img/mpl-donut-as.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmxdev/xenoeye/HEAD/docs-img/mpl-donut-as.png -------------------------------------------------------------------------------- /docs-img/gnuplot-day-ip.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmxdev/xenoeye/HEAD/docs-img/gnuplot-day-ip.png -------------------------------------------------------------------------------- /docs-img/gnuplot-day-prot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmxdev/xenoeye/HEAD/docs-img/gnuplot-day-prot.png -------------------------------------------------------------------------------- /docs-img/grafana-pie-calc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmxdev/xenoeye/HEAD/docs-img/grafana-pie-calc.png -------------------------------------------------------------------------------- /docs-img/mpl-donut-prot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmxdev/xenoeye/HEAD/docs-img/mpl-donut-prot.png -------------------------------------------------------------------------------- /docs-img/pg-data-source.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmxdev/xenoeye/HEAD/docs-img/pg-data-source.png -------------------------------------------------------------------------------- /docs-img/grafana-class-http.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmxdev/xenoeye/HEAD/docs-img/grafana-class-http.png -------------------------------------------------------------------------------- /docs-img/grafana-class-port.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmxdev/xenoeye/HEAD/docs-img/grafana-class-port.png -------------------------------------------------------------------------------- /docs-img/grafana-class-size.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmxdev/xenoeye/HEAD/docs-img/grafana-class-size.png -------------------------------------------------------------------------------- /docs-img/grafana-geoip-city.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmxdev/xenoeye/HEAD/docs-img/grafana-geoip-city.png -------------------------------------------------------------------------------- /docs-img/grafana-geoip-country.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmxdev/xenoeye/HEAD/docs-img/grafana-geoip-country.png -------------------------------------------------------------------------------- /docs-img/grafana-geoip-state.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmxdev/xenoeye/HEAD/docs-img/grafana-geoip-state.png -------------------------------------------------------------------------------- /docs-img/v2502--grafana-sample.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmxdev/xenoeye/HEAD/docs-img/v2502--grafana-sample.png -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: c 2 | before_install: 3 | - sudo apt-get -y install libpcap-dev 4 | before_script: autoreconf -i 5 | script: ./configure && make && make check 6 | 7 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "tkvdb"] 2 | path = tkvdb 3 | url = https://github.com/vmxdev/tkvdb 4 | [submodule "aajson"] 5 | path = aajson 6 | url = https://github.com/vmxdev/aajson 7 | -------------------------------------------------------------------------------- /devices.conf: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "ip": "127.0.0.1", 4 | "id": 0, 5 | "sampling-rate": 1 6 | }, 7 | { 8 | "ip": "1.2.3.4", 9 | "sampling-rate": 1000 10 | } 11 | ] 12 | 13 | -------------------------------------------------------------------------------- /xe-debug.h: -------------------------------------------------------------------------------- 1 | #ifndef xe_debug_h_included 2 | #define xe_debug_h_included 3 | 4 | #include 5 | 6 | /* debug options */ 7 | struct xe_debug 8 | { 9 | int print_flows; 10 | int print_to_syslog; 11 | FILE *fout; 12 | }; 13 | 14 | #endif 15 | 16 | -------------------------------------------------------------------------------- /lxc/xenoeye.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=xenoeye 3 | After=network.target 4 | 5 | [Service] 6 | User=xe 7 | Group=xe 8 | Type=simple 9 | Restart=always 10 | ExecStart=/usr/local/bin/xenoeye 11 | StandardError=null 12 | 13 | [Install] 14 | WantedBy=multi-user.target 15 | -------------------------------------------------------------------------------- /lxc/mo/mynet-ingress/icmp/mo.conf: -------------------------------------------------------------------------------- 1 | { 2 | "filter": "proto 1", 3 | 4 | "fwm": [ 5 | { 6 | "time": 15, 7 | "name": "total", 8 | "fields": ["packets", "octets"] 9 | } 10 | , 11 | { 12 | "time": 15, 13 | "name": "daddr", 14 | "fields": ["packets desc", "octets desc", "dst host"], 15 | "limit": 20 16 | } 17 | ] 18 | } 19 | 20 | -------------------------------------------------------------------------------- /filter-ag.def: -------------------------------------------------------------------------------- 1 | /* NAME STR FLD SCALE */ 2 | FIELD(OCTETS, "octets", in_bytes, 1) 3 | FIELD(BITS, "bits", in_bytes, 8) 4 | FIELD(PACKETS, "packets", in_pkts, 1) 5 | FIELD(IOCTETS, "ioctets", ioctets, 1) 6 | FIELD(ROCTETS, "roctets", roctets, 1) 7 | FIELD(IPACKETS, "ipackets", ipackets, 1) 8 | FIELD(RPACKETS, "rpackets", rpackets, 1) 9 | #undef FIELD 10 | -------------------------------------------------------------------------------- /lxc/mo/mynet-ingress6/icmp/mo.conf: -------------------------------------------------------------------------------- 1 | { 2 | "filter": "proto 1", 3 | 4 | "fwm": [ 5 | { 6 | "time": 15, 7 | "name": "total", 8 | "fields": ["packets", "octets"] 9 | } 10 | , 11 | { 12 | "time": 15, 13 | "name": "daddr", 14 | "fields": ["packets desc", "octets desc", "dst host6"], 15 | "limit": 20 16 | } 17 | ] 18 | } 19 | 20 | -------------------------------------------------------------------------------- /scripts/fill-db.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | EXP_DIR="/var/lib/xenoeye/exp" 4 | FAIL_DIR="/var/lib/xenoeye/expfailed/" 5 | 6 | for sqlscript in $EXP_DIR/*.sql; do 7 | psql postgresql://xenoeye:password@localhost/xenoeyedb -f "$sqlscript" 8 | if [ $? -eq 0 ]; then 9 | rm -f "$sqlscript" 10 | else 11 | mv "$sqlscript" $FAIL_DIR 12 | fi 13 | done 14 | -------------------------------------------------------------------------------- /sflow.h: -------------------------------------------------------------------------------- 1 | #ifndef sflow_h_included 2 | #define sflow_h_included 3 | 4 | 5 | struct xe_data; 6 | struct flow_packet_info; 7 | struct flow_info; 8 | 9 | struct sfdata 10 | { 11 | struct xe_data *global; 12 | size_t thread_id; 13 | struct flow_packet_info *fpi; 14 | struct flow_info *flow; 15 | }; 16 | 17 | int 18 | sflow_process(struct xe_data *global, size_t thread_id, 19 | struct flow_packet_info *fpi, int len); 20 | 21 | 22 | #endif 23 | -------------------------------------------------------------------------------- /iplist.h: -------------------------------------------------------------------------------- 1 | #ifndef iplist_h_included 2 | #define iplist_h_included 3 | 4 | #include 5 | #include 6 | #include 7 | #include "utils.h" 8 | 9 | struct iplist; 10 | 11 | int iplists_load(const char *dirname); 12 | 13 | struct iplist *iplist_get_by_name(const char *name); 14 | 15 | int iplist_match4(struct iplist *l, uint32_t addr); 16 | int iplist_match6(struct iplist *l, xe_ip *addr); 17 | 18 | /* for dump */ 19 | char *iplist_name(struct iplist *l); 20 | 21 | #endif 22 | 23 | -------------------------------------------------------------------------------- /scripts/telegram-bot/on-start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | msgs_dir=/var/lib/xenoeye/telemsg/ 4 | 5 | args=("$@") 6 | 7 | filename_src=${args[3]} 8 | filename_dst=${msgs_dir}${filename_src##*/}".n" 9 | filename_dst_tmp=${filename_dst}".tmp" 10 | 11 | dt=$(date '+%F %T.%3N'); 12 | 13 | echo -n "Overlimit detected at '${dt}', object '${args[0]}', IP ${args[4]}, proto ${args[5]}, BPS $((${args[6]}*8)), limit $((${args[7]}*8)) BPS" > ${filename_dst_tmp} 14 | mv ${filename_dst_tmp} ${filename_dst} 15 | echo ${filename_dst} 16 | -------------------------------------------------------------------------------- /scripts/telegram-bot/on-stop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | msgs_dir=/var/lib/xenoeye/telemsg/ 4 | 5 | args=("$@") 6 | 7 | filename_src=${args[3]} 8 | filename_dst=${msgs_dir}${filename_src##*/}".s" 9 | filename_dst_tmp=${filename_dst}".stmp" 10 | 11 | dt=$(date '+%F %T.%3N'); 12 | 13 | echo -n "Overlimit is over at '${dt}', object '${args[0]}', IP ${args[4]}, proto ${args[5]}, BPS $((${args[6]}*8)), limit $((${args[7]}*8)) BPS" > ${filename_dst_tmp} 14 | mv ${filename_dst_tmp} ${filename_dst} 15 | echo "BACK-"${filename_dst} 16 | -------------------------------------------------------------------------------- /scripts/xe-dbexport-pg.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | EXP_DIR="/var/lib/xenoeye/exp" 4 | FAIL_DIR="/var/lib/xenoeye/expfailed/" 5 | 6 | FILES=`ls $EXP_DIR/*.sql` 7 | if [ -z "$FILES" ]; then 8 | echo "No files" 9 | exit 10 | fi 11 | 12 | TMPFILE=$(mktemp $EXP_DIR/tmp.XXXXXX) 13 | 14 | { echo "BEGIN;" ; cat $FILES; echo "COMMIT;"; } > "$TMPFILE" 15 | psql -v "ON_ERROR_STOP=1" postgresql://xenoeye:password@localhost/xenoeyedb -f "$TMPFILE" > /dev/null 2>/dev/null 16 | 17 | if [ $? -eq 0 ]; then 18 | rm -f $FILES 19 | else 20 | mv $FILES $FAIL_DIR 21 | fi 22 | rm -r "$TMPFILE" 23 | -------------------------------------------------------------------------------- /flow-debug.h: -------------------------------------------------------------------------------- 1 | #ifndef flow_debug_h_included 2 | #define flow_debug_h_included 3 | 4 | #include 5 | #include "aajson/aajson.h" 6 | 7 | #include "xenoeye.h" 8 | 9 | void flow_debug_init(void); 10 | 11 | int flow_debug_config(struct aajson *a, aajson_val *value, 12 | struct xe_debug *debug); 13 | 14 | void flow_debug_add_field(int flength, int ftype, uint8_t *fptr, 15 | char *debug_flow_str); 16 | 17 | void flow_print_str(struct xe_debug *debug, struct flow_info *fi, 18 | char *flow_str, int is_sflow); 19 | 20 | void sflow_debug_print(struct flow_info *fi, char *str); 21 | 22 | #endif 23 | 24 | -------------------------------------------------------------------------------- /lxc/mo/mynet-ingress/tcp/mo.conf: -------------------------------------------------------------------------------- 1 | { 2 | "filter": "proto 6", 3 | 4 | "fwm": [ 5 | { 6 | "time": 15, 7 | "name": "total", 8 | "fields": ["packets", "octets"] 9 | } 10 | , 11 | { 12 | "time": 15, 13 | "name": "daddr", 14 | "fields": ["packets desc", "octets desc", "dst host"], 15 | "limit": 20 16 | } 17 | , 18 | { 19 | "time": 15, 20 | "name": "sport", 21 | "fields": ["packets desc", "octets desc", "src port"], 22 | "limit": 20 23 | } 24 | , 25 | { 26 | "time": 15, 27 | "name": "dport", 28 | "fields": ["packets desc", "octets desc", "dst port"], 29 | "limit": 20 30 | } 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /lxc/mo/mynet-ingress/udp/mo.conf: -------------------------------------------------------------------------------- 1 | { 2 | "filter": "proto 17", 3 | 4 | "fwm": [ 5 | { 6 | "time": 15, 7 | "name": "total", 8 | "fields": ["packets", "octets"] 9 | } 10 | , 11 | { 12 | "time": 15, 13 | "name": "daddr", 14 | "fields": ["packets desc", "octets desc", "dst host"], 15 | "limit": 20 16 | } 17 | , 18 | { 19 | "time": 15, 20 | "name": "sport", 21 | "fields": ["packets desc", "octets desc", "src port"], 22 | "limit": 20 23 | } 24 | , 25 | { 26 | "time": 15, 27 | "name": "dport", 28 | "fields": ["packets desc", "octets desc", "dst port"], 29 | "limit": 20 30 | } 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /lxc/mo/mynet-ingress6/tcp/mo.conf: -------------------------------------------------------------------------------- 1 | { 2 | "filter": "proto 6", 3 | 4 | "fwm": [ 5 | { 6 | "time": 15, 7 | "name": "total", 8 | "fields": ["packets", "octets"] 9 | } 10 | , 11 | { 12 | "time": 15, 13 | "name": "daddr", 14 | "fields": ["packets desc", "octets desc", "dst host6"], 15 | "limit": 20 16 | } 17 | , 18 | { 19 | "time": 15, 20 | "name": "sport", 21 | "fields": ["packets desc", "octets desc", "src port"], 22 | "limit": 20 23 | } 24 | , 25 | { 26 | "time": 15, 27 | "name": "dport", 28 | "fields": ["packets desc", "octets desc", "dst port"], 29 | "limit": 20 30 | } 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /devices.h: -------------------------------------------------------------------------------- 1 | #ifndef devices_h_included 2 | #define devices_h_included 3 | 4 | #include 5 | #include "utils.h" 6 | 7 | struct flow_packet_info; 8 | 9 | struct device 10 | { 11 | int use_ip; 12 | int ip_ver; 13 | xe_ip ip; 14 | 15 | int use_id; 16 | uint32_t id; 17 | 18 | /* marks support */ 19 | size_t n_exprs; 20 | struct filter_expr **exprs; 21 | int mark; 22 | int skip_unmarked; 23 | 24 | int sampling_rate; 25 | }; 26 | 27 | int devices_load(const char *filename); 28 | 29 | int device_get_sampling_rate(struct device *d); 30 | int device_rules_check(struct flow_info *flow, struct flow_packet_info *fpi); 31 | 32 | #endif 33 | 34 | -------------------------------------------------------------------------------- /lxc/mo/mynet-ingress6/udp/mo.conf: -------------------------------------------------------------------------------- 1 | { 2 | "filter": "proto 17", 3 | 4 | "fwm": [ 5 | { 6 | "time": 15, 7 | "name": "total", 8 | "fields": ["packets", "octets"] 9 | } 10 | , 11 | { 12 | "time": 15, 13 | "name": "daddr", 14 | "fields": ["packets desc", "octets desc", "dst host6"], 15 | "limit": 20 16 | } 17 | , 18 | { 19 | "time": 15, 20 | "name": "sport", 21 | "fields": ["packets desc", "octets desc", "src port"], 22 | "limit": 20 23 | } 24 | , 25 | { 26 | "time": 15, 27 | "name": "dport", 28 | "fields": ["packets desc", "octets desc", "dst port"], 29 | "limit": 20 30 | } 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /.github/workflows/c.yml: -------------------------------------------------------------------------------- 1 | name: C/C++ CI 2 | 3 | on: 4 | push: 5 | branches: [ "master" ] 6 | pull_request: 7 | branches: [ "master" ] 8 | 9 | jobs: 10 | build: 11 | 12 | runs-on: ubuntu-latest 13 | 14 | steps: 15 | - name: Checkout 16 | uses: actions/checkout@v3 17 | with: 18 | submodules: 'true' 19 | 20 | - name: Libraries 21 | run: sudo apt-get -y install libpcap-dev 22 | 23 | - name: autoreconf 24 | run: autoreconf -i 25 | 26 | - name: configure 27 | run: ./configure 28 | 29 | - name: make 30 | run: make 31 | 32 | - name: make check 33 | run: make check 34 | -------------------------------------------------------------------------------- /lxc/mo/mynet-ingress/udp/dns-amp/mo.conf: -------------------------------------------------------------------------------- 1 | { 2 | "filter": "src port 53", 3 | 4 | "fwm": [ 5 | { 6 | "time": 15, 7 | "name": "total", 8 | "fields": ["packets", "octets"] 9 | } 10 | , 11 | { 12 | "time": 15, 13 | "name": "sport", 14 | "fields": ["packets desc", "octets desc", "src port"], 15 | "limit": 20 16 | } 17 | , 18 | { 19 | "time": 15, 20 | "name": "dport", 21 | "fields": ["packets desc", "octets desc", "dst port"], 22 | "limit": 20 23 | } 24 | , 25 | { 26 | "time": 15, 27 | "name": "daddr", 28 | "fields": ["packets desc", "octets desc", "dst host"], 29 | "limit": 20 30 | } 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /lxc/mo/mynet-ingress/udp/ntp-amp/mo.conf: -------------------------------------------------------------------------------- 1 | { 2 | "filter": "src port 123", 3 | 4 | "fwm": [ 5 | { 6 | "time": 15, 7 | "name": "total", 8 | "fields": ["packets", "octets"] 9 | } 10 | , 11 | { 12 | "time": 15, 13 | "name": "sport", 14 | "fields": ["packets desc", "octets desc", "src port"], 15 | "limit": 20 16 | } 17 | , 18 | { 19 | "time": 15, 20 | "name": "dport", 21 | "fields": ["packets desc", "octets desc", "dst port"], 22 | "limit": 20 23 | } 24 | , 25 | { 26 | "time": 15, 27 | "name": "daddr", 28 | "fields": ["packets desc", "octets desc", "dst host"], 29 | "limit": 20 30 | } 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /lxc/mo/mynet-ingress6/udp/dns-amp/mo.conf: -------------------------------------------------------------------------------- 1 | { 2 | "filter": "src port 53", 3 | 4 | "fwm": [ 5 | { 6 | "time": 15, 7 | "name": "total", 8 | "fields": ["packets", "octets"] 9 | } 10 | , 11 | { 12 | "time": 15, 13 | "name": "sport", 14 | "fields": ["packets desc", "octets desc", "src port"], 15 | "limit": 20 16 | } 17 | , 18 | { 19 | "time": 15, 20 | "name": "dport", 21 | "fields": ["packets desc", "octets desc", "dst port"], 22 | "limit": 20 23 | } 24 | , 25 | { 26 | "time": 15, 27 | "name": "daddr", 28 | "fields": ["packets desc", "octets desc", "dst host6"], 29 | "limit": 20 30 | } 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /lxc/mo/mynet-ingress6/udp/ntp-amp/mo.conf: -------------------------------------------------------------------------------- 1 | { 2 | "filter": "src port 123", 3 | 4 | "fwm": [ 5 | { 6 | "time": 15, 7 | "name": "total", 8 | "fields": ["packets", "octets"] 9 | } 10 | , 11 | { 12 | "time": 15, 13 | "name": "sport", 14 | "fields": ["packets desc", "octets desc", "src port"], 15 | "limit": 20 16 | } 17 | , 18 | { 19 | "time": 15, 20 | "name": "dport", 21 | "fields": ["packets desc", "octets desc", "dst port"], 22 | "limit": 20 23 | } 24 | , 25 | { 26 | "time": 15, 27 | "name": "daddr", 28 | "fields": ["packets desc", "octets desc", "dst host6"], 29 | "limit": 20 30 | } 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /lxc/mo/mynet-ingress/tcp/syn/mo.conf: -------------------------------------------------------------------------------- 1 | { 2 | "filter": "tfstr(tcp-flags) 'SYN'", 3 | 4 | "fwm": [ 5 | { 6 | "time": 15, 7 | "name": "total", 8 | "fields": ["packets", "octets"] 9 | } 10 | , 11 | { 12 | "time": 15, 13 | "name": "sport", 14 | "fields": ["packets desc", "octets desc", "src port"], 15 | "limit": 20 16 | } 17 | , 18 | { 19 | "time": 15, 20 | "name": "dport", 21 | "fields": ["packets desc", "octets desc", "dst port"], 22 | "limit": 20 23 | } 24 | , 25 | { 26 | "time": 15, 27 | "name": "daddr", 28 | "fields": ["packets desc", "octets desc", "dst host"], 29 | "limit": 20 30 | } 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /lxc/mo/mynet-ingress6/tcp/syn/mo.conf: -------------------------------------------------------------------------------- 1 | { 2 | "filter": "tfstr(tcp-flags) 'SYN'", 3 | 4 | "fwm": [ 5 | { 6 | "time": 15, 7 | "name": "total", 8 | "fields": ["packets", "octets"] 9 | } 10 | , 11 | { 12 | "time": 15, 13 | "name": "sport", 14 | "fields": ["packets desc", "octets desc", "src port"], 15 | "limit": 20 16 | } 17 | , 18 | { 19 | "time": 15, 20 | "name": "dport", 21 | "fields": ["packets desc", "octets desc", "dst port"], 22 | "limit": 20 23 | } 24 | , 25 | { 26 | "time": 15, 27 | "name": "daddr", 28 | "fields": ["packets desc", "octets desc", "dst host6"], 29 | "limit": 20 30 | } 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /lxc/mo/mynet-egress/mo.conf: -------------------------------------------------------------------------------- 1 | { 2 | "filter": "src net mynet", 3 | 4 | "debug": { 5 | "dump-flows": "none" 6 | }, 7 | 8 | "fwm": [ 9 | { 10 | "time": 15, 11 | "name": "total", 12 | "fields": ["packets", "octets"] 13 | } 14 | , 15 | { 16 | "time": 15, 17 | "name": "proto", 18 | "fields": ["packets desc", "octets desc", "proto"] 19 | } 20 | , 21 | { 22 | "time": 15, 23 | "name": "saddr", 24 | "fields": ["packets desc", "octets desc", "src host"], 25 | "limit": 20 26 | } 27 | , 28 | { 29 | "time": 15, 30 | "name": "daddr", 31 | "fields": ["packets desc", "octets desc", "dst host"], 32 | "limit": 20 33 | } 34 | ] 35 | } 36 | -------------------------------------------------------------------------------- /lxc/mo/mynet-egress6/mo.conf: -------------------------------------------------------------------------------- 1 | { 2 | "filter": "src net6 mynet", 3 | 4 | "debug": { 5 | "dump-flows": "none" 6 | }, 7 | 8 | "fwm": [ 9 | { 10 | "time": 15, 11 | "name": "total", 12 | "fields": ["packets", "octets"] 13 | } 14 | , 15 | { 16 | "time": 15, 17 | "name": "proto", 18 | "fields": ["packets desc", "octets desc", "proto"] 19 | } 20 | , 21 | { 22 | "time": 15, 23 | "name": "saddr", 24 | "fields": ["packets desc", "octets desc", "src host6"], 25 | "limit": 20 26 | } 27 | , 28 | { 29 | "time": 15, 30 | "name": "daddr", 31 | "fields": ["packets desc", "octets desc", "dst host6"], 32 | "limit": 20 33 | } 34 | ] 35 | } 36 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Prerequisites 2 | *.d 3 | 4 | # Object files 5 | *.o 6 | *.ko 7 | *.obj 8 | *.elf 9 | 10 | # Linker output 11 | *.ilk 12 | *.map 13 | *.exp 14 | 15 | # Precompiled Headers 16 | *.gch 17 | *.pch 18 | 19 | # Libraries 20 | *.lib 21 | *.a 22 | *.la 23 | *.lo 24 | 25 | # Shared objects (inc. Windows DLLs) 26 | *.dll 27 | *.so 28 | *.so.* 29 | *.dylib 30 | 31 | # Executables 32 | *.exe 33 | *.out 34 | *.app 35 | *.i*86 36 | *.x86_64 37 | *.hex 38 | xenoeye 39 | 40 | # Debug files 41 | *.dSYM/ 42 | *.su 43 | *.idb 44 | *.pdb 45 | 46 | # Kernel Module Compile Results 47 | *.mod* 48 | *.cmd 49 | .tmp_versions/ 50 | modules.order 51 | Module.symvers 52 | Mkfile.old 53 | dkms.conf 54 | -------------------------------------------------------------------------------- /scripts/xe-dbexport-ch.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | EXP_DIR="/var/lib/xenoeye/exp" 4 | FAIL_DIR="/var/lib/xenoeye/expfailed/" 5 | 6 | TMPLIST=$(mktemp $EXP_DIR/tmp.XXXXXX) 7 | TMPSQL=$(mktemp $EXP_DIR/tmp.XXXXXX) 8 | find "$EXP_DIR" -type f -name "*.sql" -print0 | while IFS= read -r -d '' file; do 9 | echo "$file" >> "$TMPLIST" 10 | cat "$file" >> "$TMPSQL" 11 | done 12 | 13 | clickhouse-client --multiquery --multiline --database xe < "$TMPSQL" 14 | 15 | if [ $? -eq 0 ]; then 16 | while IFS= read -r file ; do rm -f -- "$file" ; done < "$TMPLIST" 17 | else 18 | while IFS= read -r file ; do mv "$file" "$FAIL_DIR" ; done < "$TMPLIST" 19 | fi 20 | 21 | rm -f "$TMPSQL" 22 | rm -f "$TMPLIST" 23 | -------------------------------------------------------------------------------- /tests/test_filters.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "../filter.h" 5 | 6 | int 7 | main() 8 | { 9 | struct filter_input q; 10 | struct filter_expr *e; 11 | 12 | memset(&q, 0, sizeof(struct filter_input)); 13 | 14 | q.s = "Src host 1.2.3.4 and (PORT 12345 or 54321) or "\ 15 | "dst host 4.3.2.1 and div(octets, packets) 0-100"; 16 | e = parse_filter(&q); 17 | if (!e) { 18 | printf("Filter allocation failed\n"); 19 | return EXIT_FAILURE; 20 | } 21 | 22 | if (q.error) { 23 | printf("Parse error: %s\n", q.errmsg); 24 | return EXIT_FAILURE; 25 | } 26 | 27 | filter_dump(e, stdout); 28 | 29 | filter_free(e); 30 | 31 | return EXIT_SUCCESS; 32 | } 33 | 34 | -------------------------------------------------------------------------------- /monit-objects-common.h: -------------------------------------------------------------------------------- 1 | #ifndef monit_object_common_h_included 2 | #define monit_object_common_h_included 3 | 4 | #include "filter.h" 5 | 6 | /* FIXME: remove this function? */ 7 | static inline uint64_t 8 | monit_object_nf_val(struct flow_info *flow, struct field *fld) 9 | { 10 | uintptr_t flow_fld = (uintptr_t)flow + fld->nf_offset; 11 | return get_nf_val(flow_fld, fld->size); 12 | } 13 | 14 | #define MAVG_LIM_CURR(MAVG) &(MAVG->lim[ \ 15 | atomic_load_explicit(&MAVG->lim_curr_idx, memory_order_relaxed) % 2]) 16 | 17 | #define MAVG_LIM_NOT_CURR(MAVG) &(MAVG->lim[ \ 18 | (atomic_load_explicit(&MAVG->lim_curr_idx, memory_order_relaxed) + 1) \ 19 | % 2]) 20 | 21 | #endif 22 | 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2018-2023, Vladimir Misyurov, Michael Kogan 2 | 3 | Permission to use, copy, modify, and/or distribute this software for any 4 | purpose with or without fee is hereby granted, provided that the above 5 | copyright notice and this permission notice appear in all copies. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 8 | REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 9 | AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 10 | INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 11 | LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE 12 | OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 13 | PERFORMANCE OF THIS SOFTWARE. 14 | 15 | -------------------------------------------------------------------------------- /netflow-templates.h: -------------------------------------------------------------------------------- 1 | #ifndef netflow_templates_h_included 2 | #define netflow_templates_h_included 3 | 4 | #include "netflow.h" 5 | 6 | /* 7 | * template key: source IP address version (4 or 6), Netflow version, 8 | * template ID, source IP, source ID and time 9 | */ 10 | struct template_key 11 | { 12 | uint8_t src_ip_version; 13 | uint8_t nf_version; 14 | uint16_t template_id; 15 | 16 | xe_ip source_ip; 17 | 18 | uint32_t source_id; 19 | uint32_t epoch; 20 | } __attribute__ ((__packed__)); 21 | 22 | int netflow_templates_init(struct xe_data *data); 23 | void netflow_templates_shutdown(void); 24 | 25 | void *netflow_template_find(struct template_key *tkey, 26 | int allow_templates_in_future); 27 | 28 | int netflow_template_add(struct template_key *tkey, void *t, size_t size); 29 | 30 | #endif 31 | 32 | -------------------------------------------------------------------------------- /lxc/mo/mynet-ingress/frag/mo.conf: -------------------------------------------------------------------------------- 1 | { 2 | /*"filter": "not frag-id 0",*/ 3 | "filter": "(proto 6 or 17) and (src port 0 or dst port 0)", 4 | 5 | "fwm": [ 6 | { 7 | "time": 15, 8 | "name": "total", 9 | "fields": ["packets", "octets"] 10 | } 11 | , 12 | { 13 | "time": 15, 14 | "name": "daddr", 15 | "fields": ["packets desc", "octets desc", "dst host"], 16 | "limit": 20 17 | } 18 | ] 19 | 20 | , 21 | "mavg": [ 22 | { 23 | "name": "pps", 24 | "time": 100, 25 | "dump": 10, 26 | "fields": ["dst host", "packets"], 27 | "overlimit": [ 28 | { 29 | "name": "trigger", 30 | "default": [400000], 31 | "action-script": "/var/lib//xenoeye/scripts/frag.sh", 32 | "back2norm-script": "/var/lib/xenoeye/scripts/frag-back.sh", 33 | "back2norm-time": 120 34 | } 35 | ] 36 | } 37 | ] 38 | } 39 | -------------------------------------------------------------------------------- /lxc/mo/mynet-ingress6/frag/mo.conf: -------------------------------------------------------------------------------- 1 | { 2 | /*"filter": "not frag-id 0",*/ 3 | "filter": "(proto 6 or 17) and (src port 0 or dst port 0)", 4 | 5 | "fwm": [ 6 | { 7 | "time": 15, 8 | "name": "total", 9 | "fields": ["packets", "octets"] 10 | } 11 | , 12 | { 13 | "time": 15, 14 | "name": "daddr", 15 | "fields": ["packets desc", "octets desc", "dst host6"], 16 | "limit": 20 17 | } 18 | ] 19 | 20 | , 21 | "mavg": [ 22 | { 23 | "name": "pps", 24 | "time": 100, 25 | "dump": 10, 26 | "fields": ["dst host6", "packets"], 27 | "overlimit": [ 28 | { 29 | "name": "trigger", 30 | "default": [400000], 31 | "action-script": "/var/lib//xenoeye/scripts/frag.sh", 32 | "back2norm-script": "/var/lib/xenoeye/scripts/frag-back.sh", 33 | "back2norm-time": 120 34 | } 35 | ] 36 | } 37 | ] 38 | } 39 | -------------------------------------------------------------------------------- /xenoeye.conf: -------------------------------------------------------------------------------- 1 | { 2 | "capture": [ 3 | //{"pcap" : {"interface": "eth0", "filter": "udp and port 2055"}}, 4 | {"socket": {"listen-on": "*", "port": "2055"}} 5 | ], 6 | 7 | "sflow-capture": [ 8 | //{"pcap": {"interface": "eth0", "filter": "udp and port 6343"}}, 9 | {"socket": {"listen-on": "*", "port": "6343"}} 10 | ], 11 | 12 | "templates": { 13 | "db": "/var/lib/xenoeye/templates.tkvdb" 14 | }, 15 | 16 | "debug": { 17 | /* allowed values: "none", "syslog", "/path/to/file.txt" */ 18 | "dump-flows": "none" 19 | }, 20 | 21 | "devices": "/etc/xenoeye/devices.conf", 22 | 23 | "mo-dir": "/var/lib/xenoeye/mo", 24 | 25 | "db-export": "/var/lib/xenoeye/scripts/xe-dbexport-pg.sh" 26 | /* for Clickhouse: 27 | "db-type": "ch", 28 | "db-export": "/var/lib/xenoeye/scripts/xe-dbexport-ch.sh", 29 | "ch-codec": "ZSTD(20)" // optional compression codec and level 30 | */ 31 | } 32 | 33 | -------------------------------------------------------------------------------- /flow-info.h: -------------------------------------------------------------------------------- 1 | #ifndef flow_info_h_included 2 | #define flow_info_h_included 3 | 4 | #include 5 | #include 6 | 7 | #define MAX_NF_PACKET_SIZE (64*1024) 8 | #define CLASS_NAME_MAX 32 9 | 10 | struct flow_info 11 | { 12 | #define FIELD(NAME, DESC, FLDTYPE, FLDID, SIZEMIN, SIZEMAX) \ 13 | uint8_t NAME[SIZEMAX]; \ 14 | int NAME##_size; \ 15 | int has_##NAME; 16 | #include "netflow.def" 17 | void *payload_ptr; 18 | 19 | /* virtual fields for export devices */ 20 | uint8_t dev_ip6[16]; 21 | int dev_ip6_size; 22 | int has_dev_ip6; 23 | 24 | uint8_t dev_id[4]; 25 | int dev_id_size; 26 | int has_dev_id; 27 | 28 | uint8_t dev_mark[4]; 29 | int dev_mark_size; 30 | int has_dev_mark; 31 | 32 | uint32_t sampling_rate; 33 | }; 34 | 35 | struct flow_packet_info 36 | { 37 | struct sockaddr src_addr; 38 | uint32_t src_addr_ipv4; 39 | 40 | uint32_t source_id; 41 | uint32_t epoch; 42 | uint64_t time_ns; /* nanoseconds */ 43 | uint8_t rawpacket[MAX_NF_PACKET_SIZE]; 44 | 45 | int sampling_rate; 46 | }; 47 | 48 | 49 | #endif 50 | 51 | -------------------------------------------------------------------------------- /lxc/mo/mynet-ingress/mo.conf: -------------------------------------------------------------------------------- 1 | { 2 | "filter": "dst net mynet", 3 | 4 | "debug": { 5 | "dump-flows": "none" 6 | }, 7 | 8 | "fwm": [ 9 | { 10 | "time": 15, 11 | "name": "total", 12 | "fields": ["packets", "octets"] 13 | } 14 | , 15 | { 16 | "time": 15, 17 | "name": "proto", 18 | "fields": ["packets desc", "octets desc", "proto"] 19 | } 20 | , 21 | { 22 | "time": 15, 23 | "name": "saddr", 24 | "fields": ["packets desc", "octets desc", "src host"], 25 | "limit": 20 26 | } 27 | , 28 | { 29 | "time": 15, 30 | "name": "daddr", 31 | "fields": ["packets desc", "octets desc", "dst host"], 32 | "limit": 20 33 | } 34 | , 35 | { 36 | "time": 15, 37 | "name": "country", 38 | "fields": ["packets desc", "octets desc", "country(src host)"], 39 | "limit": 20 40 | } 41 | , 42 | { 43 | "time": 15, 44 | "name": "as", 45 | "fields": ["packets desc", "octets desc", "asn(src host)", "asd(src host)"], 46 | "limit": 20 47 | } 48 | , 49 | { 50 | "time": 15, 51 | "name": "router", 52 | "fields": ["packets desc", "octets", "dev-ip"] 53 | } 54 | , 55 | { 56 | "time": 15, 57 | "name": "routerp", 58 | "fields": ["packets desc", "octets", "dev-ip", "src ifidx", "dst ifidx"], 59 | "limit": 20 60 | } 61 | ] 62 | } 63 | 64 | -------------------------------------------------------------------------------- /lxc/mo/mynet-ingress6/mo.conf: -------------------------------------------------------------------------------- 1 | { 2 | "filter": "dst net6 mynet", 3 | 4 | "debug": { 5 | "dump-flows": "none" 6 | }, 7 | 8 | "fwm": [ 9 | { 10 | "time": 15, 11 | "name": "total", 12 | "fields": ["packets", "octets"] 13 | } 14 | , 15 | { 16 | "time": 15, 17 | "name": "proto", 18 | "fields": ["packets desc", "octets desc", "proto"] 19 | } 20 | , 21 | { 22 | "time": 15, 23 | "name": "saddr", 24 | "fields": ["packets desc", "octets desc", "src host6"], 25 | "limit": 20 26 | } 27 | , 28 | { 29 | "time": 15, 30 | "name": "daddr", 31 | "fields": ["packets desc", "octets desc", "dst host6"], 32 | "limit": 20 33 | } 34 | , 35 | { 36 | "time": 15, 37 | "name": "country", 38 | "fields": ["packets desc", "octets desc", "country(src host6)"], 39 | "limit": 20 40 | } 41 | , 42 | { 43 | "time": 15, 44 | "name": "as", 45 | "fields": ["packets desc", "octets desc", "asn(src host6)", "asd(src host6)"], 46 | "limit": 20 47 | } 48 | , 49 | { 50 | "time": 15, 51 | "name": "router", 52 | "fields": ["packets desc", "octets", "dev-ip"] 53 | } 54 | , 55 | { 56 | "time": 15, 57 | "name": "routerp", 58 | "fields": ["packets desc", "octets", "dev-ip", "src ifidx", "dst ifidx"], 59 | "limit": 20 60 | } 61 | ] 62 | } 63 | 64 | -------------------------------------------------------------------------------- /scripts/mkchart-gnuplot.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "SELECT time, sum(octets)/30*8 AS ip, ips FROM 4 | ( 5 | WITH topips AS 6 | (SELECT sum(octets) AS ip, COALESCE (src_host::text, 'Other') as ips FROM ingress_bytes_by_src WHERE time >= now() - interval '1 day' GROUP BY ips ORDER BY ip desc limit 20) 7 | SELECT time, octets, COALESCE (src_host::text, 'Other') as ips FROM ingress_bytes_by_src WHERE time >= now() - interval '1 day' AND src_host::text IN (SELECT ips from topips) 8 | UNION 9 | SELECT time, octets, 'Other' as ips FROM ingress_bytes_by_src WHERE time >= now() - interval '1 day' AND src_host::text NOT IN (SELECT ips from topips) 10 | UNION 11 | SELECT time, octets, 'Other' as ips FROM ingress_bytes_by_src WHERE time >= now() - interval '1 day' AND src_host IS NULL 12 | ) AS report 13 | GROUP BY time, ips ORDER BY time \\crosstabview time ips ip" | psql postgresql://xenoeye:password@localhost/xenoeyedb > day-i-ip.csv 14 | 15 | echo "set terminal png size 1000,400 16 | set output 'day-i-ip.png' 17 | set key autotitle columnhead 18 | set xdata time 19 | set timefmt '%Y-%m-%d %H:%M:%S' 20 | set format y '%.02s%cB' 21 | set xtics rotate 22 | set datafile separator '|' 23 | set key outside 24 | plot 'day-i-ip.csv' using 1:3 with lines, for [i=4:21] '' using 1:i with lines" | gnuplot 25 | -------------------------------------------------------------------------------- /scripts/mkchart-matplotlib-ts.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib 3 | matplotlib.use('agg') 4 | import matplotlib.pyplot as plt 5 | from matplotlib import dates 6 | import psycopg2 7 | import pandas as pd 8 | 9 | INTERVAL = '24 hours' 10 | CONNSTR = 'postgresql://xenoeye:password@localhost/xenoeyedb' 11 | 12 | date_time = [] 13 | octets = [] 14 | 15 | # Load dataset 16 | query = """ 17 | SELECT time, octets/30*8 FROM ingress_all 18 | WHERE time >= now() - interval '{interval}' 19 | """.format(interval=INTERVAL) 20 | 21 | conn = psycopg2.connect(CONNSTR) 22 | cursor = conn.cursor() 23 | cursor.execute(query) 24 | records = cursor.fetchall() 25 | for record in records: 26 | date_time.append(record[0]) 27 | octets.append(int(record[1])) 28 | 29 | cursor.close() 30 | conn.close() 31 | 32 | # Build chart 33 | plt.figure(figsize = (12, 8)) 34 | 35 | # Calculate bars width 36 | wd = np.min(np.diff(dates.date2num(date_time))) * 1.1 37 | plt.bar(date_time, octets, width = wd) 38 | 39 | plt.xticks(rotation=90) 40 | 41 | mkfunc = lambda x, pos: '%1.1fGb' % (x * 1e-9) if x >= 1e9 else '%1.1fMb' % (x * 1e-6) if x >= 1e6 else '%1.1fKb' % (x * 1e-3) if x >= 1e3 else '%1.1f' % x 42 | mkformatter = matplotlib.ticker.FuncFormatter(mkfunc) 43 | plt.gca().yaxis.set_major_formatter(mkformatter) 44 | 45 | plt.tight_layout() 46 | 47 | plt.savefig('mpl-day-ts.png') 48 | -------------------------------------------------------------------------------- /configure.ac: -------------------------------------------------------------------------------- 1 | AC_INIT([xenoeye], [23.11], [vm@xenoeye.com]) 2 | AC_CONFIG_MACRO_DIR([m4]) 3 | AM_INIT_AUTOMAKE([check-news foreign silent-rules subdir-objects -Wall]) 4 | 5 | AC_PROG_MKDIR_P 6 | 7 | AM_MAINTAINER_MODE 8 | 9 | AM_PROG_AR 10 | AC_PROG_CC 11 | AM_PROG_CC_C_O 12 | AC_CONFIG_HEADERS([config.h]) 13 | LT_INIT 14 | 15 | AC_MSG_CHECKING(whether compiler understands -Wall -Wextra -pedantic) 16 | old_CFLAGS="$CFLAGS" 17 | CFLAGS="$CFLAGS -Wall -Wextra -pedantic" 18 | AC_COMPILE_IFELSE([AC_LANG_PROGRAM([],[])], 19 | AC_MSG_RESULT(yes), 20 | AC_MSG_RESULT(no) 21 | CFLAGS="$old_CFLAGS") 22 | 23 | AC_SEARCH_LIBS([pcap_open_live], [pcap], [], [ 24 | AC_MSG_ERROR([unable to find the pcap_open_live() function]) 25 | ]) 26 | 27 | AC_SEARCH_LIBS([__atomic_load_16], [atomic], [], [ 28 | AC_MSG_ERROR([unable to find the __atomic_load_16() function]) 29 | ]) 30 | 31 | AC_SEARCH_LIBS([log], [m], [], [ 32 | AC_MSG_ERROR([unable to find the log() function]) 33 | ]) 34 | 35 | #musl don't have a getprotobynumber_r function 36 | AC_CHECK_FUNC([getprotobynumber_r], [AC_DEFINE([HAVE_GETPROTOBYNUMBER_R], [1], 37 | [Define if getprotobynumber_r exists.])]) 38 | 39 | # pthreads 40 | 41 | AX_PTHREAD([ 42 | LIBS="$PTHREAD_LIBS $LIBS" 43 | CFLAGS="$CFLAGS $PTHREAD_CFLAGS" 44 | CC="$PTHREAD_CC"], AC_MSG_ERROR([Missing POSIX threads support])) 45 | 46 | AM_SILENT_RULES([yes]) 47 | 48 | AC_CONFIG_FILES([Makefile]) 49 | AC_CONFIG_MACRO_DIRS([m4]) 50 | AC_OUTPUT 51 | -------------------------------------------------------------------------------- /scripts/mkchart-prot-mpl.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib 3 | matplotlib.use('agg') 4 | import matplotlib.pyplot as plt 5 | import psycopg2 6 | 7 | INTERVAL = '5 minutes' 8 | CONNSTR = 'postgresql://xenoeye:password@localhost/xenoeyedb' 9 | PTHRESHOLD = 4.0 10 | 11 | protocols = [] 12 | octets = [] 13 | explode = [] 14 | protolabels = [] 15 | 16 | # Load dataset 17 | query = """ 18 | select iana_protocols.name, sum(octets) as oct 19 | from ingress_proto 20 | join iana_protocols on ingress_proto.proto=iana_protocols.num 21 | where time >= now() - interval '{}' 22 | group by iana_protocols.name 23 | order by oct desc 24 | """.format(INTERVAL) 25 | 26 | conn = psycopg2.connect(CONNSTR) 27 | cursor = conn.cursor() 28 | cursor.execute(query) 29 | records = cursor.fetchall() 30 | for record in records: 31 | protocols.append(record[0]) 32 | octets.append(record[1]) 33 | explode.append(0.05) 34 | 35 | cursor.close() 36 | conn.close() 37 | 38 | # prepare labels 39 | sm = sum(octets) 40 | for i in range(len(protocols)): 41 | proc = 100 * octets[i] / sm 42 | protolabels.append('{} - {:.2f}%'.format(protocols[i], 100 * octets[i] / sm)) 43 | # don't display protocol name when % less than threshold 44 | if proc < PTHRESHOLD: 45 | protocols[i] = '' 46 | 47 | # plot 48 | 49 | # pie chart 50 | plt.pie(octets, 51 | labels=protocols, 52 | autopct=lambda p: format(p, '.2f')+'%' if p > 4 else None, 53 | pctdistance=0.7, 54 | explode=explode) 55 | 56 | # draw circle 57 | centre_circle = plt.Circle((0, 0), 0.50, fc='white') 58 | fig = plt.gcf() 59 | 60 | # adding circle in pie chart 61 | fig.gca().add_artist(centre_circle) 62 | 63 | # adding title of chart 64 | plt.title('IP protocols for the last {}'.format(INTERVAL)) 65 | 66 | # add legends 67 | plt.legend(protolabels, loc='center left', bbox_to_anchor=(1, 0.5), title='IP protocols') 68 | 69 | plt.tight_layout() 70 | 71 | plt.savefig('mpl-day-i.png') 72 | -------------------------------------------------------------------------------- /geoip.h: -------------------------------------------------------------------------------- 1 | #ifndef geoip_h_included 2 | #define geoip_h_included 3 | 4 | #include 5 | #include "utils.h" 6 | 7 | #define DEFAULT_GEODB_DIR "/var/lib/xenoeye/geoip/" 8 | 9 | 10 | /* string sizes taken from files geolocationDatabaseIPv4.csv and 11 | * geolocationDatabaseIPv6.csv */ 12 | 13 | #define FOR_LIST_OF_GEOIP_FIELDS \ 14 | DO(CONTINENT, 3) \ 15 | DO(COUNTRY_CODE, 3) \ 16 | DO(COUNTRY, 35) \ 17 | DO(STATE, 64) \ 18 | DO(CITY, 51) \ 19 | DO(ZIP, 15) \ 20 | DO(LAT, 20) \ 21 | DO(LONG, 23) 22 | 23 | enum GEOIP_FIELD 24 | { 25 | #define DO(FIELD, SIZE) GEOIP_##FIELD, 26 | FOR_LIST_OF_GEOIP_FIELDS 27 | #undef DO 28 | }; 29 | 30 | struct geoip_info 31 | { 32 | #define DO(FIELD, SIZE) char FIELD[SIZE]; 33 | FOR_LIST_OF_GEOIP_FIELDS 34 | #undef DO 35 | }; 36 | 37 | struct as_info 38 | { 39 | uint32_t asn; 40 | char asd[200]; 41 | }; 42 | 43 | struct btrie_node_geo 44 | { 45 | uint32_t next[2]; 46 | int is_leaf; 47 | struct geoip_info g; 48 | }; 49 | 50 | struct btrie_node_as 51 | { 52 | uint32_t next[2]; 53 | int is_leaf; 54 | struct as_info a; 55 | }; 56 | 57 | 58 | int geoip_lookup4(uint32_t addr, struct geoip_info **g); 59 | int geoip_lookup6(xe_ip *addr, struct geoip_info **g); 60 | 61 | int as_lookup4(uint32_t addr, struct as_info **a); 62 | int as_lookup6(xe_ip *addr, struct as_info **a); 63 | 64 | void *geoip_thread(void *arg); 65 | 66 | static inline 67 | char *geoip_get_field(struct geoip_info *g, enum GEOIP_FIELD f) 68 | { 69 | switch (f) { 70 | #define DO(FIELD, SIZE) case GEOIP_##FIELD: return g->FIELD; break; 71 | FOR_LIST_OF_GEOIP_FIELDS 72 | #undef DO 73 | } 74 | return NULL; 75 | } 76 | 77 | static inline 78 | int geoip_get_field_size(enum GEOIP_FIELD f) 79 | { 80 | switch (f) { 81 | #define DO(FIELD, SIZE) case GEOIP_##FIELD: return SIZE; break; 82 | FOR_LIST_OF_GEOIP_FIELDS 83 | #undef DO 84 | } 85 | return 0; 86 | } 87 | 88 | #endif 89 | 90 | -------------------------------------------------------------------------------- /scripts/mkchart-matplotlib-donut-prot.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib 3 | matplotlib.use('agg') 4 | import matplotlib.pyplot as plt 5 | import psycopg2 6 | 7 | INTERVAL = '5 minutes' 8 | CONNSTR = 'postgresql://xenoeye:password@localhost/xenoeyedb' 9 | PTHRESHOLD = 4.0 10 | 11 | protocols = [] 12 | octets = [] 13 | explode = [] 14 | protolabels = [] 15 | 16 | # Load dataset 17 | query = """ 18 | select iana_protocols.name, sum(octets) as oct 19 | from ingress_proto 20 | join iana_protocols on ingress_proto.proto=iana_protocols.num 21 | where time >= now() - interval '{}' 22 | group by iana_protocols.name 23 | order by oct desc 24 | """.format(INTERVAL) 25 | 26 | conn = psycopg2.connect(CONNSTR) 27 | cursor = conn.cursor() 28 | cursor.execute(query) 29 | records = cursor.fetchall() 30 | for record in records: 31 | protocols.append(record[0]) 32 | octets.append(record[1]) 33 | explode.append(0.05) 34 | 35 | cursor.close() 36 | conn.close() 37 | 38 | # prepare labels 39 | sm = sum(octets) 40 | for i in range(len(protocols)): 41 | proc = 100 * octets[i] / sm 42 | protolabels.append('{} - {:.2f}%'.format(protocols[i], 100 * octets[i] / sm)) 43 | # don't display protocol name when % less than threshold 44 | if proc < PTHRESHOLD: 45 | protocols[i] = '' 46 | 47 | # plot 48 | 49 | # Pie Chart 50 | plt.pie(octets, 51 | labels=protocols, 52 | autopct=lambda p: format(p, '.2f')+'%' if p > 4 else None, 53 | pctdistance=0.7, 54 | explode=explode) 55 | 56 | # draw circle 57 | centre_circle = plt.Circle((0, 0), 0.50, fc='white') 58 | fig = plt.gcf() 59 | 60 | # Adding Circle in Pie chart 61 | fig.gca().add_artist(centre_circle) 62 | 63 | # Adding Title of chart 64 | plt.title('IP protocols for the last {}'.format(INTERVAL)) 65 | 66 | # Add Legends 67 | plt.legend(protolabels, title='IP protocols', bbox_to_anchor=(1, 1), loc='upper left') 68 | 69 | plt.tight_layout() 70 | 71 | plt.savefig('mpl-donut.png') 72 | 73 | -------------------------------------------------------------------------------- /filter.def: -------------------------------------------------------------------------------- 1 | /* NAME STR TYPE SRC DST */ 2 | FIELD(HOST, "host", ADDR4, ip4_src_addr, ip4_dst_addr) 3 | FIELD(NET, "net", ADDR4, ip4_src_addr, ip4_dst_addr) 4 | FIELD(HOST6, "host6", ADDR6, ip6_src_addr, ip6_dst_addr) 5 | FIELD(NET6, "net6", ADDR6, ip6_src_addr, ip6_dst_addr) 6 | FIELD(PORT, "port", RANGE, l4_src_port, l4_dst_port) 7 | FIELD(PROTO, "proto", RANGE, protocol, protocol) 8 | FIELD(TOS, "tos", RANGE, src_tos, dst_tos) 9 | FIELD(TCPFLAGS, "tcp-flags", RANGE, tcp_flags, tcp_flags) 10 | FIELD(IFIDX, "ifidx", RANGE, input_snmp, output_snmp) 11 | FIELD(AS, "as", RANGE, src_as, dst_as) 12 | FIELD(MIN_TTL, "min-ttl", RANGE, min_ttl, min_ttl) 13 | FIELD(MAX_TTL, "max-ttl", RANGE, max_ttl, max_ttl) 14 | FIELD(FRAG_ID, "frag-id", RANGE, frag_id, frag_id) 15 | FIELD(VLAN, "vlan", RANGE, src_vlan, dst_vlan) 16 | FIELD(END_REASON,"endreason", RANGE, flow_end_reason,flow_end_reason) 17 | FIELD(D1Q_VLAN, "d1qvlan", RANGE, dot1q_vlan, dot1q_vlan) 18 | FIELD(D1Q_CVLAN,"d1qcvlan", RANGE, dot1q_cvlan, dot1q_cvlan) 19 | FIELD(BGP_NEXT_HOP,"bgp-nh", ADDR4, bgp_next_hop, bgp_next_hop) 20 | FIELD(DIRECTION,"dir", RANGE, direction, direction) 21 | FIELD(FWD_STATUS,"fwdst", RANGE, fwd_status, fwd_status) 22 | FIELD(SAMPLER_ID,"smplr", RANGE, sampler_id, sampler_id) 23 | FIELD(INGRS_VRF,"vrf", RANGE, ingrs_vrf, egrs_vrf) 24 | FIELD(DEV_IP, "dev-ip", ADDR4, dev_ip, dev_ip) 25 | FIELD(DEV_IP6, "dev-ip6", ADDR4, dev_ip6, dev_ip6) 26 | FIELD(DEV_ID, "dev-id", RANGE, dev_id, dev_id) 27 | FIELD(DEV_MARK, "dev-mark", RANGE, dev_mark, dev_mark) 28 | FIELD(CLASS0, "class0", STRING, class0, class0) 29 | FIELD(CLASS1, "class1", STRING, class1, class1) 30 | FIELD(CLASS2, "class2", STRING, class2, class2) 31 | FIELD(CLASS3, "class3", STRING, class3, class3) 32 | FIELD(CLASS4, "class4", STRING, class4, class4) 33 | FIELD(DNS_NAME, "dns-name", STRING, dns_name, dns_name) 34 | FIELD(DNS_IPS, "dns-ips", STRING, dns_ips, dns_ips) 35 | FIELD(SNI, "sni", STRING, sni, sni) 36 | #undef FIELD 37 | -------------------------------------------------------------------------------- /scripts/mkchart-matplotlib-donut-as.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib 3 | matplotlib.use('agg') 4 | import matplotlib.pyplot as plt 5 | import psycopg2 6 | 7 | INTERVAL = '30 minutes' 8 | CONNSTR = 'postgresql://xenoeye:password@localhost/xenoeyedb' 9 | PTHRESHOLD = 2.0 10 | TOP=15 11 | 12 | ases = [] 13 | octets = [] 14 | explode = [] 15 | aslabels = [] 16 | 17 | # Load dataset 18 | query = """ 19 | with top_ases as 20 | (select src_as::text, sum(octets) as oct 21 | from ingress_bytes_by_src_as 22 | where time >= now() - interval '{interval}' 23 | group by src_as 24 | order by oct desc 25 | limit {top}) 26 | select * 27 | from top_ases 28 | union all 29 | select 'Others' as src_as, 30 | sum(octets) as oct 31 | from ingress_bytes_by_src_as 32 | where src_as::text not in 33 | (select src_as 34 | from top_ases) 35 | """.format(interval=INTERVAL, top=TOP) 36 | 37 | conn = psycopg2.connect(CONNSTR) 38 | cursor = conn.cursor() 39 | cursor.execute(query) 40 | records = cursor.fetchall() 41 | for record in records: 42 | ases.append(record[0]) 43 | octets.append(record[1]) 44 | explode.append(0.01) 45 | 46 | cursor.close() 47 | conn.close() 48 | 49 | # prepare labels 50 | sm = sum(octets) 51 | for i in range(len(ases)): 52 | proc = 100 * octets[i] / sm 53 | aslabels.append('{} - {:.2f}%'.format(ases[i], 100 * octets[i] / sm)) 54 | # don't display AS when % less than threshold 55 | if proc < PTHRESHOLD: 56 | ases[i] = '' 57 | 58 | # plot 59 | 60 | # Pie Chart 61 | plt.pie(octets, 62 | labels=ases, 63 | autopct=lambda p: format(p, '.2f')+'%' if p > 4 else None, 64 | pctdistance=0.7, 65 | explode=explode) 66 | 67 | # draw circle 68 | centre_circle = plt.Circle((0, 0), 0.50, fc='white') 69 | fig = plt.gcf() 70 | 71 | # Adding Circle in Pie chart 72 | fig.gca().add_artist(centre_circle) 73 | 74 | # Adding Title of chart 75 | plt.title('Source Autonomous Systems for the last {}'.format(INTERVAL)) 76 | 77 | # Add Legends 78 | plt.legend(aslabels, title='AS numbers', bbox_to_anchor=(1, 1), loc='upper left') 79 | 80 | plt.tight_layout() 81 | 82 | plt.savefig('mpl-donut-as.png') 83 | 84 | -------------------------------------------------------------------------------- /utils.c: -------------------------------------------------------------------------------- 1 | /* 2 | * xenoeye 3 | * 4 | * Copyright (c) 2024, Vladimir Misyurov 5 | * 6 | * Permission to use, copy, modify, and/or distribute this software for any 7 | * purpose with or without fee is hereby granted, provided that the above 8 | * copyright notice and this permission notice appear in all copies. 9 | * 10 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 11 | * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 12 | * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 13 | * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 14 | * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE 15 | * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 16 | * PERFORMANCE OF THIS SOFTWARE. 17 | */ 18 | 19 | #include "utils.h" 20 | #include "utils-data.inc" 21 | 22 | char * 23 | tcp_flags_to_str(uint8_t tf) 24 | { 25 | return tcp_flags_db[tf]; 26 | } 27 | 28 | void 29 | port_to_str(char *res, uint16_t port) 30 | { 31 | char *s = ports_db[port]; 32 | if (s[0]) { 33 | sprintf(res, "%s (%d)", s, port); 34 | return; 35 | } 36 | sprintf(res, "%d", port); 37 | } 38 | 39 | void 40 | ports_pair_to_str(char *res, uint16_t port1, uint16_t port2) 41 | { 42 | char *s1 = ports_db[port1]; 43 | char *s2 = ports_db[port2]; 44 | 45 | if (s1[0] && s2[0]) { 46 | sprintf(res, "%s(%d) -> %s(%d)", s1, port1, s2, port2); 47 | return; 48 | } 49 | 50 | if (port1 == port2) { 51 | sprintf(res, "%d -> %d", port1, port2); 52 | return; 53 | } 54 | 55 | if (port1 < port2) { 56 | if (!s1[0] && !s2[0]) { 57 | sprintf(res, "%d ->", port1); 58 | return; 59 | } else if (!s1[0] && s2[0]) { 60 | sprintf(res, "%d -> %s(%d)", port1, s2, port2); 61 | return; 62 | } else if (s1[0] && !s2[0]) { 63 | sprintf(res, "%s(%d) ->", s1, port1); 64 | return; 65 | } 66 | } else { 67 | if (!s1[0] && !s2[0]) { 68 | sprintf(res, "-> %d", port2); 69 | return; 70 | } else if (!s1[0] && s2[0]) { 71 | sprintf(res, "-> %s(%d)", s2, port2); 72 | } else if (s1[0] && !s2[0]) { 73 | sprintf(res, "%s(%d) -> %d", s1, port1, port2); 74 | return; 75 | } 76 | } 77 | } 78 | 79 | -------------------------------------------------------------------------------- /Makefile.am: -------------------------------------------------------------------------------- 1 | AM_CPPFLAGS = -I$(srcdir)/tkvdb 2 | 3 | bin_PROGRAMS = xenoeye xemkgeodb xegeoq xesflow 4 | xenoeye_SOURCES = xenoeye.c xenoeye.h xe-debug.h \ 5 | utils.h utils.c utils-data.inc netflow.h netflow.c \ 6 | netflow-templates.c netflow-templates.h \ 7 | sflow.c sflow.h xe-sni.h xe-dns.h \ 8 | tkvdb/tkvdb.c tkvdb/tkvdb.h \ 9 | aajson/aajson.h \ 10 | filter.c filter.h filter-lexer.c filter-parser.c \ 11 | filter-parser-funcs.c \ 12 | pcapture.c scapture.c \ 13 | monit-objects.c monit-objects.h \ 14 | monit-objects-fwm.c monit-objects-mavg.c \ 15 | monit-objects-mavg-act.c monit-objects-mavg-dump.c \ 16 | monit-objects-mavg-limfile.c \ 17 | monit-objects-mavg-under.c \ 18 | classification.c \ 19 | flow-debug.h flow-debug.c \ 20 | devices.h devices.c \ 21 | iplist.h iplist.c \ 22 | ip-btrie.h \ 23 | geoip.h geoip.c 24 | 25 | xemkgeodb_SOURCES = xemkgeodb.c geoip.h ip-btrie.h tkvdb/tkvdb.c tkvdb/tkvdb.h 26 | 27 | xegeoq_SOURCES = xegeoq.c geoip.h ip-btrie.h 28 | 29 | xesflow_SOURCES = xesflow.c rawparse.h sflow.h sflow-impl.h xe-sni.h xe-dns.h 30 | 31 | 32 | # checks 33 | check_PROGRAMS = test_filters 34 | test_filters_SOURCES = tests/test_filters.c \ 35 | filter.c filter-lexer.c filter-parser.c \ 36 | iplist.c filter-parser-funcs.c \ 37 | geoip.c utils.c 38 | TESTS = $(check_PROGRAMS) 39 | 40 | # config files 41 | configs = xenoeye.conf devices.conf 42 | 43 | # DB export script 44 | libexec_SCRIPTS = scripts/xe-dbexport-pg.sh scripts/xe-dbexport-ch.sh 45 | 46 | # data directories and config files 47 | install-data-local: 48 | $(MKDIR_P) "$(DESTDIR)$(localstatedir)/xenoeye/mo" 49 | $(MKDIR_P) "$(DESTDIR)$(localstatedir)/xenoeye/exp" 50 | $(MKDIR_P) "$(DESTDIR)$(localstatedir)/xenoeye/expfailed" 51 | $(MKDIR_P) "$(DESTDIR)$(localstatedir)/xenoeye/iplists" 52 | $(MKDIR_P) "$(DESTDIR)$(localstatedir)/xenoeye/notifications" 53 | $(MKDIR_P) "$(DESTDIR)$(localstatedir)/xenoeye/clsf" 54 | $(MKDIR_P) "$(DESTDIR)$(localstatedir)/xenoeye/geoip" 55 | $(MKDIR_P) "$(DESTDIR)$(sysconfdir)" 56 | @for cfg in $(configs); do \ 57 | $(INSTALL_DATA) $$cfg "$(DESTDIR)$(sysconfdir)/$$cfg.sample"; \ 58 | if test -f $(DESTDIR)$(sysconfdir)/$$cfg; then \ 59 | echo "$(DESTDIR)$(sysconfdir)/$$cfg already exists; overwrite manually"; \ 60 | else \ 61 | $(INSTALL_DATA) $$cfg "$(DESTDIR)$(sysconfdir)/$$cfg"; \ 62 | fi \ 63 | done 64 | -------------------------------------------------------------------------------- /scripts/mkchart-matplotlib-ts-ip.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib 3 | matplotlib.use('agg') 4 | import matplotlib.pyplot as plt 5 | from matplotlib import dates 6 | import psycopg2 7 | import pandas as pd 8 | 9 | INTERVAL = '24 hours' 10 | N_IPS = 10 11 | AGGR_MINUTES = 30 12 | CONNSTR = 'postgresql://xenoeye:password@localhost/xenoeyedb' 13 | 14 | date_time = [] 15 | ips = [] 16 | octets = [] 17 | 18 | # Load dataset 19 | query = """ 20 | SELECT date_trunc('hour', time) + date_part('minute', time)::int / {aggr} * interval '{aggr} min', sum(octets)/EXTRACT(epoch FROM interval '{aggr} min') * 8 AS ip, ips FROM 21 | ( 22 | WITH topips AS 23 | (SELECT sum(octets) AS ip, COALESCE (src_host::text, 'Other') as ips FROM ingress_bytes_by_src WHERE time >= now() - interval '{interval}' GROUP BY ips ORDER BY ip desc limit {n_ips}) 24 | SELECT time, octets, COALESCE (src_host::text, 'Other') as ips FROM ingress_bytes_by_src WHERE time >= now() - interval '{interval}' AND src_host::text IN (SELECT ips from topips) 25 | UNION 26 | SELECT time, octets, 'Other' as ips FROM ingress_bytes_by_src WHERE time >= now() - interval '{interval}' AND src_host::text NOT IN (SELECT ips from topips) 27 | UNION 28 | SELECT time, octets, 'Other' as ips FROM ingress_bytes_by_src WHERE time >= now() - interval '{interval}' AND src_host IS NULL 29 | ) AS report 30 | GROUP BY time, ips ORDER BY time; 31 | """.format(interval=INTERVAL, aggr=AGGR_MINUTES, n_ips=N_IPS) 32 | 33 | conn = psycopg2.connect(CONNSTR) 34 | cursor = conn.cursor() 35 | cursor.execute(query) 36 | records = cursor.fetchall() 37 | for record in records: 38 | date_time.append(record[0]) 39 | octets.append(int(record[1])) 40 | ips.append(record[2]) 41 | 42 | cursor.close() 43 | conn.close() 44 | 45 | df = pd.DataFrame({'time': date_time, 'octets': octets, 'ips': ips}) 46 | table = pd.pivot_table(df, index='time', columns='ips', values='octets', aggfunc='sum', fill_value=0) 47 | 48 | table.plot.bar(stacked=True, figsize=(12, 8), ylabel='BPS', xlabel='Time', title='') 49 | plt.legend(title='IP', bbox_to_anchor=(1.05, 1), loc='upper left') 50 | 51 | mkfunc = lambda x, pos: '%1.1fGb' % (x * 1e-9) if x >= 1e9 else '%1.1fMb' % (x * 1e-6) if x >= 1e6 else '%1.1fKb' % (x * 1e-3) if x >= 1e3 else '%1.1f' % x 52 | mkformatter = matplotlib.ticker.FuncFormatter(mkfunc) 53 | plt.gca().yaxis.set_major_formatter(mkformatter) 54 | 55 | plt.tight_layout() 56 | 57 | plt.savefig('mpl-day-ips.png') 58 | -------------------------------------------------------------------------------- /scapture.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "xenoeye.h" 6 | #include "netflow.h" 7 | #include "sflow.h" 8 | #include "flow-info.h" 9 | 10 | static void * 11 | scapture_thread(void *arg) 12 | { 13 | struct capture_thread_params params, *params_ptr; 14 | 15 | socklen_t clientlen; 16 | 17 | params_ptr = (struct capture_thread_params *)arg; 18 | params = *params_ptr; 19 | free(params_ptr); 20 | 21 | clientlen = sizeof(struct sockaddr); 22 | 23 | LOG("Starting collector thread on port %d", params.cap->port); 24 | 25 | for (;;) { 26 | ssize_t len; 27 | struct flow_packet_info pkt; 28 | 29 | len = recvfrom(params.cap->sockfd, pkt.rawpacket, 30 | MAX_NF_PACKET_SIZE, 0, 31 | &(pkt.src_addr), &clientlen); 32 | 33 | if (len < 0) { 34 | LOG("recvfrom() failed: %s", strerror(errno)); 35 | 36 | continue; 37 | } 38 | 39 | if (pkt.src_addr.sa_family == AF_INET) { 40 | struct sockaddr_in *addr; 41 | 42 | addr = (struct sockaddr_in *)&pkt.src_addr; 43 | /* we're supporting only IPv4 */ 44 | 45 | pkt.src_addr_ipv4 = 46 | *((uint32_t *)&(addr->sin_addr)); 47 | } else { 48 | pkt.src_addr_ipv4 = 0; 49 | } 50 | 51 | if (params.type == FLOW_TYPE_NETFLOW) { 52 | if (netflow_process(params.data, params.thread_idx, 53 | &pkt, len)) { 54 | /* ok */ 55 | } 56 | } else { 57 | /* sflow */ 58 | sflow_process(params.data, params.thread_idx, &pkt, 59 | len); 60 | } 61 | } 62 | 63 | close(params.cap->sockfd); 64 | 65 | return NULL; 66 | } 67 | 68 | int 69 | scapture_start(struct xe_data *data, struct capture *cap, size_t thread_idx, 70 | enum FLOW_TYPE type) 71 | { 72 | int one = 1; 73 | struct capture_thread_params *params; 74 | struct sockaddr_in serveraddr; 75 | int thread_err; 76 | 77 | params = malloc(sizeof(struct capture_thread_params)); 78 | if (!params) { 79 | LOG("malloc() failed"); 80 | goto fail_alloc; 81 | } 82 | 83 | params->data = data; 84 | params->thread_idx = thread_idx; 85 | params->cap = cap; 86 | params->type = type; 87 | 88 | cap->sockfd = socket(AF_INET, SOCK_DGRAM, 0); 89 | if (cap->sockfd < 0) { 90 | LOG("socket() failed: %s", strerror(errno)); 91 | goto fail_socket; 92 | } 93 | 94 | if (setsockopt(cap->sockfd, SOL_SOCKET, SO_REUSEADDR, 95 | (const void *)&one, sizeof(int)) == -1) { 96 | 97 | LOG("setsockopt() failed: %s", strerror(errno)); 98 | goto fail_setsockopt; 99 | } 100 | 101 | bzero((char *)&serveraddr, sizeof(serveraddr)); 102 | serveraddr.sin_family = AF_INET; 103 | 104 | /* FIXME: take address from user */ 105 | serveraddr.sin_addr.s_addr = htonl(INADDR_ANY); 106 | serveraddr.sin_port = htons(cap->port); 107 | 108 | if (bind(cap->sockfd, (struct sockaddr *)&serveraddr, 109 | sizeof(serveraddr)) < 0) { 110 | 111 | LOG("bind() failed: %s", strerror(errno)); 112 | goto fail_bind; 113 | } 114 | 115 | thread_err = pthread_create(&cap->tid, NULL, &scapture_thread, params); 116 | 117 | if (thread_err) { 118 | LOG("Can't start thread: %s", strerror(thread_err)); 119 | goto fail_thread; 120 | } 121 | 122 | return 1; 123 | 124 | /* errors */ 125 | fail_thread: 126 | fail_bind: 127 | fail_setsockopt: 128 | close(cap->sockfd); 129 | fail_socket: 130 | free(params); 131 | fail_alloc: 132 | return 0; 133 | } 134 | 135 | 136 | -------------------------------------------------------------------------------- /xenoeye.h: -------------------------------------------------------------------------------- 1 | #ifndef xenoeye_h_included 2 | #define xenoeye_h_included 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "utils.h" 10 | #include "xe-debug.h" 11 | #include "monit-objects.h" 12 | 13 | /*#define FLOWS_CNT*/ 14 | #define CLICKHOUSE_CODEC_MAXSIZE 16 15 | 16 | struct flow_info; 17 | struct filter_expr; 18 | 19 | enum XENOEYE_CAPTURE_TYPE 20 | { 21 | XENOEYE_CAPTURE_TYPE_SOCKET, 22 | XENOEYE_CAPTURE_TYPE_PCAP 23 | }; 24 | 25 | enum FLOW_TYPE 26 | { 27 | FLOW_TYPE_NETFLOW, 28 | FLOW_TYPE_SFLOW 29 | }; 30 | 31 | enum DB_TYPE 32 | { 33 | DB_PG, 34 | DB_CH 35 | }; 36 | 37 | struct capture 38 | { 39 | enum XENOEYE_CAPTURE_TYPE type; 40 | 41 | pthread_t tid; 42 | 43 | /* pcap */ 44 | pcap_t *pcap_handle; 45 | char *iface; 46 | char *filter; 47 | 48 | /* socket */ 49 | int sockfd; 50 | char *addr; 51 | unsigned int port; 52 | }; 53 | 54 | struct xe_data 55 | { 56 | size_t nmonit_objects; 57 | struct monit_object *monit_objects; 58 | 59 | struct capture *nfcap; 60 | size_t nnfcap; 61 | 62 | struct capture *sfcap; 63 | size_t nsfcap; 64 | 65 | /* numer of threads, nthreads == ncap */ 66 | size_t nthreads; 67 | 68 | /* backgriund thread for fixed windows in memory */ 69 | pthread_t fwm_tid; 70 | 71 | /* moving averages */ 72 | pthread_t mavg_dump_tid, mavg_act_tid, mavg_under_tid; 73 | _Atomic size_t mavg_db_bank_idx; 74 | 75 | /* classification thread */ 76 | pthread_t clsf_tid; 77 | 78 | /* GeoIP/AS databases reload thread */ 79 | pthread_t geoip_tid; 80 | /* config reload thread */ 81 | pthread_t config_tid; 82 | 83 | /* templates */ 84 | int allow_templates_in_future; 85 | char templates_db[PATH_MAX]; 86 | 87 | /* debug settings */ 88 | struct xe_debug debug; 89 | 90 | /* path to devices list */ 91 | char devices[PATH_MAX]; 92 | 93 | /* path to monitoring objects */ 94 | char mo_dir[PATH_MAX]; 95 | 96 | /* path to export files dir */ 97 | char exp_dir[PATH_MAX]; 98 | 99 | /* path to IP lists */ 100 | char iplists_dir[PATH_MAX]; 101 | 102 | /* path to notification files */ 103 | char notif_dir[PATH_MAX]; 104 | 105 | /* path to dir with classification */ 106 | char clsf_dir[PATH_MAX]; 107 | 108 | /* path to dir with GeoIP/AS DBs */ 109 | char geodb_dir[PATH_MAX]; 110 | 111 | /* path to DB export script */ 112 | char db_exporter_path[PATH_MAX]; 113 | 114 | /* type of DBMS */ 115 | enum DB_TYPE db_type; 116 | char ch_codec[CLICKHOUSE_CODEC_MAXSIZE]; 117 | 118 | /* notify geoip thread about reload */ 119 | atomic_int reload_geoip; 120 | 121 | /* config reload */ 122 | atomic_int reload_config; 123 | 124 | /* notify threads about stop */ 125 | atomic_int stop; 126 | 127 | #ifdef FLOWS_CNT 128 | /* flows counter */ 129 | _Atomic uint64_t nflows; 130 | pthread_t fc_tid; 131 | #endif 132 | }; 133 | 134 | 135 | /* helper struct for passing data to capture threads */ 136 | struct capture_thread_params 137 | { 138 | struct xe_data *data; 139 | struct capture *cap; 140 | size_t thread_idx; /* thread index */ 141 | enum FLOW_TYPE type; 142 | }; 143 | 144 | int scapture_start(struct xe_data *data, struct capture *cap, 145 | size_t thread_idx, enum FLOW_TYPE type); 146 | int pcapture_start(struct xe_data *data, struct capture *cap, 147 | size_t thread_idx, enum FLOW_TYPE type); 148 | 149 | #endif 150 | 151 | -------------------------------------------------------------------------------- /utils.h: -------------------------------------------------------------------------------- 1 | #ifndef utils_h_included 2 | #define utils_h_included 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #define TOKEN_MAX_SIZE 512 12 | 13 | #define TCP_FLAGS_STR_MAX_SIZE 32 14 | #define TCP_UDP_PORT_STR_MAX_SIZE 40 15 | #define TCP_UDP_PP_STR_MAX_SIZE 90 16 | 17 | #define LOG(...) \ 18 | do { \ 19 | char _buf[4096]; \ 20 | int _ret = snprintf(_buf, sizeof(_buf), __VA_ARGS__); \ 21 | if (_ret >= (int)(sizeof(_buf))) { \ 22 | syslog(LOG_DEBUG | LOG_USER, \ 23 | "Next line truncated to %d symbols", \ 24 | _ret); \ 25 | } \ 26 | syslog(LOG_DEBUG | LOG_USER, \ 27 | "%s [%s, line %d, function %s()]", \ 28 | _buf, __FILE__, __LINE__, __func__); \ 29 | } while (0) 30 | 31 | static inline char * 32 | string_trim(char *str) 33 | { 34 | char *end; 35 | 36 | /* Trim leading space */ 37 | while(isspace((unsigned char)*str)) { 38 | str++; 39 | } 40 | 41 | if(*str == 0) { 42 | /* All spaces? */ 43 | return str; 44 | } 45 | 46 | /* Trim trailing space */ 47 | end = str + strlen(str) - 1; 48 | while(end > str && isspace((unsigned char)*end)) { 49 | end--; 50 | } 51 | 52 | /* Write new null terminator character */ 53 | end[1] = '\0'; 54 | 55 | return str; 56 | } 57 | 58 | static inline void 59 | csv_next(char **line, char *val) 60 | { 61 | char *ptr = *line, *end; 62 | size_t len; 63 | 64 | /* skip spaces */ 65 | while (isspace(*ptr)) { 66 | ptr++; 67 | } 68 | 69 | if (*ptr == '\0') { 70 | /* empty */ 71 | val[0] = '\0'; 72 | return; 73 | } 74 | 75 | if (*ptr == '\"') { 76 | /* string */ 77 | ptr++; 78 | 79 | for (;;) { 80 | if (*ptr == '\0') { 81 | /* unexpected end, no closing quote */ 82 | *val = '\0'; 83 | *line = ptr; 84 | return; 85 | } else if (*ptr == '\"') { 86 | if (*(ptr + 1) == '\"') { 87 | *val = '\"'; 88 | val++; 89 | ptr += 2; 90 | } else { 91 | ptr++; 92 | break; 93 | } 94 | } else { 95 | *val = *ptr; 96 | val++; 97 | ptr++; 98 | } 99 | } 100 | 101 | end = strchr(ptr, ','); 102 | if (!end) { 103 | *line = strchr(ptr, '\0'); 104 | return; 105 | } 106 | *line = end + 1; 107 | } else { 108 | end = strchr(ptr, ','); 109 | if (!end) { 110 | /* no comma */ 111 | strcpy(val, ptr); 112 | *line = strchr(ptr, '\0'); 113 | return; 114 | } 115 | len = end - ptr; 116 | memcpy(val, ptr, len); 117 | val[len] = '\0'; 118 | *line = end + 1; 119 | } 120 | } 121 | 122 | 123 | char *tcp_flags_to_str(uint8_t tf); 124 | void port_to_str(char *res, uint16_t port); 125 | void ports_pair_to_str(char *res, uint16_t port1, uint16_t port2); 126 | 127 | typedef __int128_t xe_ip; 128 | 129 | /* __builtin_bswap128 */ 130 | static inline xe_ip 131 | bswap128(xe_ip x) 132 | { 133 | union _128_as_64 { 134 | xe_ip v; 135 | uint64_t q[2]; 136 | } u1, u2; 137 | 138 | u1.v = x; 139 | u2.q[1] = bswap_64(u1.q[0]); 140 | u2.q[0] = bswap_64(u1.q[1]); 141 | 142 | return u2.v; 143 | } 144 | 145 | #endif 146 | 147 | -------------------------------------------------------------------------------- /xe-sni.h: -------------------------------------------------------------------------------- 1 | #ifndef xe_sni_h_included 2 | #define xe_sni_h_included 3 | 4 | #define PREFIX "\t\t\t\t" 5 | 6 | struct tls_rec 7 | { 8 | uint8_t type; 9 | uint16_t version; 10 | uint16_t len; 11 | } __attribute__((packed)); 12 | 13 | struct tls_hello 14 | { 15 | uint8_t type; 16 | uint8_t len[3]; 17 | uint16_t version; 18 | uint8_t random[32]; 19 | uint8_t session_id_len; 20 | } __attribute__((packed)); 21 | 22 | struct tls_ext 23 | { 24 | uint16_t type; 25 | uint16_t len; 26 | } __attribute__((packed)); 27 | 28 | struct tls_sni 29 | { 30 | uint16_t list_len; 31 | uint8_t type; 32 | uint16_t name_len; 33 | uint8_t name[1]; 34 | } __attribute__((packed)); 35 | 36 | static int 37 | xe_sni(uint8_t *p, uint8_t *end, char *domain) 38 | { 39 | uint16_t *cipher_suites_len; 40 | uint8_t *compress_methods_len; 41 | uint16_t *ext_len; 42 | struct tls_hello *hello; 43 | struct tls_rec *rec = (struct tls_rec *)p; 44 | 45 | if (rec->type != 0x16) { 46 | return 0; 47 | } 48 | 49 | if ((rec->version != htobe16(0x0301)) 50 | && (rec->version != htobe16(0x0303))) { 51 | 52 | return 0; 53 | } 54 | 55 | LOG(PREFIX"Probably TLS Handshake (ver 0x%04x)", be16toh(rec->version)); 56 | 57 | p += sizeof(struct tls_rec); 58 | if (p >= end) { 59 | LOG(PREFIX"TLS Packet too short"); 60 | return 0; 61 | } 62 | 63 | hello = (struct tls_hello *)p; 64 | if (hello->type != 1) { 65 | LOG(PREFIX"TLS Packet type is not Client Hello"); 66 | return 0; 67 | } 68 | 69 | LOG(PREFIX"TLS Client Hello"); 70 | 71 | p += sizeof(struct tls_hello) + hello->session_id_len; 72 | if (p >= end) { 73 | LOG(PREFIX"TLS Packet too short"); 74 | return 0; 75 | } 76 | 77 | cipher_suites_len = (uint16_t *)p; 78 | p += sizeof(uint16_t) + be16toh(*cipher_suites_len); 79 | if (p >= end) { 80 | LOG(PREFIX"TLS Packet too short"); 81 | return 0; 82 | } 83 | 84 | compress_methods_len = (uint8_t *)p; 85 | p += sizeof(uint8_t) + *compress_methods_len; 86 | if (p >= end) { 87 | LOG(PREFIX"TLS Packet too short"); 88 | return 0; 89 | } 90 | 91 | ext_len = (uint16_t *)p; 92 | if (*ext_len == 0) { 93 | LOG(PREFIX"TLS Client Hello has no extensions"); 94 | return 0; 95 | } 96 | p += sizeof(uint16_t); /* skip ext len */ 97 | for (;;) { 98 | if ((p + sizeof(struct tls_ext)) >= end) { 99 | LOG(PREFIX"TLS Packet too short"); 100 | return 0; 101 | } 102 | struct tls_ext *e = (struct tls_ext *)p; 103 | 104 | LOG(PREFIX"TLS ext type: %04x, len: %d ", e->type, be16toh(e->len)); 105 | 106 | if (e->type == 0x0000) { 107 | char server_name[256]; 108 | /* sni */ 109 | p += sizeof(struct tls_ext); 110 | if ((p + sizeof(struct tls_sni)) >= end) { 111 | LOG(PREFIX"TLS Packet too short"); 112 | return 0; 113 | } 114 | struct tls_sni *sni = (struct tls_sni *)p; 115 | if (sni->type == 0x00) { 116 | uint16_t name_len = be16toh(sni->name_len); 117 | if ((p + sizeof(struct tls_sni) + name_len - 1) >= end) { 118 | LOG(PREFIX"TLS Packet too short"); 119 | return 0; 120 | } 121 | memcpy(server_name, sni->name, name_len); 122 | server_name[name_len] = '\0'; 123 | LOG(PREFIX"TLS SNI: %s", server_name); 124 | if (domain) { 125 | strcpy(domain, server_name); 126 | } 127 | } 128 | break; 129 | } 130 | p += sizeof(struct tls_ext) + be16toh(e->len); 131 | if (p >= end) { 132 | LOG(PREFIX"No TLS SNI in packet"); 133 | return 0; 134 | } 135 | } 136 | return 1; 137 | } 138 | 139 | #undef PREFIX 140 | 141 | #endif 142 | 143 | -------------------------------------------------------------------------------- /monit-objects-mavg-limfile.c: -------------------------------------------------------------------------------- 1 | /* 2 | * xenoeye 3 | * 4 | * Copyright (c) 2024, Vladimir Misyurov 5 | * 6 | * Permission to use, copy, modify, and/or distribute this software for any 7 | * purpose with or without fee is hereby granted, provided that the above 8 | * copyright notice and this permission notice appear in all copies. 9 | * 10 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 11 | * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 12 | * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 13 | * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 14 | * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE 15 | * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 16 | * PERFORMANCE OF THIS SOFTWARE. 17 | */ 18 | 19 | #include 20 | #include 21 | #include 22 | #include "monit-objects.h" 23 | 24 | static int 25 | mavg_limits_parse_line(struct mo_mavg *window, char *line, uint8_t *key, 26 | MAVG_TYPE *val) 27 | { 28 | char token[TOKEN_MAX_SIZE]; 29 | size_t i; 30 | size_t validx = 0; 31 | 32 | for (i=0; ifieldset.n; i++) { 33 | struct field *fld = &window->fieldset.fields[i]; 34 | 35 | /* get token */ 36 | csv_next(&line, token); 37 | 38 | if (fld->aggr) { 39 | val[validx] = strtod(token, NULL); 40 | validx++; 41 | } else { 42 | /* append to key */ 43 | int res; 44 | uint8_t d8; 45 | uint16_t d16; 46 | uint32_t d32; 47 | uint64_t d64; 48 | 49 | if (fld->type == FILTER_BASIC_ADDR4) { 50 | res = inet_pton(AF_INET, token, key); 51 | if (res != 1) { 52 | LOG("Can't convert '%s' to " 53 | "IPv4 address", token); 54 | return 0; 55 | } 56 | } else if (fld->type == FILTER_BASIC_ADDR6) { 57 | res = inet_pton(AF_INET6, token, key); 58 | if (res != 1) { 59 | LOG("Can't convert '%s' to " 60 | "IPv6 address", token); 61 | return 0; 62 | } 63 | } else if (fld->type == FILTER_BASIC_STRING) { 64 | memset(key, 0, fld->size); 65 | strcpy((char *)key, token); 66 | } else { 67 | /* FIXME: check? */ 68 | long long int v = atoll(token); 69 | switch (fld->size) { 70 | case 1: 71 | d8 = v; 72 | memcpy(key, &d8, 1); 73 | break; 74 | case 2: 75 | d16 = htons(v); 76 | memcpy(key, &d16, 2); 77 | break; 78 | case 4: 79 | d32 = htonl(v); 80 | memcpy(key, &d32, 4); 81 | break; 82 | case 8: 83 | d64 = htobe64(v); 84 | memcpy(key, &d64, 8); 85 | break; 86 | } 87 | } 88 | 89 | key += fld->size; 90 | } 91 | } 92 | 93 | return 1; 94 | } 95 | 96 | 97 | /* load CSV file with limits */ 98 | int 99 | mavg_limits_file_load(struct mo_mavg *window, struct mavg_limit *l) 100 | { 101 | tkvdb_datum dtk, dtv; 102 | TKVDB_RES rc; 103 | uint8_t *key; 104 | MAVG_TYPE *val; 105 | 106 | FILE *f = fopen(l->file, "r"); 107 | if (!f) { 108 | LOG("Can't open file '%s': %s", l->file, strerror(errno)); 109 | l->db->free(l->db); 110 | return 0; 111 | } 112 | 113 | key = window->thr_data[0].key; 114 | val = alloca(sizeof(MAVG_TYPE) * window->fieldset.n_aggr); 115 | 116 | dtk.data = key; 117 | dtk.size = window->thr_data[0].keysize; 118 | 119 | dtv.data = val; 120 | dtv.size = sizeof(MAVG_TYPE) * window->fieldset.n_aggr; 121 | 122 | for (;;) { 123 | char line[2048], *trline; 124 | 125 | if (!fgets(line, sizeof(line) - 1, f)) { 126 | break; 127 | } 128 | 129 | trline = string_trim(line); 130 | if (strlen(trline) == 0) { 131 | /* skip empty line */ 132 | continue; 133 | } 134 | if (trline[0] == '#') { 135 | /* skip comment */ 136 | continue; 137 | } 138 | 139 | if (!mavg_limits_parse_line(window, trline, key, val)) { 140 | continue; 141 | } 142 | 143 | /* append to limits database */ 144 | rc = l->db->put(l->db, &dtk, &dtv); 145 | if (rc != TKVDB_OK) { 146 | LOG("Can't add item from '%s' to limits db, code %d", 147 | l->file, rc); 148 | } 149 | } 150 | fclose(f); 151 | 152 | return 1; 153 | } 154 | 155 | -------------------------------------------------------------------------------- /netflow.h: -------------------------------------------------------------------------------- 1 | #ifndef netflow_h_included 2 | #define netflow_h_included 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include "xenoeye.h" 9 | 10 | #define MAX_FLOWS_PER_PACKET 100 11 | 12 | #define MAX_FIELDS_PER_FLOW 100 13 | 14 | #define MAX_FLOW_VAL_LEN 32 15 | 16 | enum NF_FIELD_TYPE 17 | { 18 | NF_FIELD_IP_ADDR, 19 | NF_FIELD_INT, 20 | NF_FIELD_STRING, 21 | NF_FIELD_BYTES 22 | }; 23 | 24 | /* netflow v5 */ 25 | struct nf5_header 26 | { 27 | uint16_t version; 28 | uint16_t count; 29 | uint32_t sys_uptime; 30 | uint32_t unix_secs; 31 | uint32_t unix_nsecs; 32 | 33 | uint32_t flow_sequence; 34 | uint8_t engine_type; 35 | uint8_t engine_id; 36 | uint16_t sampling; 37 | } __attribute__ ((__packed__)); 38 | 39 | #define NF5_FIELDS \ 40 | FIELD(1, uint32_t, src_addr, ip4_src_addr, 8) \ 41 | FIELD(1, uint32_t, dst_addr, ip4_dst_addr, 12) \ 42 | FIELD(1, uint32_t, next_hop, ip4_next_hop, 15) \ 43 | FIELD(1, uint16_t, input_snmp, input_snmp, 10) \ 44 | FIELD(1, uint16_t, output_snmp, output_snmp, 14) \ 45 | FIELD(1, uint32_t, packets, in_pkts, 2) \ 46 | FIELD(1, uint32_t, octets, in_bytes, 1) \ 47 | FIELD(1, uint32_t, first, first_switched, 22) \ 48 | FIELD(1, uint32_t, last, last_switched, 21) \ 49 | FIELD(1, uint16_t, src_port, l4_src_port, 7) \ 50 | FIELD(1, uint16_t, dst_port, l4_dst_port, 11) \ 51 | FIELD(0, uint8_t, pad1, pad1, 65530) \ 52 | FIELD(1, uint8_t, tcp_flags, tcp_flags, 6) \ 53 | FIELD(1, uint8_t, protocol, protocol, 4) \ 54 | FIELD(1, uint8_t, tos, src_tos, 5) \ 55 | FIELD(1, uint16_t, src_as, src_as, 16) \ 56 | FIELD(1, uint16_t, dst_as, dst_as, 17) \ 57 | FIELD(1, uint8_t, src_mask, src_mask, 9) \ 58 | FIELD(1, uint8_t, dst_mask, dst_mask, 13) \ 59 | FIELD(0, uint16_t, pad2, pad2, 65531) 60 | 61 | struct nf5_flow 62 | { 63 | #define FIELD(USE, TYPE, V5, V9, ID) \ 64 | TYPE V5; 65 | NF5_FIELDS 66 | #undef FIELD 67 | } __attribute__ ((__packed__)); 68 | 69 | struct nf5_packet 70 | { 71 | struct nf5_header header; 72 | struct nf5_flow flows[1]; 73 | } __attribute__ ((__packed__)); 74 | 75 | /* netflow v9 */ 76 | struct nf9_header 77 | { 78 | uint16_t version; 79 | uint16_t count; 80 | uint32_t sys_uptime; 81 | uint32_t unix_secs; 82 | 83 | uint32_t package_sequence; 84 | uint32_t source_id; 85 | } __attribute__ ((__packed__)); 86 | 87 | struct nf9_fieldtype_and_len 88 | { 89 | uint16_t type; 90 | uint16_t length; 91 | } __attribute__ ((__packed__)); 92 | 93 | struct nf9_flowset_header 94 | { 95 | uint16_t flowset_id; 96 | uint16_t length; 97 | } __attribute__ ((__packed__)); 98 | 99 | struct nf9_template_item 100 | { 101 | uint16_t template_id; 102 | uint16_t field_count; 103 | struct nf9_fieldtype_and_len typelen[1]; 104 | } __attribute__ ((__packed__)); 105 | 106 | /* IPFIX */ 107 | struct ipfix_header 108 | { 109 | uint16_t version; 110 | uint16_t length; 111 | uint32_t export_time; 112 | 113 | uint32_t sequence_number; 114 | uint32_t observation_domain; 115 | } __attribute__ ((__packed__)); 116 | 117 | /* IPFIX templates */ 118 | struct ipfix_template_header 119 | { 120 | uint16_t template_id; 121 | uint16_t field_count; 122 | } __attribute__ ((__packed__)); 123 | 124 | struct ipfix_inf_element_iana 125 | { 126 | uint16_t id; 127 | uint16_t length; 128 | } __attribute__ ((__packed__)); 129 | 130 | struct ipfix_inf_element_enterprise 131 | { 132 | uint16_t id; 133 | uint16_t length; 134 | uint32_t number; 135 | } __attribute__ ((__packed__)); 136 | 137 | struct ipfix_stored_template 138 | { 139 | struct ipfix_template_header header; 140 | 141 | struct ipfix_inf_element_enterprise elements[1]; 142 | } __attribute__ ((__packed__)); 143 | 144 | /* flowset */ 145 | struct ipfix_flowset_header 146 | { 147 | uint16_t flowset_id; 148 | uint16_t length; 149 | } __attribute__ ((__packed__)); 150 | 151 | struct flow_packet_info; 152 | 153 | void netflow_process_init(void); 154 | int netflow_process(struct xe_data *data, size_t thread_id, 155 | struct flow_packet_info *npi, int len); 156 | 157 | 158 | #endif 159 | 160 | -------------------------------------------------------------------------------- /scripts/telegram-bot/xe-tele-bot.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import asyncio 3 | import os 4 | 5 | from aiogram import Bot, Dispatcher, executor, types 6 | 7 | API_TOKEN = '...' 8 | 9 | # list of chat id's 10 | CHATS = [] 11 | 12 | POLL_INTERVAL = 10 13 | 14 | MSGS_DIR = "/var/lib/xenoeye/telemsg/" 15 | 16 | # Configure logging 17 | logging.basicConfig(level=logging.INFO) 18 | 19 | # Initialize bot and dispatcher 20 | bot = Bot(token=API_TOKEN, parse_mode=types.ParseMode.HTML) 21 | dp = Dispatcher(bot) 22 | 23 | 24 | @dp.message_handler(commands=['start', 'help']) 25 | async def send_welcome(message: types.Message): 26 | """ 27 | This handler will be called when user sends `/start` or `/help` command 28 | """ 29 | await message.reply("Hi!\nI'm xenoeye bot\nPowered by aiogram.") 30 | 31 | @dp.message_handler(commands=['id']) 32 | async def send_id(message: types.Message): 33 | await message.reply("Chat id: {}".format(message.chat.id)) 34 | 35 | @dp.message_handler() 36 | async def any(message: types.Message): 37 | await message.answer("Only /id or /help commands allowed") 38 | 39 | 40 | async def check_events(): 41 | for filename in os.listdir(MSGS_DIR): 42 | full_name = os.path.join(MSGS_DIR, filename) 43 | if not os.path.isfile(full_name): 44 | continue 45 | filename_wo_ext = os.path.splitext(full_name)[0] 46 | 47 | if filename.endswith(".s"): 48 | # anomaly is gone 49 | # green 50 | msg_txt = "🟢 " 51 | f = open(full_name) 52 | msg_txt += f.read() 53 | f.close() 54 | pfile_name = filename_wo_ext + ".p" 55 | 56 | if os.path.isfile(pfile_name): 57 | f = open(pfile_name, "r") 58 | lines = f.readlines() 59 | for line in lines: 60 | # reply to start messages 61 | ls = line.split(":") 62 | chat_id = ls[0] 63 | msg_id = ls[1] 64 | await bot.send_message(chat_id, msg_txt, reply_to_message_id=msg_id) 65 | os.remove(full_name) 66 | os.remove(pfile_name) 67 | continue 68 | else: 69 | if not os.path.isfile(filename_wo_ext + ".n"): 70 | # anomaly ended but no previous files left 71 | for chat_id in CHATS: 72 | await bot.send_message(chat_id, msg_txt) 73 | os.remove(full_name) 74 | continue 75 | 76 | if os.path.isfile(filename_wo_ext + ".n"): 77 | # anomaly is gone but was not reported 78 | # yellow 79 | msg_txt = "🟡 " 80 | f = open(filename_wo_ext + ".n", "r") 81 | msg_txt += f.read() 82 | f.close() 83 | 84 | msg_txt += "\n" 85 | f = open(filename_wo_ext + ".s", "r") 86 | msg_txt += f.read() 87 | f.close() 88 | 89 | for chat_id in CHATS: 90 | await bot.send_message(chat_id, msg_txt) 91 | 92 | os.remove(full_name) 93 | os.remove(filename_wo_ext + ".n") 94 | # next file 95 | continue 96 | 97 | # second pass 98 | for filename in os.listdir(MSGS_DIR): 99 | full_name = os.path.join(MSGS_DIR, filename) 100 | if not os.path.isfile(full_name): 101 | continue 102 | filename_wo_ext = os.path.splitext(full_name)[0] 103 | if filename.endswith(".n"): 104 | # new anomaly 105 | # red 106 | msg_txt = "🔴 " 107 | f = open(full_name, "r") 108 | msg_txt += f.read() 109 | f.close() 110 | out_name = filename_wo_ext + ".-tmp" 111 | f = open(out_name, "w") 112 | for chat_id in CHATS: 113 | msg = await bot.send_message(chat_id, msg_txt) 114 | f.write("{}:{}\n".format(chat_id, msg["message_id"])) 115 | f.close() 116 | os.rename(out_name, filename_wo_ext + ".p") 117 | os.remove(full_name) 118 | 119 | 120 | def repeat(coro, loop): 121 | asyncio.ensure_future(coro(), loop=loop) 122 | loop.call_later(POLL_INTERVAL, repeat, coro, loop) 123 | 124 | 125 | if __name__ == '__main__': 126 | loop = asyncio.get_event_loop() 127 | loop.call_later(POLL_INTERVAL, repeat, check_events, loop) 128 | executor.start_polling(dp, loop=loop, skip_updates=True) 129 | -------------------------------------------------------------------------------- /netflow.def: -------------------------------------------------------------------------------- 1 | /* name description type id, size_min, size_max */ 2 | FIELD(in_bytes, "Bytes", NF_FIELD_INT, 1, 4, 8) 3 | FIELD(in_pkts, "Packets", NF_FIELD_INT, 2, 4, 8) 4 | FIELD(protocol, "Protocol", NF_FIELD_INT, 4, 1, 1) 5 | FIELD(src_tos, "Src TOS", NF_FIELD_INT, 5, 1, 1) 6 | FIELD(dst_tos, "Dst TOS", NF_FIELD_INT, 55, 1, 1) 7 | FIELD(tcp_flags, "TCP flags", NF_FIELD_INT, 6, 1, 1) 8 | FIELD(l4_src_port, "Src port", NF_FIELD_INT, 7, 2, 2) 9 | FIELD(ip4_src_addr, "IPv4 src addr", NF_FIELD_IP_ADDR, 8, 4, 4) 10 | FIELD(src_mask, "Src mask", NF_FIELD_INT, 9, 1, 1) 11 | FIELD(input_snmp, "Input SNMP index", NF_FIELD_INT, 10, 2, 4) 12 | FIELD(l4_dst_port, "Dst port", NF_FIELD_INT, 11, 2, 2) 13 | FIELD(ip4_dst_addr, "IPv4 dst addr", NF_FIELD_IP_ADDR, 12, 4, 4) 14 | FIELD(dst_mask, "Dst mask", NF_FIELD_INT, 13, 1, 1) 15 | FIELD(output_snmp, "Output SNMP index", NF_FIELD_INT, 14, 2, 4) 16 | FIELD(ip4_next_hop, "IPv4 next hop", NF_FIELD_IP_ADDR, 15, 4, 4) 17 | FIELD(src_as, "Src AS", NF_FIELD_INT, 16, 2, 4) 18 | FIELD(dst_as, "Dst AS", NF_FIELD_INT, 17, 2, 4) 19 | FIELD(bgp_next_hop, "BGP IPv4 next hop", NF_FIELD_IP_ADDR, 18, 4, 4) 20 | FIELD(last_switched, "Flow end time", NF_FIELD_INT, 21, 4, 4) 21 | FIELD(first_switched, "Flow start time", NF_FIELD_INT, 22, 4, 4) 22 | FIELD(ip6_src_addr, "IPv6 src addr", NF_FIELD_IP_ADDR, 27, 16, 16) 23 | FIELD(ip6_dst_addr, "IPv6 dst addr", NF_FIELD_IP_ADDR, 28, 16, 16) 24 | FIELD(icmp_type, "ICMP type", NF_FIELD_INT, 32, 2, 2) 25 | FIELD(sampler_id, "Flow sampler id", NF_FIELD_INT, 48, 1, 2) 26 | FIELD(min_ttl, "Min TTL", NF_FIELD_INT, 52, 1, 1) 27 | FIELD(max_ttl, "Max TTL", NF_FIELD_INT, 53, 1, 1) 28 | FIELD(frag_id, "Fragment id", NF_FIELD_INT, 54, 2, 4) 29 | FIELD(src_vlan, "Src VLAN", NF_FIELD_INT, 58, 2, 2) 30 | FIELD(dst_vlan, "Dst VLAN", NF_FIELD_INT, 59, 2, 2) 31 | FIELD(ip_protocol_version, "IP version", NF_FIELD_INT, 60, 1, 1) 32 | FIELD(direction, "Flow direction", NF_FIELD_INT, 61, 1, 1) 33 | FIELD(if_name, "Interface name", NF_FIELD_STRING, 82, 1, 16) 34 | FIELD(fwd_status, "Forwarding status", NF_FIELD_INT, 89, 1, 1) 35 | FIELD(flow_start_ms, "Flow start ms", NF_FIELD_INT, 152,8, 8) 36 | FIELD(flow_end_ms, "Flow end ms", NF_FIELD_INT, 153,8, 8) 37 | FIELD(flow_end_reason, "Flow end reason", NF_FIELD_INT, 136,1, 1) 38 | FIELD(ioctets, "Initator octets", NF_FIELD_INT, 231,8, 8) 39 | FIELD(roctets, "Responder octets", NF_FIELD_INT, 232,2, 8) 40 | FIELD(ipackets, "Initator packets", NF_FIELD_INT, 298,8, 8) 41 | FIELD(rpackets, "Responder packets", NF_FIELD_INT, 239,8, 8) 42 | FIELD(ingrs_vrf, "Ingress VRFID", NF_FIELD_INT, 234,4, 4) 43 | FIELD(egrs_vrf, "Egress VRFID", NF_FIELD_INT, 235,4, 4) 44 | FIELD(dot1q_vlan, "Dot1q VLAN", NF_FIELD_INT, 243,2, 2) 45 | FIELD(dot1q_cvlan, "Dot1q customer VLAN",NF_FIELD_INT, 245,2, 2) 46 | FIELD(dev_ip, "*dev-ip", NF_FIELD_IP_ADDR, 65500,4,4) 47 | FIELD(dns_name, "DNS Domain", NF_FIELD_STRING, 65510,1,256) 48 | FIELD(dns_ips, "DNS Addresses", NF_FIELD_STRING, 65511,1,512) 49 | FIELD(sni, "SNI domain name", NF_FIELD_STRING, 65512,1,256) 50 | FIELD(class0, "Class0", NF_FIELD_STRING, 65520,1,CLASS_NAME_MAX) 51 | FIELD(class1, "Class1", NF_FIELD_STRING, 65521,1,CLASS_NAME_MAX) 52 | FIELD(class2, "Class2", NF_FIELD_STRING, 65522,1,CLASS_NAME_MAX) 53 | FIELD(class3, "Class3", NF_FIELD_STRING, 65523,1,CLASS_NAME_MAX) 54 | FIELD(class4, "Class4", NF_FIELD_STRING, 65524,1,CLASS_NAME_MAX) 55 | FIELD(pad1, "Padding", NF_FIELD_INT, 65530,1,1) 56 | FIELD(pad2, "Padding", NF_FIELD_INT, 65531,2,2) 57 | #undef FIELD 58 | -------------------------------------------------------------------------------- /xe-dns.h: -------------------------------------------------------------------------------- 1 | #ifndef xe_dns_included 2 | #define xe_dns_included 3 | 4 | #include 5 | #include 6 | 7 | #define PREFIX "\t\t\t\t" 8 | 9 | struct dns_ans_data 10 | { 11 | uint16_t type; 12 | uint16_t aclass; 13 | uint32_t ttl; 14 | uint16_t data_len; 15 | uint8_t data[1]; 16 | } __attribute__((packed)); 17 | 18 | static inline int 19 | xe_dns(uint8_t *p, uint8_t *end, char *domain, char *ips) 20 | { 21 | HEADER *dns_h = (HEADER *)p; 22 | int i; 23 | char qname[_POSIX_HOST_NAME_MAX + 1]; 24 | char *nptr = qname; 25 | int count; 26 | uint8_t *base; 27 | int first_addr = 1; 28 | 29 | /* check flags */ 30 | if ((dns_h->qr != 1) || (dns_h->opcode != 0) || (dns_h->rcode != 0)) { 31 | return 0; 32 | } 33 | 34 | if ((dns_h->qdcount == 0) || (dns_h->ancount == 0)) { 35 | return 0; 36 | } 37 | 38 | LOG(PREFIX"Probably DNS response"); 39 | 40 | base = p; 41 | p += sizeof(HEADER); 42 | 43 | count = be16toh(dns_h->qdcount); 44 | 45 | memset(qname, 0, sizeof(qname)); 46 | for (i=0; i= end) { 59 | LOG(PREFIX"DNS packet too short"); 60 | return 0; 61 | } 62 | for (j=0; j= end) { 72 | LOG(PREFIX"DNS packet too short"); 73 | return 0; 74 | } 75 | } 76 | LOG(PREFIX"DNS domain: %s", qname); 77 | if (domain) { 78 | strcpy(domain, qname); 79 | } 80 | 81 | /* answers */ 82 | count = be16toh(dns_h->ancount); 83 | for (i=0; i= end) { 106 | LOG(PREFIX"DNS packet too short"); 107 | goto end; 108 | } 109 | c = *p; 110 | } 111 | if ((aptr + c) >= aend) { 112 | LOG(PREFIX"Malformed DNS packet"); 113 | goto end; 114 | } 115 | 116 | p++; 117 | if ((p + c) >= end) { 118 | LOG(PREFIX"DNS packet too short"); 119 | goto end; 120 | } 121 | 122 | for (j=0; j end) { 134 | LOG(PREFIX"DNS packet too short"); 135 | goto end; 136 | } 137 | 138 | struct dns_ans_data *ad = (struct dns_ans_data *)p; 139 | unsigned int l_ad = sizeof(struct dns_ans_data) - 1 140 | + be16toh(ad->data_len); 141 | 142 | if (ad->type == htobe16(0x0001)) { 143 | /* type A */ 144 | char addr[INET6_ADDRSTRLEN + 1]; 145 | if (ad->data_len == htobe16(4)) { 146 | inet_ntop(AF_INET, ad->data, addr, 147 | INET_ADDRSTRLEN); 148 | LOG(PREFIX"DNS ip: %s", addr); 149 | if (ips) { 150 | if (first_addr) { 151 | first_addr = 0; 152 | ips[0] = '{'; 153 | ips[1] = '\0'; 154 | } else { 155 | strcat(ips, ","); 156 | } 157 | strcat(ips, addr); 158 | } 159 | } 160 | } else if (ad->type == htobe16(28)) { 161 | /* AAAA */ 162 | if ((p + sizeof(struct dns_ans_data) + 15) > end) { 163 | LOG(PREFIX"DNS packet too short"); 164 | goto end; 165 | } 166 | char addr[INET6_ADDRSTRLEN + 1]; 167 | if (ad->data_len == htobe16(16)) { 168 | inet_ntop(AF_INET6, ad->data, addr, 169 | INET6_ADDRSTRLEN); 170 | LOG(PREFIX"DNS ip: %s", addr); 171 | if (ips) { 172 | if (first_addr) { 173 | first_addr = 0; 174 | ips[0] = '{'; 175 | ips[1] = '\0'; 176 | } else { 177 | strcat(ips, ","); 178 | } 179 | strcat(ips, addr); 180 | } 181 | } 182 | } 183 | if ((p + l_ad) > end) { 184 | LOG(PREFIX"DNS packet too short"); 185 | goto end; 186 | } 187 | p += l_ad; 188 | } 189 | end: 190 | if (first_addr) { 191 | /* no ips */ 192 | return 0; 193 | } 194 | if (ips) { 195 | strcat(ips, "}"); 196 | } 197 | return 1; 198 | } 199 | 200 | #undef PREFIX 201 | 202 | #endif 203 | 204 | -------------------------------------------------------------------------------- /scripts/telegram-bot/xe-tele-bot-aio3.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import asyncio 3 | import os 4 | 5 | from aiogram import Bot, Dispatcher, types 6 | from aiogram.client.default import DefaultBotProperties 7 | from aiogram.enums import ParseMode 8 | from aiogram.types import Message 9 | from aiogram.filters import Command, CommandObject, CommandStart 10 | 11 | API_TOKEN = '...' 12 | 13 | # list of chat id's 14 | CHATS = [] 15 | 16 | POLL_INTERVAL = 10 17 | 18 | MSGS_DIR = "/var/lib/xenoeye/telemsg/" 19 | 20 | # Configure logging 21 | logging.basicConfig(level=logging.INFO) 22 | 23 | # Initialize bot and dispatcher 24 | bot = Bot(token=API_TOKEN, default=DefaultBotProperties(parse_mode=ParseMode.HTML)) 25 | dp = Dispatcher() 26 | 27 | 28 | @dp.channel_post(Command("help")) 29 | @dp.channel_post(CommandStart()) 30 | @dp.message(Command("help")) 31 | @dp.message(CommandStart()) 32 | async def send_welcome(message: Message): 33 | """ 34 | This handler will be called when user sends `/start` or `/help` command 35 | """ 36 | await message.reply("Hi!\nI'm xenoeye bot\nPowered by aiogram.") 37 | 38 | @dp.channel_post(Command("id")) 39 | @dp.message(Command("id")) 40 | async def send_id(message: types.Message): 41 | await message.reply("Chat id: {}".format(message.chat.id)) 42 | 43 | async def check_events(): 44 | for filename in os.listdir(MSGS_DIR): 45 | full_name = os.path.join(MSGS_DIR, filename) 46 | if not os.path.isfile(full_name): 47 | continue 48 | filename_wo_ext = os.path.splitext(full_name)[0] 49 | 50 | if filename.endswith(".s"): 51 | # anomaly is gone 52 | # green 53 | msg_txt = "🟢 " 54 | f = open(full_name) 55 | msg_txt += f.read() 56 | f.close() 57 | pfile_name = filename_wo_ext + ".p" 58 | 59 | if os.path.isfile(pfile_name): 60 | f = open(pfile_name, "r") 61 | lines = f.readlines() 62 | for line in lines: 63 | # reply to start messages 64 | ls = line.split(":") 65 | chat_id = ls[0] 66 | msg_id = ls[1] 67 | await bot.send_message(chat_id, msg_txt, reply_to_message_id=msg_id) 68 | os.remove(full_name) 69 | os.remove(pfile_name) 70 | continue 71 | else: 72 | if not os.path.isfile(filename_wo_ext + ".n"): 73 | # anomaly ended but no previous files left 74 | for chat_id in CHATS: 75 | await bot.send_message(chat_id, msg_txt) 76 | os.remove(full_name) 77 | continue 78 | 79 | if os.path.isfile(filename_wo_ext + ".n"): 80 | # anomaly is gone but was not reported 81 | # yellow 82 | msg_txt = "🟡 " 83 | f = open(filename_wo_ext + ".n", "r") 84 | msg_txt += f.read() 85 | f.close() 86 | 87 | msg_txt += "\n" 88 | f = open(filename_wo_ext + ".s", "r") 89 | msg_txt += f.read() 90 | f.close() 91 | 92 | for chat_id in CHATS: 93 | await bot.send_message(chat_id, msg_txt) 94 | 95 | os.remove(full_name) 96 | os.remove(filename_wo_ext + ".n") 97 | # next file 98 | continue 99 | 100 | # second pass 101 | for filename in os.listdir(MSGS_DIR): 102 | full_name = os.path.join(MSGS_DIR, filename) 103 | if not os.path.isfile(full_name): 104 | continue 105 | filename_wo_ext = os.path.splitext(full_name)[0] 106 | if filename.endswith(".n"): 107 | # new anomaly 108 | # red 109 | msg_txt = "🔴 " 110 | f = open(full_name, "r") 111 | msg_txt += f.read() 112 | f.close() 113 | out_name = filename_wo_ext + ".-tmp" 114 | f = open(out_name, "w") 115 | for chat_id in CHATS: 116 | msg = await bot.send_message(chat_id, msg_txt) 117 | f.write("{}:{}\n".format(chat_id, msg.message_id)) 118 | f.close() 119 | os.rename(out_name, filename_wo_ext + ".p") 120 | os.remove(full_name) 121 | 122 | 123 | def repeat(coro, loop): 124 | asyncio.ensure_future(coro(), loop=loop) 125 | loop.call_later(POLL_INTERVAL, repeat, coro, loop) 126 | 127 | async def main(): 128 | loop = asyncio.get_event_loop() 129 | loop.call_later(POLL_INTERVAL, repeat, check_events, loop) 130 | await dp.start_polling(bot) 131 | 132 | if __name__ == "__main__": 133 | asyncio.run(main()) 134 | -------------------------------------------------------------------------------- /sflow.c: -------------------------------------------------------------------------------- 1 | /* 2 | * xenoeye 3 | * 4 | * Copyright (c) 2025, Vladimir Misyurov 5 | * 6 | * Permission to use, copy, modify, and/or distribute this software for any 7 | * purpose with or without fee is hereby granted, provided that the above 8 | * copyright notice and this permission notice appear in all copies. 9 | * 10 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 11 | * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 12 | * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 13 | * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 14 | * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE 15 | * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 16 | * PERFORMANCE OF THIS SOFTWARE. 17 | */ 18 | 19 | #include "utils.h" 20 | #include "flow-info.h" 21 | #include "sflow.h" 22 | #include "filter.h" 23 | #include "flow-debug.h" 24 | #include "devices.h" 25 | 26 | #define COPY_TO_FLOW(D, F, R, N) \ 27 | do { \ 28 | memcpy(D->F, R, N); \ 29 | D->has_##F = 1; \ 30 | D->F##_size = N; \ 31 | } while (0) 32 | 33 | #define USER_TYPE struct flow_info * 34 | 35 | #define ON_VLAN1(D, V) \ 36 | COPY_TO_FLOW(D, src_vlan, &V->h_vlan_TCI, 2); \ 37 | COPY_TO_FLOW(D, dst_vlan, &V->h_vlan_TCI, 2); 38 | 39 | #define ON_VLAN2(D, V) \ 40 | COPY_TO_FLOW(D, dot1q_vlan, &V->h_vlan_TCI, 2); 41 | 42 | #define ON_IP(D, V) \ 43 | COPY_TO_FLOW(D, ip4_src_addr, &V->saddr, 4); \ 44 | COPY_TO_FLOW(D, ip4_dst_addr, &V->daddr, 4); \ 45 | COPY_TO_FLOW(D, src_tos, &V->tos, 1); \ 46 | COPY_TO_FLOW(D, dst_tos, &V->tos, 1); \ 47 | COPY_TO_FLOW(D, min_ttl, &V->ttl, 1); \ 48 | COPY_TO_FLOW(D, max_ttl, &V->ttl, 1); \ 49 | COPY_TO_FLOW(D, protocol, &V->protocol, 1); 50 | /* TODO: fragmented packets? */ 51 | 52 | #define ON_IP6(D, V) \ 53 | COPY_TO_FLOW(D, ip6_src_addr, &V->ip6_src, 16); \ 54 | COPY_TO_FLOW(D, ip6_dst_addr, &V->ip6_src, 16); \ 55 | COPY_TO_FLOW(D, min_ttl, &V->ip6_ctlun.ip6_un1.ip6_un1_hlim, 1); \ 56 | COPY_TO_FLOW(D, max_ttl, &V->ip6_ctlun.ip6_un1.ip6_un1_hlim, 1); \ 57 | COPY_TO_FLOW(D, protocol, &nexthdr, 1); 58 | 59 | #define ON_TCP(D, V) \ 60 | COPY_TO_FLOW(D, l4_src_port, &V->th_sport, 2); \ 61 | COPY_TO_FLOW(D, l4_dst_port, &V->th_dport, 2); \ 62 | COPY_TO_FLOW(D, tcp_flags, &V->th_flags, 1); 63 | 64 | #define ON_UDP(D, V) \ 65 | COPY_TO_FLOW(D, l4_src_port, &V->uh_sport, 2); \ 66 | COPY_TO_FLOW(D, l4_dst_port, &V->uh_dport, 2); 67 | 68 | #define ON_ICMP(D, V) \ 69 | COPY_TO_FLOW(D, icmp_type, &V->type, 1); 70 | /* TODO: ICMP code? */ 71 | 72 | #define ON_PAYLOAD(D, V) D->payload_ptr = V; 73 | 74 | #include "rawparse.h" 75 | 76 | static int xe_sni(uint8_t *p, uint8_t *end, char *domain); 77 | static int xe_dns(uint8_t *p, uint8_t *end, char *domain, char *ips); 78 | 79 | static void 80 | process_mo_sflow_rec(struct sfdata *s, uint8_t *end, struct monit_object *mos, 81 | size_t n_mo) 82 | { 83 | size_t i; 84 | for (i=0; iexpr, s->flow)) { 88 | continue; 89 | } 90 | 91 | if (mo->payload_parse_dns && s->flow->payload_ptr) { 92 | if (xe_dns(s->flow->payload_ptr, end, 93 | (char *)s->flow->dns_name, 94 | (char *)s->flow->dns_ips)) { 95 | 96 | s->flow->has_dns_name = 1; 97 | s->flow->has_dns_ips = 1; 98 | } 99 | } 100 | 101 | if (mo->payload_parse_sni && s->flow->payload_ptr) { 102 | if (xe_sni(s->flow->payload_ptr, 103 | end, (char *)s->flow->sni)) { 104 | 105 | s->flow->has_sni = 1; 106 | } 107 | } 108 | 109 | monit_object_process_nf(s->global, mo, s->thread_id, 110 | s->fpi->time_ns, s->flow); 111 | 112 | if (mo->debug.print_flows) { 113 | char debug_flow_str[1024]; 114 | sflow_debug_print(s->flow, debug_flow_str); 115 | 116 | flow_print_str(&mo->debug, s->flow, debug_flow_str, 1); 117 | } 118 | 119 | /* child objects */ 120 | if (mo->n_mo) { 121 | process_mo_sflow_rec(s, end, mo->mos, mo->n_mo); 122 | } 123 | } 124 | } 125 | 126 | static inline int 127 | sf5_eth(struct sfdata *s, uint8_t *p, enum RP_TYPE t, uint32_t header_len) 128 | { 129 | uint8_t *end = p + header_len; 130 | 131 | if (rawpacket_parse(p, end, t, s->flow) 132 | < RP_PARSER_STATE_NO_IP) { 133 | 134 | /* Skip non-IP samples */ 135 | return 1; 136 | } 137 | /* check interfaces */ 138 | if (!device_rules_check(s->flow, s->fpi)) { 139 | /* no error */ 140 | return 1; 141 | } 142 | 143 | /* debug print */ 144 | if (s->global->debug.print_flows) { 145 | char debug_flow_str[1024]; 146 | sflow_debug_print(s->flow, debug_flow_str); 147 | 148 | flow_print_str(&s->global->debug, s->flow, debug_flow_str, 1); 149 | } 150 | 151 | process_mo_sflow_rec(s, end, 152 | s->global->monit_objects, s->global->nmonit_objects); 153 | 154 | #ifdef FLOWS_CNT 155 | atomic_fetch_add_explicit(&s->global->nflows, 1, memory_order_relaxed); 156 | #endif 157 | return 1; 158 | } 159 | 160 | /* disable logging */ 161 | #undef LOG 162 | #define LOG(...) 163 | 164 | #include "xe-sni.h" 165 | #include "xe-dns.h" 166 | 167 | #include "sflow-impl.h" 168 | 169 | -------------------------------------------------------------------------------- /geoip.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include "xenoeye.h" 16 | #include "geoip.h" 17 | #include "ip-btrie.h" 18 | 19 | /* geo */ 20 | static struct btrie_node_geo * _Atomic _geodb4 = NULL; 21 | static size_t _geo4size = 0; 22 | static struct btrie_node_geo * _Atomic _geodb6 = NULL; 23 | static size_t _geo6size = 0; 24 | 25 | /* as */ 26 | static struct btrie_node_as * _Atomic _asdb4 = NULL; 27 | static size_t _as4size = 0; 28 | static struct btrie_node_as * _Atomic _asdb6 = NULL; 29 | static size_t _as6size = 0; 30 | 31 | 32 | /* geo */ 33 | int 34 | geoip_lookup4(uint32_t addr, struct geoip_info **g) 35 | { 36 | uint8_t *addr_ptr = (uint8_t *)&addr; 37 | struct btrie_node_geo *geo4 = atomic_load_explicit(&_geodb4, 38 | memory_order_relaxed); 39 | 40 | IP_BTRIE_LOOKUP(geo4, 4 * 8); 41 | 42 | *g = &geo4[node].g; 43 | 44 | return 1; 45 | } 46 | 47 | int 48 | geoip_lookup6(xe_ip *addr, struct geoip_info **g) 49 | { 50 | uint8_t *addr_ptr = (uint8_t *)addr; 51 | struct btrie_node_geo *geo6 = atomic_load_explicit(&_geodb6, 52 | memory_order_relaxed); 53 | 54 | IP_BTRIE_LOOKUP(geo6, 16 * 8); 55 | 56 | *g = &geo6[node].g; 57 | 58 | return 1; 59 | } 60 | 61 | /* as */ 62 | int 63 | as_lookup4(uint32_t addr, struct as_info **a) 64 | { 65 | uint8_t *addr_ptr = (uint8_t *)&addr; 66 | struct btrie_node_as *as4 = atomic_load_explicit(&_asdb4, 67 | memory_order_relaxed); 68 | 69 | IP_BTRIE_LOOKUP(as4, 4 * 8); 70 | 71 | *a = &as4[node].a; 72 | 73 | return 1; 74 | } 75 | 76 | int 77 | as_lookup6(xe_ip *addr, struct as_info **a) 78 | { 79 | uint8_t *addr_ptr = (uint8_t *)addr; 80 | struct btrie_node_as *as6 = atomic_load_explicit(&_asdb6, 81 | memory_order_relaxed); 82 | 83 | IP_BTRIE_LOOKUP(as6, 16 * 8); 84 | 85 | *a = &as6[node].a; 86 | 87 | return 1; 88 | } 89 | 90 | 91 | static void * 92 | mmap_db(struct xe_data *data, const char *dbname, size_t *size) 93 | { 94 | void *addr = NULL; 95 | struct stat st; 96 | int fd; 97 | 98 | char path[PATH_MAX + 8]; 99 | 100 | *size = 0; 101 | sprintf(path, "%s/%s.db", data->geodb_dir, dbname); 102 | 103 | fd = open(path, O_RDONLY); 104 | if (fd == -1) { 105 | LOG("Can't open file '%s': %s", path, strerror(errno)); 106 | return NULL; 107 | } 108 | 109 | if (fstat(fd, &st) != 0) { 110 | LOG("fstat() failed on file '%s': %s", path, strerror(errno)); 111 | goto fail_fstat; 112 | } 113 | 114 | *size = st.st_size; 115 | addr = mmap(NULL, *size, PROT_READ, MAP_PRIVATE, fd, 0); 116 | if (addr == (void *) -1) { 117 | LOG("mmap() failed on file '%s': %s", path, strerror(errno)); 118 | addr = NULL; 119 | goto fail_mmap; 120 | } 121 | 122 | fail_fstat: 123 | fail_mmap: 124 | close(fd); 125 | 126 | return addr; 127 | } 128 | 129 | static void 130 | geoip_reload(struct xe_data *data) 131 | { 132 | struct btrie_node_geo *geo4, *geo6; 133 | struct btrie_node_geo *geo4old, *geo6old; 134 | struct btrie_node_as *as4, *as6; 135 | struct btrie_node_as *as4old, *as6old; 136 | size_t geo4size, geo6size; 137 | size_t as4size, as6size; 138 | 139 | /* save old pointers */ 140 | geo4old = atomic_load_explicit(&_geodb4, memory_order_relaxed); 141 | geo6old = atomic_load_explicit(&_geodb6, memory_order_relaxed); 142 | as4old = atomic_load_explicit(&_asdb4, memory_order_relaxed); 143 | as6old = atomic_load_explicit(&_asdb6, memory_order_relaxed); 144 | 145 | /* create new mappings */ 146 | geo4 = mmap_db(data, "geo4", &geo4size); 147 | geo6 = mmap_db(data, "geo6", &geo6size); 148 | as4 = mmap_db(data, "as4", &as4size); 149 | as6 = mmap_db(data, "as6", &as6size); 150 | 151 | /* replace atomically */ 152 | atomic_store_explicit(&_geodb4, geo4, memory_order_relaxed); 153 | atomic_store_explicit(&_geodb6, geo6, memory_order_relaxed); 154 | atomic_store_explicit(&_asdb4, as4, memory_order_relaxed); 155 | atomic_store_explicit(&_asdb6, as6, memory_order_relaxed); 156 | 157 | /* wait for stalled requests */ 158 | usleep(100); 159 | 160 | /* remove old mappings */ 161 | #define UNMAP(X, S) \ 162 | do { \ 163 | if (X) { \ 164 | if (munmap(X, S) != 0) { \ 165 | LOG("munmap() failed: %s", strerror(errno)); \ 166 | } \ 167 | } \ 168 | } while (0) 169 | 170 | UNMAP(geo4old, _geo4size); 171 | UNMAP(geo6old, _geo6size); 172 | UNMAP(as4old, _as4size); 173 | UNMAP(as6old, _as6size); 174 | #undef UNMAP 175 | 176 | /* update db sizes */ 177 | _geo4size = geo4size; 178 | _geo6size = geo6size; 179 | _as4size = as4size; 180 | _as6size = as6size; 181 | } 182 | 183 | void * 184 | geoip_thread(void *arg) 185 | { 186 | struct xe_data *data = (struct xe_data *)arg; 187 | 188 | LOG("geoip: starting helper thread"); 189 | for (;;) { 190 | if (atomic_load_explicit(&data->stop, memory_order_relaxed)) { 191 | /* stop */ 192 | break; 193 | } 194 | 195 | if (atomic_load_explicit(&data->reload_geoip, 196 | memory_order_relaxed)) { 197 | 198 | atomic_store_explicit(&data->reload_geoip, 0, 199 | memory_order_relaxed); 200 | LOG("Reloading geo/as databases"); 201 | geoip_reload(data); 202 | LOG("geo/as databases reloaded"); 203 | } 204 | 205 | usleep(10000); 206 | } 207 | 208 | return NULL; 209 | } 210 | 211 | -------------------------------------------------------------------------------- /netflow-templates.c: -------------------------------------------------------------------------------- 1 | /* 2 | * xenoeye 3 | * 4 | * Copyright (c) 2019-2022, Vladimir Misyurov, Michael Kogan 5 | * 6 | * Permission to use, copy, modify, and/or distribute this software for any 7 | * purpose with or without fee is hereby granted, provided that the above 8 | * copyright notice and this permission notice appear in all copies. 9 | * 10 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 11 | * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 12 | * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 13 | * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 14 | * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE 15 | * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 16 | * PERFORMANCE OF THIS SOFTWARE. 17 | */ 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | #include "utils.h" 26 | #include "netflow-templates.h" 27 | #include "tkvdb/tkvdb.h" 28 | 29 | static const char *templates_db; /* path to templates db file */ 30 | static tkvdb_tr *mem_dbs[2] = {NULL, NULL}; /* database in memory */ 31 | static _Atomic size_t db_idx = 0; /* index of current mem_db */ 32 | 33 | /* load templates from disk to mem db */ 34 | static int 35 | templates_load(size_t idx) 36 | { 37 | tkvdb *db; 38 | 39 | tkvdb_cursor *c; 40 | TKVDB_RES rc; 41 | tkvdb_tr *tr; 42 | tkvdb_tr *dst; 43 | int ret = 0; 44 | 45 | dst = mem_dbs[idx]; 46 | 47 | db = tkvdb_open(templates_db, NULL); 48 | if (!db) { 49 | LOG("Can't open database '%s': %s", templates_db, 50 | strerror(errno)); 51 | goto fail_db; 52 | } 53 | 54 | tr = tkvdb_tr_create(db, NULL); 55 | if (!tr) { 56 | LOG("Can't create transaction"); 57 | goto fail_tr; 58 | } 59 | 60 | tr->begin(tr); 61 | 62 | c = tkvdb_cursor_create(tr); 63 | if (!c) { 64 | LOG("tkvdb_cursor_create() failed"); 65 | goto cursor_fail; 66 | } 67 | 68 | if (c->first(c) != TKVDB_OK) { 69 | /* empty on-disk database */ 70 | goto empty; 71 | } 72 | 73 | do { 74 | tkvdb_datum dtk, dtv; 75 | 76 | dtk = c->key_datum(c); 77 | dtv = c->val_datum(c); 78 | 79 | rc = dst->put(dst, &dtk, &dtv); 80 | if (rc != TKVDB_OK) { 81 | LOG("Can't load template item, skipping"); 82 | } 83 | } while (c->next(c) == TKVDB_OK); 84 | 85 | c->free(c); 86 | 87 | empty: 88 | tr->rollback(tr); 89 | tr->free(tr); 90 | ret = 1; 91 | 92 | cursor_fail: 93 | fail_tr: 94 | tkvdb_close(db); 95 | fail_db: 96 | 97 | return ret; 98 | } 99 | 100 | int 101 | netflow_templates_init(struct xe_data *globl) 102 | { 103 | size_t i; 104 | 105 | templates_db = globl->templates_db; 106 | 107 | for (i=0; i<2; i++) { 108 | mem_dbs[i] = tkvdb_tr_create(NULL, NULL); 109 | if (!mem_dbs[i]) { 110 | LOG("Can't create mem db"); 111 | goto fail_tr; 112 | } 113 | 114 | mem_dbs[i]->begin(mem_dbs[i]); 115 | } 116 | 117 | templates_load(0); 118 | 119 | return 1; 120 | 121 | fail_tr: 122 | netflow_templates_shutdown(); 123 | 124 | return 0; 125 | } 126 | 127 | void 128 | netflow_templates_shutdown(void) 129 | { 130 | size_t i; 131 | 132 | for (i=0; i<2; i++) { 133 | if (mem_dbs[i]) { 134 | mem_dbs[i]->free(mem_dbs[i]); 135 | mem_dbs[i] = NULL; 136 | } 137 | } 138 | } 139 | 140 | void * 141 | netflow_template_find(struct template_key *tkey, int allow_templates_in_future) 142 | { 143 | tkvdb_cursor *c; 144 | tkvdb_datum dtk; 145 | TKVDB_RES rc; 146 | void *ret = NULL; 147 | struct template_key *key; 148 | tkvdb_tr *tr; 149 | 150 | if (allow_templates_in_future) { 151 | key = alloca(sizeof(struct template_key)); 152 | *key = *tkey; 153 | key->epoch = 0xffffff; 154 | } else { 155 | key = tkey; 156 | } 157 | 158 | /* select current db bank */ 159 | tr = mem_dbs[atomic_load_explicit(&db_idx, memory_order_relaxed) % 2]; 160 | 161 | /* search for the most recent template */ 162 | c = tkvdb_cursor_create(tr); 163 | dtk.data = key; 164 | dtk.size = sizeof(struct template_key); 165 | 166 | rc = c->seek(c, &dtk, TKVDB_SEEK_LE); 167 | if ((rc == TKVDB_OK) 168 | && (c->keysize(c) == sizeof(struct template_key))) { 169 | 170 | /* size of key without time */ 171 | size_t sk = sizeof(struct template_key) - sizeof(uint32_t); 172 | 173 | /* compare keys without time */ 174 | if (memcmp(c->key(c), key, sk) == 0) { 175 | ret = c->val(c); 176 | } 177 | } 178 | c->free(c); 179 | 180 | return ret; 181 | } 182 | 183 | int 184 | netflow_template_add(struct template_key *tkey, void *t, size_t size) 185 | { 186 | tkvdb *db; 187 | tkvdb_datum dtk, dtv; 188 | 189 | tkvdb_tr *tr; 190 | size_t idx; 191 | 192 | LOG("Adding template"); 193 | 194 | /* add template to on-disk database */ 195 | db = tkvdb_open(templates_db, NULL); 196 | if (!db) { 197 | LOG("Can't open database '%s': %s", templates_db, 198 | strerror(errno)); 199 | goto fail_db; 200 | } 201 | 202 | tr = tkvdb_tr_create(db, NULL); 203 | if (!tr) { 204 | LOG("Can't create transaction"); 205 | goto fail_tr; 206 | } 207 | 208 | tr->begin(tr); 209 | 210 | dtk.data = tkey; 211 | dtk.size = sizeof(struct template_key); 212 | 213 | dtv.data = t; 214 | dtv.size = size; 215 | 216 | if (tr->put(tr, &dtk, &dtv) != TKVDB_OK) { 217 | LOG("Can't put template in storage"); 218 | return 0; 219 | } 220 | if (tr->commit(tr) != TKVDB_OK) { 221 | LOG("Can't commit transaction"); 222 | return 0; 223 | } 224 | tkvdb_close(db); 225 | 226 | /* get index of inactive mem db */ 227 | idx = (atomic_load_explicit(&db_idx, memory_order_relaxed) + 1) % 2; 228 | 229 | /* reset inactive mem db */ 230 | tr = mem_dbs[idx]; 231 | tr->rollback(tr); 232 | tr->begin(tr); 233 | 234 | /* load new db from disk to the inactive mem db */ 235 | if (!templates_load(idx)) { 236 | goto fail_load; 237 | } 238 | 239 | /* swap banks atomically, inactive db becomes active */ 240 | atomic_fetch_add_explicit(&db_idx, 1, memory_order_relaxed); 241 | 242 | LOG("Template added"); 243 | 244 | return 1; 245 | 246 | fail_tr: 247 | tkvdb_close(db); 248 | fail_load: 249 | fail_db: 250 | return 0; 251 | } 252 | 253 | -------------------------------------------------------------------------------- /rawparse.h: -------------------------------------------------------------------------------- 1 | #ifndef rawparse_h_included 2 | #define rawparse_h_included 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | struct vlan_hdr 13 | { 14 | uint16_t h_vlan_TCI; 15 | uint16_t h_vlan_encapsulated_proto; 16 | }; 17 | 18 | enum RP_PARSER_STATE 19 | { 20 | RP_PARSER_STATE_NO_ETHER, 21 | RP_PARSER_STATE_NO_VLAN, 22 | RP_PARSER_STATE_NO_IP, 23 | RP_PARSER_STATE_NO_IP_PROTO, 24 | 25 | RP_PARSER_STATE_OK 26 | }; 27 | 28 | enum RP_TYPE 29 | { 30 | RP_TYPE_ETHER, 31 | RP_TYPE_IPv4, 32 | RP_TYPE_IPv6 33 | }; 34 | 35 | #define IP_MF 0x2000 36 | #define IP_OFFSET 0x1FFF 37 | 38 | /* no global include guards */ 39 | #endif 40 | 41 | #ifndef USER_TYPE 42 | #define USER_TYPE void * 43 | #endif 44 | 45 | #ifndef ON_ETH 46 | #define ON_ETH(D, E) 47 | #endif 48 | 49 | #ifndef ON_VLAN1 50 | #define ON_VLAN1(D, V) 51 | #endif 52 | 53 | #ifndef ON_VLAN2 54 | #define ON_VLAN2(D, V) 55 | #endif 56 | 57 | #ifndef ON_HPROTO 58 | #define ON_HPROTO(D, P) 59 | #endif 60 | 61 | #ifndef ON_IP 62 | #define ON_IP(D, I) 63 | #endif 64 | 65 | #ifndef ON_FRAG 66 | #define ON_FRAG(D) 67 | #endif 68 | 69 | #ifndef ON_IP6 70 | #define ON_IP6(D, I) 71 | #endif 72 | 73 | #ifndef ON_UDP 74 | #define ON_UDP(D, U) (void)U 75 | #endif 76 | 77 | #ifndef ON_TCP 78 | #define ON_TCP(D, T) (void)T 79 | #endif 80 | 81 | #ifndef ON_ICMP 82 | #define ON_ICMP(D, I) (void)I 83 | #endif 84 | 85 | #ifndef ON_PAYLOAD 86 | #define ON_PAYLOAD(D, P) 87 | #endif 88 | 89 | static inline enum RP_PARSER_STATE 90 | rawpacket_parse(uint8_t *ptr, uint8_t *end, enum RP_TYPE t, USER_TYPE data) 91 | { 92 | uint16_t h_proto; 93 | struct ethhdr *eth; 94 | struct udphdr *udp; 95 | struct tcphdr *tcp; 96 | struct icmphdr *icmp; 97 | 98 | if ((ptr + sizeof(struct ethhdr)) >= end) { 99 | return RP_PARSER_STATE_NO_ETHER; 100 | } 101 | 102 | if (t != RP_TYPE_ETHER) { 103 | if (t == RP_TYPE_IPv4) { 104 | h_proto = htons(ETH_P_IP); 105 | goto ip; 106 | } else if (t == RP_TYPE_IPv6) { 107 | h_proto = htons(ETH_P_IPV6); 108 | goto ip; 109 | } 110 | return RP_PARSER_STATE_NO_ETHER; 111 | } 112 | 113 | eth = (struct ethhdr *)ptr; 114 | ptr += sizeof(struct ethhdr); 115 | 116 | ON_ETH(data, eth); 117 | 118 | h_proto = eth->h_proto; 119 | 120 | if ((h_proto == htobe16(ETH_P_8021Q)) || (h_proto == htobe16(ETH_P_8021AD))) { 121 | struct vlan_hdr *vhdr; 122 | 123 | if (ptr + sizeof(struct vlan_hdr) >= end) { 124 | return RP_PARSER_STATE_NO_VLAN; 125 | } 126 | vhdr = (struct vlan_hdr *)ptr; 127 | ptr += sizeof(struct vlan_hdr); 128 | 129 | ON_VLAN1(data, vhdr); 130 | 131 | h_proto = vhdr->h_vlan_encapsulated_proto; 132 | } 133 | if ((h_proto == htobe16(ETH_P_8021Q)) || (h_proto == htobe16(ETH_P_8021AD))) { 134 | struct vlan_hdr *vhdr; 135 | 136 | if (ptr + sizeof(struct vlan_hdr) >= end) { 137 | return RP_PARSER_STATE_NO_VLAN; 138 | } 139 | vhdr = (struct vlan_hdr *)ptr; 140 | ptr += sizeof(struct vlan_hdr); 141 | 142 | ON_VLAN2(data, vhdr); 143 | 144 | h_proto = vhdr->h_vlan_encapsulated_proto; 145 | } 146 | 147 | ON_HPROTO(data, h_proto); 148 | 149 | ip: 150 | if (h_proto == htons(ETH_P_IP)) { 151 | uint64_t ihl_len; 152 | struct iphdr *iph; 153 | 154 | if (ptr + sizeof(struct iphdr) >= end) { 155 | return RP_PARSER_STATE_NO_IP; 156 | } 157 | iph = (struct iphdr *)ptr; 158 | 159 | /* is fragment? */ 160 | if ((iph->frag_off & htons(IP_MF | IP_OFFSET)) != 0) { 161 | ON_IP(data, iph); 162 | ON_FRAG(data); 163 | return RP_PARSER_STATE_NO_IP_PROTO; 164 | } 165 | 166 | ihl_len = iph->ihl * 4; 167 | ptr += ihl_len; 168 | 169 | if (iph->protocol == IPPROTO_IPIP) { 170 | if (ptr + sizeof(struct iphdr) >= end) { 171 | return RP_PARSER_STATE_NO_IP; 172 | } 173 | iph = (struct iphdr *)ptr; 174 | 175 | ihl_len = iph->ihl * 4; 176 | ptr += ihl_len; 177 | } 178 | 179 | ON_IP(data, iph); 180 | 181 | if (iph->protocol == IPPROTO_TCP) { 182 | goto tcp; 183 | } else if (iph->protocol == IPPROTO_UDP) { 184 | goto udp; 185 | } else if (iph->protocol == IPPROTO_ICMP) { 186 | goto icmp; 187 | } else { 188 | return RP_PARSER_STATE_NO_IP_PROTO; 189 | } 190 | } else if (h_proto == htons(ETH_P_IPV6)) { 191 | uint64_t ihl_len = sizeof(struct ip6_hdr); 192 | uint64_t nexthdr; 193 | struct iphdr *iph; 194 | struct ip6_hdr *ip6h; 195 | 196 | if (ptr + sizeof(struct ip6_hdr) >= end) { 197 | return RP_PARSER_STATE_NO_IP; 198 | } 199 | ip6h = (struct ip6_hdr *)ptr; 200 | ptr += sizeof(struct ip6_hdr); 201 | 202 | nexthdr = ip6h->ip6_ctlun.ip6_un1.ip6_un1_nxt; 203 | 204 | if (nexthdr == IPPROTO_IPIP) { 205 | if (ptr + sizeof(struct iphdr) >= end) { 206 | return RP_PARSER_STATE_NO_IP; 207 | } 208 | iph = (struct iphdr *)ptr; 209 | 210 | ihl_len += iph->ihl * 4; 211 | nexthdr = iph->protocol; 212 | ptr += ihl_len; 213 | 214 | ON_IP(data, iph); 215 | } else if (nexthdr == IPPROTO_IPV6) { 216 | if (ptr + sizeof(struct ip6_hdr) >= end) { 217 | return RP_PARSER_STATE_NO_IP; 218 | } 219 | ip6h = (struct ip6_hdr *)ptr; 220 | ptr += sizeof(struct ip6_hdr); 221 | 222 | ihl_len += sizeof(struct ip6_hdr); 223 | //nexthdr = ip6h->nexthdr; 224 | nexthdr = ip6h->ip6_ctlun.ip6_un1.ip6_un1_nxt; 225 | 226 | ON_IP6(data, ip6h); 227 | } else { 228 | ON_IP6(data, ip6h); 229 | } 230 | 231 | if (nexthdr == IPPROTO_TCP) { 232 | goto tcp; 233 | } else if (nexthdr == IPPROTO_UDP) { 234 | goto udp; 235 | } else if (nexthdr == IPPROTO_ICMP) { 236 | goto icmp; 237 | } else if (nexthdr == IPPROTO_ICMPV6) { 238 | goto icmp; 239 | } else { 240 | return RP_PARSER_STATE_NO_IP_PROTO; 241 | } 242 | } else { 243 | /* non-ip */ 244 | return RP_PARSER_STATE_NO_IP; 245 | } 246 | 247 | tcp: 248 | if (ptr + sizeof(struct tcphdr) >= end) { 249 | return RP_PARSER_STATE_NO_IP_PROTO; 250 | } 251 | tcp = (struct tcphdr *)ptr; 252 | ptr += sizeof(struct tcphdr); 253 | ON_TCP(data, tcp); 254 | 255 | goto payload; 256 | 257 | udp: 258 | if (ptr + sizeof(struct udphdr) >= end) { 259 | return RP_PARSER_STATE_NO_IP_PROTO; 260 | } 261 | udp = (struct udphdr *)ptr; 262 | ptr += sizeof(struct udphdr); 263 | ON_UDP(data, udp); 264 | 265 | goto payload; 266 | 267 | icmp: 268 | if (ptr + sizeof(struct icmphdr) >= end) { 269 | return RP_PARSER_STATE_NO_IP_PROTO; 270 | } 271 | icmp = (struct icmphdr *)ptr; 272 | ptr += sizeof(struct icmphdr); 273 | ON_ICMP(data, icmp); 274 | 275 | goto payload; 276 | 277 | payload: 278 | ON_PAYLOAD(data, ptr); 279 | 280 | return RP_PARSER_STATE_OK; 281 | } 282 | 283 | -------------------------------------------------------------------------------- /filter-lexer.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include "filter.h" 8 | 9 | #define CHECK_END(I) \ 10 | do { \ 11 | if (*(I->s) == '\0') { \ 12 | I->end = 1; \ 13 | return; \ 14 | } \ 15 | } while (0) 16 | 17 | #define CHECK_END_UNEXP(I, ERR) \ 18 | do { \ 19 | if (*(I->s) == '\0') { \ 20 | I->end = 1; \ 21 | I->error = 1; \ 22 | strcpy(I->errmsg, ERR); \ 23 | return; \ 24 | } \ 25 | } while (0) 26 | 27 | #define SINGLE_SYM_TOKEN(I, T) \ 28 | do { \ 29 | I->current_token.data.str[0] = *(I->s); \ 30 | I->current_token.data.str[1] = '\0'; \ 31 | I->current_token.str_len = 1; \ 32 | I->s++; \ 33 | I->col++; \ 34 | I->current_token.id = T; \ 35 | } while (0) 36 | 37 | #define EXPECT(I, F) \ 38 | do { \ 39 | F(I); \ 40 | if (I->end || I->error) return; \ 41 | } while (0) 42 | 43 | static void 44 | c_style_comment(struct filter_input *q) 45 | { 46 | for (;;) { 47 | q->s++; 48 | CHECK_END_UNEXP(q, 49 | "unexpected end of input inside the comment"); 50 | 51 | q->col++; 52 | if (*(q->s) == '*') { 53 | /* end of comment? */ 54 | q->s++; 55 | CHECK_END_UNEXP(q, 56 | "unexpected end of input inside the comment"); 57 | 58 | q->col++; 59 | if (*(q->s) == '/') { 60 | /* end of comment */ 61 | break; 62 | } 63 | } else if (*(q->s) == '\n') { 64 | q->line++; 65 | q->col = 1; 66 | } 67 | } 68 | } 69 | 70 | static void 71 | one_line_comment(struct filter_input *q) 72 | { 73 | for (;;) { 74 | q->s++; 75 | CHECK_END(q); 76 | 77 | q->col++; 78 | if (*(q->s) == '\n') { 79 | /* end of comment */ 80 | 81 | q->line++; 82 | q->col = 1; 83 | break; 84 | } 85 | } 86 | } 87 | 88 | static void 89 | whitespace(struct filter_input *q) 90 | { 91 | for (;;) { 92 | CHECK_END(q); 93 | if ((*(q->s) == ' ') || (*(q->s) == '\t')) { 94 | q->col++; 95 | } else if ((*(q->s) == '\n') || (*(q->s) == '\r')) { 96 | q->line++; 97 | q->col = 1; 98 | } else if (*(q->s) == '/') { 99 | q->s++; 100 | 101 | CHECK_END_UNEXP(q, 102 | "unexpected end of input after '/'"); 103 | 104 | q->col++; 105 | if (*(q->s) == '*') { 106 | EXPECT(q, c_style_comment); 107 | } else if (*(q->s) == '/') { 108 | EXPECT(q, one_line_comment); 109 | } else { 110 | q->col--; 111 | q->s--; 112 | break; 113 | } 114 | } else { 115 | /* end of whitespace */ 116 | break; 117 | } 118 | q->s++; 119 | } 120 | } 121 | 122 | static int 123 | id_sym(int c) 124 | { 125 | int stop_symbols[] = {' ', '\t', ',', '\r', '\n', '(', ')', 0}; 126 | int *sptr = stop_symbols; 127 | 128 | if (c == '\0') { 129 | return 0; 130 | } 131 | 132 | while (*sptr) { 133 | if (c == *sptr) { 134 | return 0; 135 | } 136 | sptr++; 137 | } 138 | 139 | return 1; 140 | } 141 | 142 | static int 143 | read_str_token(const char *sample, enum TOKEN_ID *id) 144 | { 145 | #define MATCH(S) strcasecmp(sample, S) == 0 146 | 147 | if (MATCH("src")) { 148 | *id = SRC; 149 | } else if (MATCH("dst")) { 150 | *id = DST; 151 | 152 | #define FIELD(NAME, STR, TYPE, SRC, DST) \ 153 | } else if (MATCH(STR)) { \ 154 | *id = NAME; 155 | #include "filter.def" 156 | 157 | } else if (MATCH("or")) { 158 | *id = OR; 159 | } else if (MATCH("and")) { 160 | *id = AND; 161 | } else if (MATCH("not")) { 162 | *id = NOT; 163 | /* filter fields */ 164 | } else if (MATCH("asc")) { 165 | *id = ASC; 166 | } else if (MATCH("desc")) { 167 | *id = DESC; 168 | /* functions */ 169 | } else if (MATCH("div")) { 170 | *id = DIV; 171 | } else if (MATCH("div_r")) { 172 | *id = DIV_R; 173 | } else if (MATCH("div_l")) { 174 | *id = DIV_L; 175 | } else if (MATCH("min")) { 176 | *id = MIN; 177 | } else if (MATCH("mfreq")) { 178 | *id = MFREQ; 179 | } else if (MATCH("tfstr")) { 180 | *id = TFSTR; 181 | } else if (MATCH("portstr")) { 182 | *id = PORTSTR; 183 | } else if (MATCH("ppstr")) { 184 | *id = PPSTR; 185 | /* geoip */ 186 | #define DO(FIELD, SIZE) \ 187 | } else if (MATCH(#FIELD)) { \ 188 | *id = FIELD; 189 | FOR_LIST_OF_GEOIP_FIELDS 190 | #undef DO 191 | /* as */ 192 | } else if (MATCH("asn")) { 193 | *id = ASN; 194 | } else if (MATCH("asd")) { 195 | *id = ASD; 196 | 197 | #define FIELD(NAME, STR, FLD, SCALE) \ 198 | } else if (MATCH(STR)) { \ 199 | *id = NAME; 200 | #include "filter-ag.def" 201 | 202 | } else { 203 | /* unknown string */ 204 | return 0; 205 | } 206 | 207 | return 1; 208 | #undef MATCH 209 | } 210 | 211 | void 212 | read_token(struct filter_input *q) 213 | { 214 | q->current_token.str_len = 0; 215 | 216 | EXPECT(q, whitespace); 217 | 218 | if (*(q->s) == '(') { 219 | SINGLE_SYM_TOKEN(q, LPAREN); 220 | } else if (*(q->s) == ')') { 221 | SINGLE_SYM_TOKEN(q, RPAREN); 222 | } else if (*(q->s) == ',') { 223 | SINGLE_SYM_TOKEN(q, COMMA); 224 | } else if (*(q->s) == '\'') { 225 | /* string */ 226 | q->s++; 227 | q->col++; 228 | 229 | do { 230 | /* FIXME: check for \n? */ 231 | q->current_token.data.str[q->current_token.str_len] 232 | = *(q->s); 233 | q->current_token.str_len++; 234 | 235 | q->s++; 236 | q->col++; 237 | } while (*(q->s) != '\''); 238 | 239 | q->s++; 240 | q->col++; 241 | 242 | q->current_token.data.str[q->current_token.str_len] = '\0'; 243 | 244 | q->current_token.id = STRING; 245 | } else { 246 | /* read rest of token */ 247 | do { 248 | q->current_token.data.str[q->current_token.str_len] 249 | = *(q->s); 250 | q->current_token.str_len++; 251 | 252 | q->s++; 253 | q->col++; 254 | } while (id_sym(*(q->s))); 255 | 256 | q->current_token.data.str[q->current_token.str_len] = '\0'; 257 | 258 | if (!read_str_token(q->current_token.data.str, 259 | &q->current_token.id)) { 260 | 261 | /* check if it int or range */ 262 | char *endptr; 263 | long int res; 264 | 265 | res = strtol(q->current_token.data.str, &endptr, 0); 266 | if (*endptr == '\0') { 267 | /* number */ 268 | q->current_token.id = INT_RANGE; 269 | q->current_token.data.range.low 270 | = q->current_token.data.range.high 271 | = res; 272 | } else if (*endptr == '-') { 273 | long int res2; 274 | char *endptr2; 275 | 276 | res2 = strtol(endptr + 1, &endptr2, 0); 277 | if (*endptr2 == '\0') { 278 | q->current_token.id = INT_RANGE; 279 | q->current_token.data.range.low = res; 280 | q->current_token.data.range.high = res2; 281 | } else { 282 | q->current_token.id = ID; 283 | } 284 | } else { 285 | q->current_token.id = ID; 286 | } 287 | } 288 | } 289 | } 290 | 291 | -------------------------------------------------------------------------------- /ip-btrie.h: -------------------------------------------------------------------------------- 1 | #ifndef ip_btrie_h_included 2 | #define ip_btrie_h_included 3 | 4 | #define IP_BTRIE_ADD(DB, SIZE, NODE) \ 5 | int i; \ 6 | uint32_t node, next; \ 7 | \ 8 | if (!DB) { \ 9 | /* empty database */ \ 10 | DB = calloc(1, sizeof(struct NODE)); \ 11 | if (!DB) { \ 12 | LOG("Not enough memory"); \ 13 | return 0; \ 14 | } \ 15 | SIZE = 1; \ 16 | } \ 17 | \ 18 | node = 0; \ 19 | \ 20 | for (i=0; i 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | #include "iplist.h" 27 | 28 | struct bitwise_trie_node 29 | { 30 | uint32_t next[2]; 31 | int is_leaf; 32 | }; 33 | 34 | struct iplist 35 | { 36 | char name[PATH_MAX]; 37 | struct bitwise_trie_node *nodes4; 38 | struct bitwise_trie_node *nodes6; 39 | size_t n4, n6; 40 | }; 41 | 42 | static struct iplist *iplists = NULL; 43 | size_t n_iplists = 0; 44 | 45 | char * 46 | iplist_name(struct iplist *l) 47 | { 48 | return l->name; 49 | } 50 | 51 | static int 52 | iplist_add4(struct iplist *l, uint32_t addr, int mask) 53 | { 54 | int i; 55 | uint32_t node, next; 56 | uint8_t *addr_ptr = (uint8_t *)&addr; 57 | 58 | if (!l->nodes4) { 59 | /* empty database */ 60 | l->nodes4 = calloc(1, sizeof(struct bitwise_trie_node)); 61 | if (!l->nodes4) { 62 | return 0; 63 | } 64 | l->n4 = 1; 65 | } 66 | 67 | node = 0; 68 | 69 | for (i=0; inodes4[node].next[bit]; 79 | if (next) { 80 | node = next; 81 | continue; 82 | } 83 | 84 | tmp = realloc(l->nodes4, 85 | (l->n4 + 1) * sizeof(struct bitwise_trie_node)); 86 | 87 | if (!tmp) { 88 | free(l->nodes4); 89 | l->nodes4 = NULL; 90 | l->n4 = 0; 91 | return 0; 92 | } 93 | 94 | l->nodes4 = tmp; 95 | memset(&l->nodes4[l->n4], 0, sizeof(struct bitwise_trie_node)); 96 | l->nodes4[node].next[bit] = l->n4; 97 | node = l->n4; 98 | l->n4++; 99 | } 100 | l->nodes4[node].is_leaf = 1; 101 | 102 | return 1; 103 | } 104 | 105 | static int 106 | iplist_add6(struct iplist *l, xe_ip *addr, int mask) 107 | { 108 | int i; 109 | uint32_t node, next; 110 | uint8_t *addr_ptr = (uint8_t *)addr; 111 | 112 | if (!l->nodes6) { 113 | /* empty database */ 114 | l->nodes6 = calloc(1, sizeof(struct bitwise_trie_node)); 115 | if (!l->nodes6) { 116 | return 0; 117 | } 118 | l->n6 = 1; 119 | } 120 | 121 | node = 0; 122 | 123 | for (i=0; inodes6[node].next[bit]; 133 | if (next) { 134 | node = next; 135 | continue; 136 | } 137 | 138 | tmp = realloc(l->nodes6, 139 | (l->n6 + 1) * sizeof(struct bitwise_trie_node)); 140 | 141 | if (!tmp) { 142 | free(l->nodes6); 143 | l->nodes6 = NULL; 144 | l->n6 = 0; 145 | return 0; 146 | } 147 | 148 | l->nodes6 = tmp; 149 | memset(&l->nodes6[l->n6], 0, sizeof(struct bitwise_trie_node)); 150 | l->nodes6[node].next[bit] = l->n6; 151 | node = l->n6; 152 | l->n6++; 153 | } 154 | l->nodes6[node].is_leaf = 1; 155 | 156 | return 1; 157 | } 158 | 159 | int 160 | iplist_match4(struct iplist *l, uint32_t addr) 161 | { 162 | int i; 163 | uint32_t node = 0, next = 0; 164 | uint8_t *addr_ptr = (uint8_t *)&addr; 165 | 166 | if (!l->nodes4) { 167 | return 0; 168 | } 169 | 170 | for (i=0; i<32; i++) { 171 | int bit, bit_n; 172 | uint8_t byte; 173 | 174 | byte = addr_ptr[i / 8]; 175 | bit_n = 7 - (i % 8); 176 | bit = !!(byte & (1 << bit_n)); 177 | 178 | next = l->nodes4[node].next[bit]; 179 | if (!next) { 180 | break; 181 | } 182 | node = next; 183 | } 184 | 185 | return l->nodes4[node].is_leaf ? 1 : 0; 186 | } 187 | 188 | int 189 | iplist_match6(struct iplist *l, xe_ip *addr) 190 | { 191 | int i; 192 | uint32_t node = 0, next = 0; 193 | uint8_t *addr_ptr = (uint8_t *)addr; 194 | 195 | if (!l->nodes6) { 196 | return 0; 197 | } 198 | 199 | for (i=0; i<16*8; i++) { 200 | int bit, bit_n; 201 | uint8_t byte; 202 | 203 | byte = addr_ptr[i / 8]; 204 | bit_n = 7 - (i % 8); 205 | bit = !!(byte & (1 << bit_n)); 206 | 207 | next = l->nodes6[node].next[bit]; 208 | if (!next) { 209 | break; 210 | } 211 | node = next; 212 | } 213 | 214 | return l->nodes6[node].is_leaf ? 1 : 0; 215 | } 216 | 217 | static int 218 | iplist_try_load(const char *filename, const char *listname) 219 | { 220 | struct iplist *tmp; 221 | FILE *f; 222 | int line_no = 0; 223 | 224 | f = fopen(filename, "r"); 225 | if (!f) { 226 | return 0; 227 | } 228 | 229 | tmp = realloc(iplists, 230 | (n_iplists + 1) * sizeof(struct iplist)); 231 | if (!tmp) { 232 | goto realloc_fail; 233 | } 234 | 235 | memset(&tmp[n_iplists], 0, sizeof(struct iplist)); 236 | strcpy(tmp[n_iplists].name, listname); 237 | 238 | for (;;) { 239 | /* lines with comments can be pretty long */ 240 | char line[1024]; 241 | char *str_addr; 242 | uint32_t addr; 243 | xe_ip addr6; 244 | char *mask_sym; 245 | int mask = 32; 246 | 247 | if (!fgets(line, sizeof(line) - 1, f)) { 248 | break; 249 | } 250 | 251 | if (feof(f)) { 252 | break; 253 | } 254 | line_no++; 255 | 256 | line[strcspn(line, "\n#")] = 0; 257 | str_addr = string_trim(line); 258 | if (strlen(str_addr) == 0) { 259 | continue; 260 | } 261 | 262 | mask_sym = strchr(str_addr, '/'); 263 | if (mask_sym) { 264 | char *endptr; 265 | 266 | *mask_sym = '\0'; 267 | mask_sym++; 268 | mask = strtol(mask_sym, &endptr, 10); 269 | if (*endptr != '\0') { 270 | /* incorrect mask */ 271 | continue; 272 | } 273 | } 274 | 275 | if (inet_pton(AF_INET, str_addr, &addr)) { 276 | iplist_add4(&tmp[n_iplists], addr, mask); 277 | } else if (inet_pton(AF_INET6, str_addr, &addr6)) { 278 | iplist_add6(&tmp[n_iplists], &addr6, mask); 279 | } else { 280 | /* can't parse */ 281 | LOG("Can't parse address '%s', list '%s', line %d", 282 | str_addr, filename, line_no); 283 | } 284 | } 285 | 286 | iplists = tmp; 287 | n_iplists++; 288 | 289 | return 1; 290 | 291 | realloc_fail: 292 | fclose(f); 293 | return 0; 294 | } 295 | 296 | int 297 | iplists_load(const char *dirname) 298 | { 299 | DIR *d; 300 | struct dirent *dir; 301 | 302 | d = opendir(dirname); 303 | if (!d) { 304 | return 0; 305 | } 306 | 307 | while ((dir = readdir(d)) != NULL) { 308 | char path[PATH_MAX]; 309 | 310 | if (dir->d_name[0] == '.') { 311 | continue; 312 | } 313 | sprintf(path, "%s/%s", dirname, dir->d_name); 314 | iplist_try_load(path, dir->d_name); 315 | } 316 | closedir(d); 317 | 318 | return 1; 319 | } 320 | 321 | struct iplist * 322 | iplist_get_by_name(const char *name) 323 | { 324 | size_t i; 325 | struct iplist *ret = NULL; 326 | 327 | for (i=0; i 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | 31 | #include "xenoeye.h" 32 | #include "geoip.h" 33 | #include "ip-btrie.h" 34 | 35 | /* geo */ 36 | static struct btrie_node_geo * _Atomic _geodb4 = NULL; 37 | static size_t _geo4size = 0; 38 | static struct btrie_node_geo * _Atomic _geodb6 = NULL; 39 | static size_t _geo6size = 0; 40 | 41 | /* as */ 42 | static struct btrie_node_as * _Atomic _asdb4 = NULL; 43 | static size_t _as4size = 0; 44 | static struct btrie_node_as * _Atomic _asdb6 = NULL; 45 | static size_t _as6size = 0; 46 | 47 | 48 | /* geo */ 49 | int 50 | geoip_lookup4(uint32_t addr, struct geoip_info **g) 51 | { 52 | uint8_t *addr_ptr = (uint8_t *)&addr; 53 | struct btrie_node_geo *geo4 = atomic_load_explicit(&_geodb4, 54 | memory_order_relaxed); 55 | 56 | IP_BTRIE_LOOKUP(geo4, 4 * 8); 57 | 58 | *g = &geo4[node].g; 59 | 60 | return 1; 61 | } 62 | 63 | int 64 | geoip_lookup6(xe_ip *addr, struct geoip_info **g) 65 | { 66 | uint8_t *addr_ptr = (uint8_t *)addr; 67 | struct btrie_node_geo *geo6 = atomic_load_explicit(&_geodb6, 68 | memory_order_relaxed); 69 | 70 | IP_BTRIE_LOOKUP(geo6, 16 * 8); 71 | 72 | *g = &geo6[node].g; 73 | 74 | return 1; 75 | } 76 | 77 | /* as */ 78 | int 79 | as_lookup4(uint32_t addr, struct as_info **a) 80 | { 81 | uint8_t *addr_ptr = (uint8_t *)&addr; 82 | struct btrie_node_as *as4 = atomic_load_explicit(&_asdb4, 83 | memory_order_relaxed); 84 | 85 | IP_BTRIE_LOOKUP(as4, 4 * 8); 86 | 87 | *a = &as4[node].a; 88 | 89 | return 1; 90 | } 91 | 92 | int 93 | as_lookup6(xe_ip *addr, struct as_info **a) 94 | { 95 | uint8_t *addr_ptr = (uint8_t *)addr; 96 | struct btrie_node_as *as6 = atomic_load_explicit(&_asdb6, 97 | memory_order_relaxed); 98 | 99 | IP_BTRIE_LOOKUP(as6, 16 * 8); 100 | 101 | *a = &as6[node].a; 102 | 103 | return 1; 104 | } 105 | 106 | 107 | static void * 108 | mmap_db(const char *dbdir, const char *dbname, size_t *size) 109 | { 110 | void *addr = NULL; 111 | struct stat st; 112 | int fd; 113 | 114 | char path[PATH_MAX + 8]; 115 | 116 | *size = 0; 117 | sprintf(path, "%s/%s.db", dbdir, dbname); 118 | 119 | fd = open(path, O_RDONLY); 120 | if (fd == -1) { 121 | fprintf(stderr, "Can't open file '%s': %s\n", path, 122 | strerror(errno)); 123 | return NULL; 124 | } 125 | 126 | if (fstat(fd, &st) != 0) { 127 | fprintf(stderr, "fstat() failed on file '%s': %s\n", path, 128 | strerror(errno)); 129 | goto fail_fstat; 130 | } 131 | 132 | *size = st.st_size; 133 | addr = mmap(NULL, *size, PROT_READ, MAP_PRIVATE, fd, 0); 134 | if (addr == (void *) -1) { 135 | fprintf(stderr, "mmap() failed on file '%s': %s\n", path, 136 | strerror(errno)); 137 | addr = NULL; 138 | goto fail_mmap; 139 | } 140 | 141 | fail_fstat: 142 | fail_mmap: 143 | close(fd); 144 | 145 | return addr; 146 | } 147 | 148 | static void 149 | print_info4(const char *addr, uint32_t ip4) 150 | { 151 | struct geoip_info *g; 152 | struct as_info *a; 153 | 154 | if (!geoip_lookup4(ip4, &g)) { 155 | printf("%s geo: ?\n", addr); 156 | } else { 157 | printf("%s geo: %s, %s, %s, %s, %s, %s, %s, %s\n", addr, 158 | g->CONTINENT, g->COUNTRY_CODE, 159 | g->COUNTRY, g->STATE, g->CITY, g->ZIP, 160 | g->LAT, g->LONG); 161 | } 162 | 163 | if (!as_lookup4(ip4, &a)) { 164 | printf("%s as: ?\n", addr); 165 | } else { 166 | printf("%s as: %u, %s\n", addr, htobe32(a->asn), a->asd); 167 | } 168 | } 169 | 170 | static void 171 | print_info6(const char *addr, xe_ip *ip6) 172 | { 173 | struct geoip_info *g; 174 | struct as_info *a; 175 | 176 | if (!geoip_lookup6(ip6, &g)) { 177 | printf("%s geo: ?\n", addr); 178 | } else { 179 | printf("%s geo: %s, %s, %s, %s, %s, %s, %s, %s\n", addr, 180 | g->CONTINENT, g->COUNTRY_CODE, 181 | g->COUNTRY, g->STATE, g->CITY, g->ZIP, 182 | g->LAT, g->LONG); 183 | } 184 | 185 | if (!as_lookup6(ip6, &a)) { 186 | printf("%s as: ?\n", addr); 187 | } else { 188 | printf("%s as: %u, %s\n", addr, htobe32(a->asn), a->asd); 189 | } 190 | } 191 | 192 | static void 193 | print_usage(const char *progname) 194 | { 195 | fprintf(stderr, 196 | "Usage: %s [-i db_dir] ip_addr1 [ip_addr2 ...]\n", progname); 197 | fprintf(stderr, "\t-i /path/to/dir: where GeoIP/AS databases are placed\n"); 198 | fprintf(stderr, "\tip_addr1, ip_addr2, etc.: IPv4 or IPv6 addresses to lookup\n"); 199 | fprintf(stderr, "\n %s -h\n", progname); 200 | fprintf(stderr, "\t-h: print this message\n"); 201 | } 202 | 203 | 204 | int 205 | main(int argc, char *argv[]) 206 | { 207 | int opt, i; 208 | char in_dir[PATH_MAX] = "./"; /* current dir by default */ 209 | 210 | while ((opt = getopt(argc, argv, "hi:")) != -1) { 211 | switch (opt) { 212 | case 'i': 213 | strcpy(in_dir, optarg); 214 | break; 215 | 216 | case 'h': 217 | default: 218 | print_usage(argv[0]); 219 | return EXIT_FAILURE; 220 | } 221 | } 222 | 223 | if (optind == argc) { 224 | fprintf(stderr, "No input files\n"); 225 | print_usage(argv[0]); 226 | return EXIT_FAILURE; 227 | } 228 | 229 | /* create mappings */ 230 | _geodb4 = mmap_db(in_dir, "geo4", &_geo4size); 231 | _geodb6 = mmap_db(in_dir, "geo6", &_geo6size); 232 | _asdb4 = mmap_db(in_dir, "as4", &_as4size); 233 | _asdb6 = mmap_db(in_dir, "as6", &_as6size); 234 | 235 | for (i=optind; i 2 | #include 3 | #include 4 | 5 | #include "xenoeye.h" 6 | #include "sflow.h" 7 | 8 | static void sflow_parse_payload(uint8_t *end, uint8_t *p); 9 | 10 | #undef LOG 11 | #define LOG(...) \ 12 | do { \ 13 | char _buf[4096]; \ 14 | int _ret = snprintf(_buf, sizeof(_buf), __VA_ARGS__); \ 15 | if (_ret >= (int)(sizeof(_buf))) { \ 16 | fprintf(stderr, \ 17 | "Next line truncated to %d symbols", \ 18 | _ret); \ 19 | } \ 20 | printf("%s [%s, line %d, function %s()]\n", \ 21 | _buf, __FILE__, __LINE__, __func__); \ 22 | } while (0) 23 | 24 | #include "xe-dns.h" 25 | #include "xe-sni.h" 26 | 27 | #define USER_TYPE uint8_t * 28 | 29 | #define ON_ETH(D, V) \ 30 | do { \ 31 | (void)data; \ 32 | char buf[32]; \ 33 | sprintf(buf, "%02x:%02x:%02x:%02x:%02x:%02x", \ 34 | V->h_source[0], V->h_source[1], V->h_source[2], \ 35 | V->h_source[3], V->h_source[4], V->h_source[5]); \ 36 | LOG("\t\t\tEthernet src: %s", buf); \ 37 | sprintf(buf, "%02x:%02x:%02x:%02x:%02x:%02x", \ 38 | V->h_dest[0], V->h_dest[1], V->h_dest[2], \ 39 | V->h_dest[3], V->h_dest[4], V->h_dest[5]); \ 40 | LOG("\t\t\tEthernet dst: %s", buf); \ 41 | LOG("\t\t\tEthernet proto: 0x%x", be16toh(V->h_proto)); \ 42 | } while (0) 43 | 44 | #define ON_VLAN1(D, V) \ 45 | LOG("\t\t\tVLAN %d", be16toh(V->h_vlan_TCI)); 46 | 47 | #define ON_VLAN2(D, V) \ 48 | LOG("\t\t\tVLAN2 %d", be16toh(V->h_vlan_TCI)); 49 | 50 | #define ON_IP(D, V) \ 51 | do { \ 52 | char s[INET_ADDRSTRLEN + 1]; \ 53 | inet_ntop(AF_INET, &V->saddr, s, INET_ADDRSTRLEN); \ 54 | LOG("\t\t\tIPv4 src: %s", s); \ 55 | inet_ntop(AF_INET, &V->daddr, s, INET_ADDRSTRLEN); \ 56 | LOG("\t\t\tIPv4 dst: %s", s); \ 57 | LOG("\t\t\tTOS: 0x%0x", V->tos); \ 58 | LOG("\t\t\tID: %d", be16toh(V->id)); \ 59 | LOG("\t\t\tTTL: %d", V->ttl); \ 60 | LOG("\t\t\tIP protocol: %d", V->protocol); \ 61 | } while (0) 62 | 63 | #define ON_IP6(D, V) \ 64 | do { \ 65 | char s[INET6_ADDRSTRLEN + 1]; \ 66 | inet_ntop(AF_INET6, &V->ip6_src, s, INET6_ADDRSTRLEN); \ 67 | LOG("\t\t\tIPv6 src: %s", s); \ 68 | inet_ntop(AF_INET6, &V->ip6_dst, s, INET6_ADDRSTRLEN); \ 69 | LOG("\t\t\tIPv6 dst: %s", s); \ 70 | LOG("\t\t\tTTL: %d", V->ip6_ctlun.ip6_un1.ip6_un1_hlim); \ 71 | LOG("\t\t\tIP protocol: %d", (int)nexthdr); \ 72 | } while (0) 73 | 74 | #define ON_UDP(D, V) \ 75 | do { \ 76 | LOG("\t\t\tUDP src port: %d", be16toh(V->uh_sport)); \ 77 | LOG("\t\t\tUDP dst port: %d", be16toh(V->uh_dport)); \ 78 | } while (0) 79 | 80 | #define ON_TCP(D, V) \ 81 | do { \ 82 | LOG("\t\t\tTCP src port: %d", be16toh(V->th_sport)); \ 83 | LOG("\t\t\tTCP dst port: %d", be16toh(V->th_dport)); \ 84 | LOG("\t\t\tTCP flags: 0x%0x", V->th_flags); \ 85 | } while (0) 86 | 87 | #define ON_ICMP(D, V) \ 88 | do { \ 89 | LOG("\t\t\tICMP type: %d", V->type); \ 90 | LOG("\t\t\tICMP code: %d", V->code); \ 91 | } while (0) 92 | 93 | #define ON_PAYLOAD(D, P) sflow_parse_payload(D, P); 94 | 95 | #include "rawparse.h" 96 | 97 | #undef ON_ICMP 98 | #undef ON_TCP 99 | #undef ON_UDP 100 | #undef ON_IP 101 | #undef ON_ETH 102 | 103 | static void 104 | sflow_parse_payload(uint8_t *end, uint8_t *p) 105 | { 106 | if (xe_sni(p, end, NULL)) { 107 | return; 108 | } 109 | if (xe_dns(p, end, NULL, NULL)) { 110 | return; 111 | } 112 | } 113 | 114 | static inline int 115 | sf5_eth(struct sfdata *s, uint8_t *p, enum RP_TYPE t, uint32_t header_len) 116 | { 117 | (void)s; 118 | uint8_t *end = p + header_len; 119 | 120 | if (rawpacket_parse(p, end, t, end) 121 | < RP_PARSER_STATE_NO_IP) { 122 | 123 | /* Skip non-IP samples */ 124 | return 0; 125 | } 126 | 127 | return 1; 128 | } 129 | 130 | #include "sflow-impl.h" 131 | 132 | #undef USER_TYPE 133 | #define USER_TYPE uint8_t ** 134 | 135 | #undef ON_PAYLOAD 136 | #define ON_PAYLOAD(D, P) *D = P; 137 | 138 | #define rawpacket_parse rawpacket_parse_sflow 139 | 140 | #include "rawparse.h" 141 | 142 | static void 143 | print_usage(const char *prog_name) 144 | { 145 | LOG("Usage: %s -i eth0 [-f \"udp and port 6543\"]", prog_name); 146 | } 147 | 148 | 149 | int 150 | main(int argc, char *argv[]) 151 | { 152 | int opt; 153 | 154 | char *ifname = NULL; 155 | char *filter = ""; 156 | 157 | pcap_t *p_handle; 158 | char errbuf[PCAP_ERRBUF_SIZE]; 159 | struct bpf_program fp; 160 | 161 | int ret = EXIT_FAILURE; 162 | 163 | while ((opt = getopt(argc, argv, "f:hi:")) != -1) { 164 | switch (opt) { 165 | case 'i': 166 | ifname = optarg; 167 | break; 168 | 169 | case 'f': 170 | filter = optarg; 171 | break; 172 | 173 | case 'h': 174 | default: 175 | print_usage(argv[0]); 176 | return EXIT_FAILURE; 177 | } 178 | } 179 | 180 | if (!ifname) { 181 | LOG("Interface name is required"); 182 | print_usage(argv[0]); 183 | return EXIT_FAILURE; 184 | } 185 | 186 | p_handle = pcap_open_live(ifname, BUFSIZ, 1, 1000, errbuf); 187 | if (p_handle == NULL) { 188 | LOG("Can't open device '%s': %s", ifname, errbuf); 189 | return EXIT_FAILURE; 190 | } 191 | 192 | if (pcap_compile(p_handle, &fp, filter, 193 | 1, PCAP_NETMASK_UNKNOWN) == -1) { 194 | 195 | LOG("Can't parse filter '%s': %s", filter, 196 | pcap_geterr(p_handle)); 197 | goto fail_filter; 198 | } 199 | 200 | if (pcap_setfilter(p_handle, &fp) == -1) { 201 | LOG("Can't set filter '%s': %s", filter, 202 | pcap_geterr(p_handle)); 203 | goto fail_setfilter; 204 | } 205 | 206 | for (;;) { 207 | int rc; 208 | struct pcap_pkthdr *header; 209 | const unsigned char *packet; 210 | struct flow_packet_info fpi; 211 | uint8_t *sflow_data = NULL; 212 | 213 | rc = pcap_next_ex(p_handle, &header, &packet); 214 | if (rc == 1) { 215 | enum RP_PARSER_STATE ps; 216 | uint8_t *ptr = (uint8_t *)packet; 217 | int len; 218 | 219 | ps = rawpacket_parse_sflow(ptr, ptr + header->caplen, 220 | RP_TYPE_ETHER, &sflow_data); 221 | if (ps != RP_PARSER_STATE_OK) { 222 | continue; 223 | } 224 | if (!sflow_data) { 225 | continue; 226 | } 227 | len = header->caplen - (sflow_data - packet); 228 | 229 | memcpy(fpi.rawpacket, sflow_data, len); 230 | sflow_process(NULL, 0, &fpi, len); 231 | } else { 232 | LOG("Error reading the packets: %s", 233 | pcap_geterr(p_handle)); 234 | } 235 | } 236 | 237 | ret = EXIT_SUCCESS; 238 | 239 | fail_setfilter: 240 | pcap_freecode(&fp); 241 | fail_filter: 242 | pcap_close(p_handle); 243 | 244 | return ret; 245 | } 246 | 247 | -------------------------------------------------------------------------------- /devices.c: -------------------------------------------------------------------------------- 1 | /* 2 | * xenoeye 3 | * 4 | * Copyright (c) 2021-2025, Vladimir Misyurov, Michael Kogan 5 | * 6 | * Permission to use, copy, modify, and/or distribute this software for any 7 | * purpose with or without fee is hereby granted, provided that the above 8 | * copyright notice and this permission notice appear in all copies. 9 | * 10 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 11 | * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 12 | * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 13 | * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 14 | * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE 15 | * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 16 | * PERFORMANCE OF THIS SOFTWARE. 17 | */ 18 | #include 19 | #include 20 | 21 | #include "filter.h" 22 | #include "devices.h" 23 | #include "flow-info.h" 24 | #include "aajson/aajson.h" 25 | 26 | struct devices_info 27 | { 28 | struct device *devices; 29 | size_t n_devices; 30 | }; 31 | 32 | static struct devices_info devices = {NULL, 0}; 33 | 34 | static int 35 | config_adjust_devs_size(struct devices_info *devs, size_t idx) 36 | { 37 | struct device *tmp; 38 | 39 | if (devs->n_devices >= (idx + 1)) { 40 | return 1; 41 | } 42 | 43 | tmp = realloc(devs->devices, (idx + 1) * sizeof(struct device)); 44 | if (!tmp) { 45 | LOG("realloc() failed"); 46 | return 0; 47 | } 48 | 49 | devs->devices = tmp; 50 | devs->n_devices = idx + 1; 51 | 52 | /* init new device */ 53 | memset(&devs->devices[devs->n_devices - 1], 0, sizeof(struct device)); 54 | 55 | return 1; 56 | } 57 | 58 | #define STRCMP(A, I, S) strcmp(A->path_stack[I].data.path_item, S) 59 | 60 | static int 61 | config_callback(struct aajson *a, aajson_val *value, void *user) 62 | { 63 | struct devices_info *devs = user; 64 | size_t idx; 65 | 66 | if (a->path_stack_pos < 2) { 67 | return 1; 68 | } 69 | 70 | if (a->path_stack[1].type != AAJSON_PATH_ITEM_ARRAY) { 71 | return 1; 72 | } 73 | 74 | idx = a->path_stack[1].data.array_idx; 75 | 76 | if (!config_adjust_devs_size(devs, idx)) { 77 | return 0; 78 | } 79 | 80 | if (STRCMP(a, 2, "ip") == 0) { 81 | unsigned char buf[sizeof(struct in6_addr)]; 82 | int rc; 83 | 84 | devs->devices[devs->n_devices - 1].use_ip = 1; 85 | 86 | /* FIXME: add IPv6 */ 87 | rc = inet_pton(AF_INET, value->str, buf); 88 | if (rc <= 0) { 89 | LOG("Can't parse IP '%s'", value->str); 90 | return 0; 91 | } 92 | 93 | devs->devices[devs->n_devices - 1].ip_ver = 4; 94 | devs->devices[devs->n_devices - 1].ip = 0; 95 | memcpy(&devs->devices[devs->n_devices - 1].ip, buf, 4); 96 | } 97 | 98 | if (STRCMP(a, 2, "id") == 0) { 99 | devs->devices[devs->n_devices - 1].use_id = 1; 100 | devs->devices[devs->n_devices - 1].id 101 | = htonl(atoi(value->str)); 102 | } 103 | 104 | if (STRCMP(a, 2, "sampling-rate") == 0) { 105 | devs->devices[devs->n_devices - 1].sampling_rate 106 | = atoi(value->str); 107 | } 108 | 109 | if (STRCMP(a, 2, "mark") == 0) { 110 | struct filter_input input; 111 | struct device *d = &devs->devices[devs->n_devices - 1]; 112 | struct filter_expr **tmp = realloc(d->exprs, sizeof(struct filter_expr *) * (d->n_exprs + 1)); 113 | if (!tmp) { 114 | LOG("realloc() failed"); 115 | return 0; 116 | } 117 | d->exprs = tmp; 118 | 119 | memset(&input, 0, sizeof(input)); 120 | input.s = value->str; 121 | 122 | d->exprs[d->n_exprs] = parse_filter(&input); 123 | if (input.error) { 124 | LOG("Can't parse filter '%s'. Parse error: %s", 125 | value->str, input.errmsg); 126 | return 0; 127 | } 128 | 129 | d->n_exprs++; 130 | } 131 | 132 | if (STRCMP(a, 2, "skip-unmarked") == 0) { 133 | if (value->type == AAJSON_VALUE_TRUE) { 134 | devs->devices[devs->n_devices - 1].skip_unmarked = 1; 135 | } 136 | } 137 | 138 | return 1; 139 | } 140 | #undef STRCMP 141 | 142 | int 143 | devices_load(const char *filename) 144 | { 145 | FILE *f; 146 | long len; 147 | struct aajson conf_json; 148 | int ret = 0; 149 | char *file; 150 | 151 | f = fopen(filename, "rb"); 152 | if (!f) { 153 | LOG("Can't open config file '%s'", filename); 154 | goto fail_open; 155 | } 156 | 157 | fseek(f, 0, SEEK_END); 158 | len = ftell(f); 159 | 160 | fseek(f, 0, SEEK_SET); 161 | file = malloc(len); 162 | if (!file) { 163 | LOG("Can't allocate %ld bytes", len); 164 | goto fail_alloc; 165 | } 166 | 167 | if (fread(file, 1, len, f) != (size_t)len) { 168 | LOG("Can't read config file '%s'", filename); 169 | goto fail_read; 170 | } 171 | 172 | aajson_init(&conf_json, file); 173 | aajson_parse(&conf_json, &config_callback, &devices); 174 | 175 | if (conf_json.error) { 176 | LOG("Can't parse config file '%s': line %lu, col %lu: %s", 177 | filename, conf_json.line, conf_json.col, 178 | conf_json.errmsg); 179 | 180 | free(devices.devices); 181 | devices.devices = NULL; 182 | devices.n_devices = 0; 183 | 184 | goto fail_parse; 185 | } 186 | 187 | ret = 1; 188 | 189 | fail_parse: 190 | fail_read: 191 | free(file); 192 | fail_alloc: 193 | fclose(f); 194 | fail_open: 195 | return ret; 196 | } 197 | 198 | 199 | int 200 | device_get_sampling_rate(struct device *d) 201 | { 202 | size_t i; 203 | int found = 0; 204 | 205 | for (i=0; iuse_ip && db->use_id) { 209 | /* check all fields */ 210 | if ((d->ip_ver == db->ip_ver) 211 | && (d->ip == db->ip) && (d->id == db->id)) { 212 | 213 | found = 1; 214 | d->sampling_rate = db->sampling_rate; 215 | break; 216 | } 217 | } else if (db->use_ip) { 218 | /* only IP */ 219 | if ((d->ip_ver == db->ip_ver) && (d->ip == db->ip)) { 220 | found = 1; 221 | d->sampling_rate = db->sampling_rate; 222 | break; 223 | } 224 | } if (db->use_id) { 225 | /* only ID */ 226 | if (d->id == db->id) { 227 | found = 1; 228 | d->sampling_rate = db->sampling_rate; 229 | break; 230 | } 231 | } 232 | } 233 | 234 | return found; 235 | } 236 | 237 | static int 238 | device_get_mark(struct device *d, struct flow_info *fi) 239 | { 240 | size_t i; 241 | int found = 0; 242 | struct device *db; 243 | 244 | d->mark = 0; 245 | 246 | for (i=0; iuse_ip && db->use_id) { 250 | /* check all fields */ 251 | if ((d->ip_ver == db->ip_ver) 252 | && (d->ip == db->ip) && (d->id == db->id)) { 253 | 254 | found = 1; 255 | break; 256 | } 257 | } else if (db->use_ip) { 258 | /* only IP */ 259 | if ((d->ip_ver == db->ip_ver) && (d->ip == db->ip)) { 260 | found = 1; 261 | break; 262 | } 263 | } if (db->use_id) { 264 | /* only ID */ 265 | if (d->id == db->id) { 266 | found = 1; 267 | break; 268 | } 269 | } 270 | } 271 | 272 | if (!found) { 273 | return 0; 274 | } 275 | 276 | d->skip_unmarked = db->skip_unmarked; 277 | for (i=0; in_exprs; i++) { 278 | if (filter_match(db->exprs[i], fi)) { 279 | d->mark++; 280 | } 281 | } 282 | 283 | return 1; 284 | } 285 | 286 | int 287 | device_rules_check(struct flow_info *flow, struct flow_packet_info *fpi) 288 | { 289 | struct device dev; 290 | uint32_t mark; 291 | 292 | /* FIXME: add IPv6 */ 293 | dev.ip_ver = 4; 294 | dev.ip = 0; 295 | memcpy(&dev.ip, &fpi->src_addr_ipv4, 4); 296 | 297 | dev.id = fpi->source_id; 298 | 299 | if (!device_get_mark(&dev, flow)) { 300 | /* device not found */ 301 | return 1; 302 | } 303 | 304 | if (dev.skip_unmarked && (dev.mark == 0)) { 305 | return 0; 306 | } 307 | 308 | mark = htobe32(dev.mark); 309 | memcpy(&flow->dev_mark[0], &mark, sizeof(uint32_t)); 310 | flow->dev_mark_size = sizeof(uint32_t); 311 | flow->has_dev_mark = 1; 312 | 313 | return 1; 314 | } 315 | 316 | -------------------------------------------------------------------------------- /monit-objects-mavg-dump.c: -------------------------------------------------------------------------------- 1 | /* 2 | * xenoeye 3 | * 4 | * Copyright (c) 2022-2025, Vladimir Misyurov, Michael Kogan 5 | * 6 | * Permission to use, copy, modify, and/or distribute this software for any 7 | * purpose with or without fee is hereby granted, provided that the above 8 | * copyright notice and this permission notice appear in all copies. 9 | * 10 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 11 | * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 12 | * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 13 | * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 14 | * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE 15 | * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 16 | * PERFORMANCE OF THIS SOFTWARE. 17 | */ 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | #include "utils.h" 28 | #include "netflow.h" 29 | #include "monit-objects.h" 30 | #include "monit-objects-common.h" 31 | 32 | #define MAVG_VAL(DATUM, I, SIZE) ((struct mavg_val *)&DATUM[SIZE * I]) 33 | 34 | static int 35 | mavg_dump_tr(FILE *out, struct mo_mavg *mavg, tkvdb_tr *tr, 36 | size_t val_itemsize) 37 | { 38 | size_t i; 39 | int ret = 0; 40 | tkvdb_cursor *c; 41 | size_t mem_used; 42 | 43 | MAVG_TYPE wnd_size_ns; 44 | 45 | struct timespec tmsp; 46 | uint64_t time_ns; 47 | 48 | struct mavg_limits *lim_curr = MAVG_LIM_CURR(mavg); 49 | 50 | if (clock_gettime(CLOCK_REALTIME_COARSE, &tmsp) < 0) { 51 | LOG("clock_gettime() failed: %s", strerror(errno)); 52 | } 53 | time_ns = tmsp.tv_sec * 1e9 + tmsp.tv_nsec; 54 | 55 | /* time window in nanoseconds */ 56 | wnd_size_ns = (MAVG_TYPE)mavg->size_secs * 1e9; 57 | 58 | c = tkvdb_cursor_create(tr); 59 | if (!c) { 60 | LOG("tkvdb_cursor_create() failed"); 61 | goto cursor_fail; 62 | } 63 | 64 | if (c->first(c) != TKVDB_OK) { 65 | ret = 1; 66 | goto empty; 67 | } 68 | 69 | /* print memory used by database */ 70 | mem_used = tr->mem(tr); 71 | fprintf(out, "mem used/avail: %luM/%luM (%lu/%lu bytes)\n", 72 | 1 + mem_used / (1024 * 1024), mavg->db_mem / (1024 * 1024), 73 | mem_used, mavg->db_mem); 74 | 75 | /* iterate over all set */ 76 | do { 77 | uint8_t *data = c->key(c); 78 | uint8_t *pval = c->val(c); 79 | int need_print = 0; 80 | 81 | /* array of vals */ 82 | MAVG_TYPE vals[mavg->fieldset.n_aggr]; 83 | 84 | /* calculate vals */ 85 | for (i=0; ifieldset.n_aggr; i++) { 86 | struct mavg_val *val; 87 | 88 | val = MAVG_VAL(pval, i, val_itemsize); 89 | vals[i] = val->val; 90 | 91 | /* correct value */ 92 | if (time_ns > (val->time_prev + wnd_size_ns)) { 93 | vals[i] = 0.0; 94 | } else { 95 | vals[i] = vals[i] - (time_ns - val->time_prev) 96 | / wnd_size_ns * vals[i]; 97 | vals[i] /= (MAVG_TYPE)mavg->size_secs; 98 | } 99 | 100 | if ((uint64_t)vals[i] != 0) { 101 | need_print = 1; 102 | } 103 | } 104 | 105 | /* skip all-zero vals */ 106 | if (!need_print) { 107 | continue; 108 | } 109 | 110 | 111 | for (i=0; ifieldset.n_naggr; i++) { 112 | struct field *fld = &mavg->fieldset.naggr[i]; 113 | monit_object_field_print(fld, out, data, 1); 114 | 115 | data += fld->size; 116 | } 117 | 118 | fprintf(out, " :: "); 119 | 120 | for (i=0; ifieldset.n_aggr; i++) { 121 | size_t j; 122 | struct mavg_val *val; 123 | val = MAVG_VAL(pval, i, val_itemsize); 124 | 125 | fprintf(out, "%lu ", (uint64_t)vals[i]); 126 | 127 | /* limits */ 128 | if (lim_curr->noverlimit > 0) { 129 | fprintf(out, "("); 130 | for (j=0; jnoverlimit; j++) { 131 | /* */ 132 | MAVG_TYPE limit; 133 | limit = atomic_load_explicit( 134 | &val->limits[j], 135 | memory_order_relaxed); 136 | fprintf(out, "%lu ", (uint64_t)limit); 137 | } 138 | fprintf(out, ")"); 139 | } 140 | 141 | if (lim_curr->nunderlimit > 0) { 142 | fprintf(out, "["); 143 | for (j=0; jnunderlimit; j++) { 144 | size_t lidx = j + lim_curr->noverlimit; 145 | MAVG_TYPE limit; 146 | limit = atomic_load_explicit( 147 | &val->limits[lidx], 148 | memory_order_relaxed); 149 | fprintf(out, "%lu ", (uint64_t)limit); 150 | } 151 | fprintf(out, "]"); 152 | } 153 | 154 | } 155 | 156 | fprintf(out, "\n"); 157 | } while (c->next(c) == TKVDB_OK); 158 | 159 | fprintf(out, "\n"); 160 | ret = 1; 161 | empty: 162 | c->free(c); 163 | 164 | cursor_fail: 165 | return ret; 166 | } 167 | 168 | static int 169 | mavg_dump_do(struct mo_mavg *mavg, size_t nthreads, struct monit_object *mo, 170 | int append) 171 | { 172 | FILE *f; 173 | char dump_path[PATH_MAX * 2]; 174 | size_t i; 175 | char timebuf[100]; 176 | time_t t; 177 | 178 | t = time(NULL); 179 | if (t == (time_t)-1) { 180 | LOG("time() failed: %s", strerror(errno)); 181 | return 0; 182 | } 183 | 184 | sprintf(dump_path, append? "%s/%s.adump" : "%s/%s.dump", 185 | mo->dir, mavg->name); 186 | 187 | if (strlen(dump_path) >= PATH_MAX) { 188 | LOG("Filename too big: %s/%s", mo->dir, mavg->name); 189 | return 0; 190 | } 191 | 192 | f = fopen(dump_path, append? "a": "w"); 193 | if (!f) { 194 | LOG("Can't open '%s': %s", dump_path, strerror(errno)); 195 | return 0; 196 | } 197 | 198 | fprintf(f, "%s", ctime_r(&t, timebuf)); 199 | 200 | for (i=0; ithr_data[i].db, 204 | memory_order_relaxed); 205 | 206 | mavg_dump_tr(f, mavg, db, mavg->thr_data[i].val_itemsize); 207 | } 208 | 209 | if (append) { 210 | /* extra empty line */ 211 | fprintf(f, "\n"); 212 | } 213 | 214 | fclose(f); 215 | 216 | return 1; 217 | } 218 | 219 | static int 220 | mavg_dump(struct mo_mavg *mavg, size_t nthreads, struct monit_object *mo) 221 | { 222 | char enabled[PATH_MAX * 2]; 223 | struct stat statbuf; 224 | int dump = 0, append = 0; 225 | 226 | sprintf(enabled, "%s/%s.d", mo->dir, mavg->name); 227 | if (strlen(enabled) >= PATH_MAX) { 228 | LOG("Filename too big: %s/%s", mo->dir, mavg->name); 229 | return 0; 230 | } 231 | 232 | if (stat(enabled, &statbuf) == 0) { 233 | dump = 1; 234 | } 235 | 236 | sprintf(enabled, "%s/%s.a", mo->dir, mavg->name); 237 | if (stat(enabled, &statbuf) == 0) { 238 | append = 1; 239 | } 240 | 241 | if (!dump && !append) { 242 | /* skip */ 243 | return 0; 244 | } 245 | 246 | if (dump) { 247 | mavg_dump_do(mavg, nthreads, mo, 0); 248 | } 249 | 250 | if (append) { 251 | mavg_dump_do(mavg, nthreads, mo, 1); 252 | } 253 | 254 | return 1; 255 | } 256 | 257 | static void 258 | mavg_dump_rec(struct xe_data *globl, struct monit_object *mos, size_t n_mo, 259 | time_t t) 260 | { 261 | size_t i, j; 262 | 263 | for (i=0; inmavg; j++) { 268 | struct mo_mavg *mavg = &mo->mavgs[j]; 269 | 270 | if (mavg->dump_secs == 0) { 271 | /* skip */ 272 | continue; 273 | } 274 | 275 | if ((mavg->last_dump_check + mavg->dump_secs) <= t) { 276 | /* time to dump */ 277 | mavg_dump(mavg, globl->nthreads, mo); 278 | 279 | mavg->last_dump_check = t; 280 | } 281 | } 282 | 283 | if (mo->n_mo) { 284 | mavg_dump_rec(globl, mo->mos, mo->n_mo, t); 285 | } 286 | } 287 | } 288 | 289 | void * 290 | mavg_dump_thread(void *arg) 291 | { 292 | struct xe_data *globl = (struct xe_data *)arg; 293 | 294 | for (;;) { 295 | time_t t; 296 | 297 | if (atomic_load_explicit(&globl->stop, memory_order_relaxed)) { 298 | /* stop */ 299 | break; 300 | } 301 | 302 | t = time(NULL); 303 | if (t == ((time_t)-1)) { 304 | LOG("time() failed: %s", strerror(errno)); 305 | return NULL; 306 | } 307 | 308 | mavg_dump_rec(globl, globl->monit_objects, 309 | globl->nmonit_objects, t); 310 | 311 | sleep(1); 312 | } 313 | 314 | return NULL; 315 | } 316 | 317 | -------------------------------------------------------------------------------- /monit-objects.h: -------------------------------------------------------------------------------- 1 | #ifndef monit_objects_h_included 2 | #define monit_objects_h_included 3 | 4 | #include "xe-debug.h" 5 | #include "aajson/aajson.h" 6 | #include "filter.h" 7 | 8 | #include "tkvdb.h" 9 | 10 | #define FWM_DEFAULT_TIMEOUT 30 11 | 12 | #define MAVG_DEFAULT_SIZE 5 13 | 14 | #define MAVG_DEFAULT_BACK2NORM 30 15 | 16 | #define MAVG_DEFAULT_DB_SIZE (1024*1024*256) 17 | 18 | #define MAVG_SCRIPT_STR_SIZE (10*1024) 19 | 20 | /*#define MAVG_TYPE __float128*/ 21 | #define MAVG_TYPE double 22 | 23 | #define CLSF_DEFAULT_TIMEOUT 30 24 | #define CLASSES_MAX 5 25 | 26 | /* helper for classes processing */ 27 | #define FOR_LIST_OF_CLASSES \ 28 | DO(0, class0) \ 29 | DO(1, class1) \ 30 | DO(2, class2) \ 31 | DO(3, class3) \ 32 | DO(4, class4) 33 | 34 | struct xe_data; 35 | struct flow_info; 36 | 37 | struct two_banks_db 38 | { 39 | tkvdb_tr *bank[2]; 40 | 41 | /* current bank */ 42 | atomic_size_t idx; 43 | }; 44 | 45 | struct mo_fieldset 46 | { 47 | /* all fields */ 48 | size_t n; 49 | struct field *fields; 50 | 51 | /* key fields (non-aggregable, without packets/octets/etc) */ 52 | size_t n_naggr; 53 | struct field *naggr; 54 | 55 | /* aggregable fields */ 56 | size_t n_aggr; 57 | struct field *aggr; 58 | }; 59 | 60 | struct fwm_thread_data 61 | { 62 | /* using two banks */ 63 | tkvdb_tr *trs[2]; 64 | 65 | /* current bank */ 66 | tkvdb_tr *_Atomic tr; 67 | 68 | uint8_t *key; 69 | uint64_t *val; 70 | 71 | size_t keysize, valsize; 72 | }; 73 | 74 | struct mo_fwm 75 | { 76 | int is_extended; 77 | atomic_int is_active; 78 | 79 | char name[TOKEN_MAX_SIZE]; 80 | struct mo_fieldset fieldset; 81 | 82 | time_t last_export; 83 | int time; 84 | 85 | int limit; 86 | 87 | int dont_create_index; 88 | 89 | /* window has DNS/SNI, processed in a special way */ 90 | int has_dns_field; 91 | int has_sni_field; 92 | 93 | /* each thread has it's own data */ 94 | struct fwm_thread_data *thread_data; 95 | }; 96 | 97 | 98 | /* classification */ 99 | struct classification_thread_data 100 | { 101 | /* using two banks */ 102 | tkvdb_tr *trs[2]; 103 | 104 | /* current bank index */ 105 | atomic_size_t tr_idx; 106 | 107 | uint8_t *key; 108 | uint64_t val; 109 | 110 | size_t keysize; 111 | }; 112 | 113 | struct mo_classification 114 | { 115 | int id; 116 | 117 | time_t last_export; 118 | int time; 119 | unsigned int top_percents; 120 | 121 | size_t nfields; 122 | struct field *fields; 123 | 124 | struct field *val; 125 | 126 | /* each thread has it's own data */ 127 | struct classification_thread_data *thread_data; 128 | 129 | /* classidier db */ 130 | struct two_banks_db db; 131 | }; 132 | 133 | 134 | /* moving average */ 135 | struct mavg_val 136 | { 137 | _Atomic MAVG_TYPE val; 138 | _Atomic uint64_t time_prev; 139 | 140 | /* growing array (noverlimit + nunderlimit items) */ 141 | _Atomic MAVG_TYPE limits[1]; 142 | }; 143 | 144 | struct mavg_thread_data 145 | { 146 | /* atomic pointer to database */ 147 | tkvdb_tr *_Atomic db; 148 | int db_is_full; 149 | 150 | uint8_t *key; 151 | uint8_t *val; /* array of struct mavg_val */ 152 | 153 | size_t keysize, valsize, val_itemsize, key_fullsize; 154 | 155 | /* per-thread database of overlimited items, 2 banks */ 156 | tkvdb_tr *ovr_db[2]; 157 | }; 158 | 159 | struct mavg_limit_ext_stat 160 | { 161 | char mo_name[TOKEN_MAX_SIZE]; 162 | char name[TOKEN_MAX_SIZE]; 163 | _Atomic int *ptr; 164 | }; 165 | 166 | struct mavg_limit 167 | { 168 | char name[PATH_MAX]; 169 | char file[PATH_MAX]; 170 | 171 | MAVG_TYPE back2norm_time_ns; 172 | 173 | char action_script[MAVG_SCRIPT_STR_SIZE]; 174 | char back2norm_script[MAVG_SCRIPT_STR_SIZE]; 175 | 176 | /* extended statistics on overlimit */ 177 | struct mavg_limit_ext_stat *ext_stat; 178 | size_t n_ext_stat; 179 | 180 | tkvdb_tr *db; 181 | 182 | /* array of defaults */ 183 | MAVG_TYPE *def; 184 | }; 185 | 186 | 187 | struct mavg_limits 188 | { 189 | struct mavg_limit *overlimit; 190 | size_t noverlimit; 191 | 192 | struct mavg_limit *underlimit; 193 | size_t nunderlimit; 194 | }; 195 | 196 | enum MAVG_LIM_STATE 197 | { 198 | MAVG_LIM_NEW, 199 | MAVG_LIM_UPDATE, 200 | MAVG_LIM_ALMOST_GONE, 201 | MAVG_LIM_GONE 202 | }; 203 | 204 | struct mavg_lim_data 205 | { 206 | enum MAVG_LIM_STATE state; 207 | uint64_t time_dump, time_last, time_back2norm; 208 | MAVG_TYPE val; 209 | MAVG_TYPE limit; 210 | MAVG_TYPE back2norm_time_ns; 211 | }; 212 | 213 | struct mo_mavg 214 | { 215 | char notif_pfx[PATH_MAX]; /* prefix for notification files */ 216 | 217 | char name[TOKEN_MAX_SIZE]; 218 | unsigned int size_secs; 219 | struct mo_fieldset fieldset; 220 | unsigned int dump_secs; 221 | 222 | time_t last_dump_check; 223 | 224 | uint64_t start_ns; 225 | 226 | /* limits */ 227 | struct mavg_limits lim[2]; 228 | /* atomic index of current limits bank */ 229 | atomic_size_t lim_curr_idx; 230 | 231 | /* per-mavg database of overlimited items */ 232 | tkvdb_tr *ovrerlm_db; 233 | 234 | /* underlimited items */ 235 | tkvdb_tr *underlm_db; 236 | 237 | size_t db_mem; 238 | 239 | /* each thread has it's own data */ 240 | size_t nthreads; 241 | struct mavg_thread_data *thr_data; 242 | }; 243 | 244 | struct monit_object 245 | { 246 | char dir[PATH_MAX]; 247 | char name[PATH_MAX]; 248 | 249 | /* path to config file */ 250 | char mo_path[PATH_MAX]; 251 | /* modification time */ 252 | struct timespec modif_time; 253 | /* when reloading config this is not 0 */ 254 | int is_reloading; 255 | 256 | struct filter_expr *expr; 257 | 258 | struct xe_debug debug; 259 | 260 | /* fixed windows in memory */ 261 | size_t nfwm; 262 | struct mo_fwm *fwms; 263 | 264 | /* moving averages */ 265 | size_t nmavg; 266 | struct mo_mavg *mavgs; 267 | 268 | /* classifications */ 269 | size_t nclassifications; 270 | struct mo_classification *classifications; 271 | 272 | /* sFlow packet payload parsing */ 273 | int payload_parse_dns; 274 | int payload_parse_sni; 275 | 276 | /* hierarchical objects */ 277 | size_t n_mo; 278 | struct monit_object *mos; 279 | }; 280 | 281 | 282 | int monit_objects_init(struct xe_data *data); 283 | int monit_objects_free(struct xe_data *data); 284 | 285 | int monit_objects_reload(struct xe_data *data); 286 | 287 | int monit_object_match(struct monit_object *mo, struct flow_info *fi); 288 | int monit_object_process_nf(struct xe_data *globl, struct monit_object *mo, 289 | size_t thread_id, uint64_t time_ns, struct flow_info *flow); 290 | 291 | void monit_object_field_print(struct field *fld, FILE *f, uint8_t *data, 292 | int print_spaces); 293 | void monit_object_field_print_str(struct field *fld, char *str, uint8_t *data, 294 | int print_spaces); 295 | 296 | void monit_object_key_add_fld(struct field *fld, uint8_t *key, 297 | struct flow_info *flow); 298 | 299 | /* fixed windows in memory */ 300 | int fwm_config(struct aajson *a, aajson_val *value, struct monit_object *mo); 301 | int fwm_fields_init(size_t nthreads, struct mo_fwm *fwm); 302 | void *fwm_bg_thread(void *); 303 | 304 | /* moving averages */ 305 | int mavg_config(struct aajson *a, aajson_val *value, struct monit_object *mo); 306 | int mavg_fields_init(size_t nthreads, struct mo_mavg *mavg); 307 | int mavg_limits_init(struct mo_mavg *mavg, int is_reloading); 308 | int mavg_limits_file_load(struct mo_mavg *mavg, struct mavg_limit *l); 309 | void monit_objects_mavg_link_ext_stat(struct xe_data *globl); 310 | int monit_object_mavg_process_nf(struct xe_data *globl, 311 | struct monit_object *mo, size_t thread_id, 312 | uint64_t time_ns, struct flow_info *flow); 313 | void mavg_limits_update(struct xe_data *globl, struct monit_object *mo); 314 | void mavg_limits_free(struct mo_mavg *mavg); 315 | 316 | /* classification */ 317 | void *classification_bg_thread(void *); 318 | int classification_config(struct aajson *a, aajson_val *value, 319 | struct monit_object *mo); 320 | int classification_fields_init(size_t nthreads, 321 | struct mo_classification *clsf); 322 | int classification_process_nf(struct monit_object *mo, size_t thread_id, 323 | struct flow_info *flow); 324 | 325 | void *mavg_dump_thread(void *); 326 | void *mavg_act_thread(void *); 327 | void *mavg_check_underlimit_thread(void *); 328 | 329 | int act(struct mo_mavg *mw, tkvdb_tr *db, MAVG_TYPE wnd_size_ns, char *mo_name, 330 | int is_overlim); 331 | 332 | #endif 333 | 334 | -------------------------------------------------------------------------------- /pcapture.c: -------------------------------------------------------------------------------- 1 | /* 2 | * xenoeye 3 | * 4 | * Copyright (c) 2020-2024, Vladimir Misyurov, Michael Kogan 5 | * 6 | * Permission to use, copy, modify, and/or distribute this software for any 7 | * purpose with or without fee is hereby granted, provided that the above 8 | * copyright notice and this permission notice appear in all copies. 9 | * 10 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 11 | * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 12 | * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 13 | * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 14 | * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE 15 | * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 16 | * PERFORMANCE OF THIS SOFTWARE. 17 | */ 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | 31 | #include "utils.h" 32 | #include "xenoeye.h" 33 | #include "netflow.h" 34 | #include "sflow.h" 35 | #include "flow-info.h" 36 | 37 | 38 | #define SIZE_UDP 8 /* length of UDP header */ 39 | 40 | /* ethernet headers are always exactly 14 bytes */ 41 | #define SIZE_ETHERNET 14 42 | 43 | 44 | /* Ethernet addresses are 6 bytes */ 45 | #define ETHER_ADDR_LEN 6 46 | 47 | /* Ethernet header */ 48 | struct sniff_ethernet { 49 | u_char ether_dhost[ETHER_ADDR_LEN]; /* Destination host address */ 50 | u_char ether_shost[ETHER_ADDR_LEN]; /* Source host address */ 51 | u_short ether_type; /* IP? ARP? RARP? etc */ 52 | } __attribute__ ((packed)); 53 | 54 | /* IP header */ 55 | struct sniff_ip { 56 | u_char ip_vhl; /* version << 4 | header length >> 2 */ 57 | u_char ip_tos; /* type of service */ 58 | u_short ip_len; /* total length */ 59 | u_short ip_id; /* identification */ 60 | u_short ip_off; /* fragment offset field */ 61 | #define IP_RF 0x8000 /* reserved fragment flag */ 62 | #define IP_DF 0x4000 /* don't fragment flag */ 63 | #define IP_MF 0x2000 /* more fragments flag */ 64 | #define IP_OFFMASK 0x1fff /* mask for fragmenting bits */ 65 | u_char ip_ttl; /* time to live */ 66 | u_char ip_p; /* protocol */ 67 | u_short ip_sum; /* checksum */ 68 | 69 | /* source and dest address */ 70 | struct in_addr ip_src, ip_dst; 71 | } __attribute__ ((packed)); 72 | #define IP_HL(ip) (((ip)->ip_vhl) & 0x0f) 73 | #define IP_V(ip) (((ip)->ip_vhl) >> 4) 74 | 75 | /* TCP header */ 76 | typedef u_int tcp_seq; 77 | 78 | struct sniff_tcp { 79 | u_short th_sport; /* source port */ 80 | u_short th_dport; /* destination port */ 81 | tcp_seq th_seq; /* sequence number */ 82 | tcp_seq th_ack; /* acknowledgement number */ 83 | u_char th_offx2; /* data offset, rsvd */ 84 | #define TH_OFF(th) (((th)->th_offx2 & 0xf0) >> 4) 85 | u_char th_flags; 86 | #define TH_FIN 0x01 87 | #define TH_SYN 0x02 88 | #define TH_RST 0x04 89 | #define TH_PUSH 0x08 90 | #define TH_ACK 0x10 91 | #define TH_URG 0x20 92 | #define TH_ECE 0x40 93 | #define TH_CWR 0x80 94 | #define TH_FLAGS (TH_FIN|TH_SYN|TH_RST|TH_ACK|TH_URG|TH_ECE|TH_CWR) 95 | u_short th_win; /* window */ 96 | u_short th_sum; /* checksum */ 97 | u_short th_urp; /* urgent pointer */ 98 | } __attribute__ ((packed)); 99 | 100 | /* UDP header */ 101 | 102 | struct sniff_udp 103 | { 104 | u_short uh_sport; /* source port */ 105 | u_short uh_dport; /* destination port */ 106 | u_short uh_ulen; /* udp length */ 107 | u_short uh_sum; /* udp checksum */ 108 | } __attribute__ ((packed)); 109 | 110 | 111 | static void 112 | pcap_packet(struct capture_thread_params *params, 113 | struct pcap_pkthdr *header, const unsigned char *packet) 114 | { 115 | /*const struct sniff_ethernet *ethernet;*/ /* The ethernet header [1] */ 116 | const struct sniff_ip *ip; /* The IP header */ 117 | const struct sniff_udp *udp; /* The UDP header */ 118 | const unsigned char *payload; /* Packet payload */ 119 | 120 | struct flow_packet_info pkt; /* flow packet */ 121 | 122 | int size_ip; 123 | int size_payload; 124 | 125 | (void)header; 126 | /* define ethernet header */ 127 | /*ethernet = (struct sniff_ethernet *)(packet);*/ 128 | 129 | /* define/compute ip header offset */ 130 | ip = (struct sniff_ip *)(packet + SIZE_ETHERNET); 131 | if (!packet) { 132 | return; 133 | } 134 | size_ip = IP_HL(ip)*4; 135 | if (size_ip < 20) { 136 | /*LOG("Invalid IP header length: %u bytes", size_ip);*/ 137 | return; 138 | } 139 | 140 | /* determine protocol */ 141 | if (ip->ip_p != IPPROTO_UDP) { 142 | return; 143 | } 144 | 145 | /* 146 | * OK, this packet is UDP. 147 | */ 148 | 149 | /* define/compute udp header offset */ 150 | udp = (struct sniff_udp*)(packet + SIZE_ETHERNET + size_ip); 151 | 152 | pkt.src_addr_ipv4 = ip->ip_src.s_addr; 153 | 154 | /* define/compute udp payload (segment) offset */ 155 | payload = (u_char *)(packet + SIZE_ETHERNET + size_ip + SIZE_UDP); 156 | 157 | /* compute udp payload (segment) size */ 158 | size_payload = ntohs(ip->ip_len) - (size_ip + SIZE_UDP); 159 | if (size_payload > ntohs(udp->uh_ulen)) { 160 | size_payload = ntohs(udp->uh_ulen); 161 | } 162 | 163 | memcpy(pkt.rawpacket, payload, size_payload); 164 | if (params->type == FLOW_TYPE_NETFLOW) { 165 | if (netflow_process(params->data, params->thread_idx, &pkt, 166 | size_payload)) { 167 | /* ok */ 168 | /*data->packets_processed++;*/ 169 | } 170 | } else { 171 | /* sflow */ 172 | sflow_process(params->data, params->thread_idx, &pkt, 173 | size_payload); 174 | } 175 | } 176 | 177 | static void * 178 | pcapture_thread(void *arg) 179 | { 180 | struct capture_thread_params params, *params_ptr; 181 | 182 | struct pcap_pkthdr *header; 183 | const unsigned char *packet; 184 | int rc; 185 | /*int64_t sec = 0;*/ 186 | 187 | params_ptr = (struct capture_thread_params *)arg; 188 | params = *params_ptr; 189 | free(params_ptr); 190 | 191 | LOG("Starting collector thread on interface '%s', filter '%s'", 192 | params.cap->iface, params.cap->filter); 193 | 194 | for (;;) { 195 | rc = pcap_next_ex(params.cap->pcap_handle, &header, &packet); 196 | if (rc >= 0) { 197 | pcap_packet(¶ms, header, packet); 198 | } else { 199 | LOG("Error reading the packets: %s", 200 | pcap_geterr(params.cap->pcap_handle)); 201 | } 202 | 203 | #if 0 204 | { 205 | struct timespec ts; 206 | struct pcap_stat ps; 207 | 208 | clock_gettime(CLOCK_MONOTONIC_COARSE, &ts); 209 | if ((ts.tv_sec / 10) != sec) { 210 | pcap_stats(params.cap->pcap_handle, &ps); 211 | LOG("thread: %lu, recv: %u, drop: %u, ifdrop: %u", 212 | params.thread_idx, 213 | ps.ps_recv, ps.ps_drop, ps.ps_ifdrop); 214 | sec = ts.tv_sec / 10; 215 | } 216 | } 217 | #endif 218 | } 219 | return NULL; 220 | } 221 | 222 | int 223 | pcapture_start(struct xe_data *data, struct capture *cap, size_t thread_idx, 224 | enum FLOW_TYPE type) 225 | { 226 | struct capture_thread_params *params; 227 | char errbuf[PCAP_ERRBUF_SIZE]; 228 | struct bpf_program fp; 229 | int thread_err; 230 | 231 | params = malloc(sizeof(struct capture_thread_params)); 232 | if (!params) { 233 | LOG("malloc() failed"); 234 | goto fail_alloc; 235 | } 236 | params->data = data; 237 | params->thread_idx = thread_idx; 238 | params->cap = cap; 239 | params->type = type; 240 | 241 | cap->pcap_handle = pcap_open_live(cap->iface, BUFSIZ, 1, 1000, errbuf); 242 | if (cap->pcap_handle == NULL) { 243 | LOG("Couldn't open device %s: %s", cap->iface, errbuf); 244 | 245 | goto fail_pcap_open; 246 | } 247 | 248 | if (pcap_compile(cap->pcap_handle, &fp, cap->filter, 249 | 1, PCAP_NETMASK_UNKNOWN) == -1) { 250 | 251 | LOG("Couldn't parse filter '%s': %s", cap->filter, 252 | pcap_geterr(cap->pcap_handle)); 253 | 254 | goto fail_compile; 255 | } 256 | 257 | if (pcap_setfilter(cap->pcap_handle, &fp) == -1) { 258 | LOG("Couldn't install filter %s: %s", cap->filter, 259 | pcap_geterr(cap->pcap_handle)); 260 | 261 | goto fail_setfilter; 262 | } 263 | 264 | thread_err = pthread_create(&cap->tid, NULL, &pcapture_thread, params); 265 | 266 | if (thread_err) { 267 | LOG("Can't start thread: %s", strerror(thread_err)); 268 | goto fail_thread; 269 | } 270 | 271 | return 1; 272 | 273 | /* errors */ 274 | fail_thread: 275 | fail_setfilter: 276 | fail_compile: 277 | pcap_close(cap->pcap_handle); 278 | fail_pcap_open: 279 | free(params); 280 | fail_alloc: 281 | return 0; 282 | } 283 | 284 | -------------------------------------------------------------------------------- /lxc/grafana/Routers.json: -------------------------------------------------------------------------------- 1 | { 2 | "__inputs": [ 3 | { 4 | "name": "DS_GRAFANA-POSTGRESQL-DATASOURCE", 5 | "label": "grafana-postgresql-datasource", 6 | "description": "", 7 | "type": "datasource", 8 | "pluginId": "grafana-postgresql-datasource", 9 | "pluginName": "PostgreSQL" 10 | } 11 | ], 12 | "__elements": {}, 13 | "__requires": [ 14 | { 15 | "type": "grafana", 16 | "id": "grafana", 17 | "name": "Grafana", 18 | "version": "11.5.1" 19 | }, 20 | { 21 | "type": "datasource", 22 | "id": "grafana-postgresql-datasource", 23 | "name": "PostgreSQL", 24 | "version": "1.0.0" 25 | }, 26 | { 27 | "type": "panel", 28 | "id": "netsage-sankey-panel", 29 | "name": "Sankey Panel", 30 | "version": "1.1.3" 31 | }, 32 | { 33 | "type": "panel", 34 | "id": "timeseries", 35 | "name": "Time series", 36 | "version": "" 37 | } 38 | ], 39 | "annotations": { 40 | "list": [ 41 | { 42 | "builtIn": 1, 43 | "datasource": { 44 | "type": "grafana", 45 | "uid": "-- Grafana --" 46 | }, 47 | "enable": true, 48 | "hide": true, 49 | "iconColor": "rgba(0, 211, 255, 1)", 50 | "name": "Annotations & Alerts", 51 | "type": "dashboard" 52 | } 53 | ] 54 | }, 55 | "editable": true, 56 | "fiscalYearStartMonth": 0, 57 | "graphTooltip": 0, 58 | "id": null, 59 | "links": [ 60 | { 61 | "asDropdown": true, 62 | "icon": "external link", 63 | "includeVars": true, 64 | "keepTime": true, 65 | "tags": [], 66 | "targetBlank": false, 67 | "title": "", 68 | "tooltip": "", 69 | "type": "dashboards", 70 | "url": "" 71 | } 72 | ], 73 | "panels": [ 74 | { 75 | "datasource": { 76 | "type": "grafana-postgresql-datasource", 77 | "uid": "${DS_GRAFANA-POSTGRESQL-DATASOURCE}" 78 | }, 79 | "fieldConfig": { 80 | "defaults": { 81 | "color": { 82 | "mode": "palette-classic" 83 | }, 84 | "custom": { 85 | "axisBorderShow": false, 86 | "axisCenteredZero": false, 87 | "axisColorMode": "text", 88 | "axisLabel": "", 89 | "axisPlacement": "auto", 90 | "barAlignment": 0, 91 | "barWidthFactor": 0.9, 92 | "drawStyle": "bars", 93 | "fillOpacity": 50, 94 | "gradientMode": "none", 95 | "hideFrom": { 96 | "legend": false, 97 | "tooltip": false, 98 | "viz": false 99 | }, 100 | "insertNulls": false, 101 | "lineInterpolation": "linear", 102 | "lineWidth": 1, 103 | "pointSize": 5, 104 | "scaleDistribution": { 105 | "type": "linear" 106 | }, 107 | "showPoints": "auto", 108 | "spanNulls": false, 109 | "stacking": { 110 | "group": "A", 111 | "mode": "normal" 112 | }, 113 | "thresholdsStyle": { 114 | "mode": "off" 115 | } 116 | }, 117 | "mappings": [], 118 | "thresholds": { 119 | "mode": "absolute", 120 | "steps": [ 121 | { 122 | "color": "green", 123 | "value": null 124 | }, 125 | { 126 | "color": "red", 127 | "value": 80 128 | } 129 | ] 130 | }, 131 | "unit": "binbps" 132 | }, 133 | "overrides": [] 134 | }, 135 | "gridPos": { 136 | "h": 11, 137 | "w": 24, 138 | "x": 0, 139 | "y": 0 140 | }, 141 | "id": 1, 142 | "options": { 143 | "legend": { 144 | "calcs": [], 145 | "displayMode": "list", 146 | "placement": "bottom", 147 | "showLegend": true 148 | }, 149 | "tooltip": { 150 | "hideZeros": false, 151 | "mode": "single", 152 | "sort": "none" 153 | } 154 | }, 155 | "pluginVersion": "11.5.1", 156 | "targets": [ 157 | { 158 | "datasource": { 159 | "type": "grafana-postgresql-datasource", 160 | "uid": "${DS_GRAFANA-POSTGRESQL-DATASOURCE}" 161 | }, 162 | "editorMode": "code", 163 | "format": "table", 164 | "rawQuery": true, 165 | "rawSql": "select tm as time, val as \"router\", name from xe_rep('\"${mo}-ingress_router\"', 'dev_ip', 'octets', '/15*8', $$ $__timeFilter(time) $$, 20);", 166 | "refId": "A", 167 | "sql": { 168 | "columns": [ 169 | { 170 | "parameters": [], 171 | "type": "function" 172 | } 173 | ], 174 | "groupBy": [ 175 | { 176 | "property": { 177 | "type": "string" 178 | }, 179 | "type": "groupBy" 180 | } 181 | ], 182 | "limit": 50 183 | } 184 | } 185 | ], 186 | "title": "Routers", 187 | "transformations": [ 188 | { 189 | "id": "prepareTimeSeries", 190 | "options": { 191 | "format": "multi" 192 | } 193 | } 194 | ], 195 | "type": "timeseries" 196 | }, 197 | { 198 | "datasource": { 199 | "type": "grafana-postgresql-datasource", 200 | "uid": "${DS_GRAFANA-POSTGRESQL-DATASOURCE}" 201 | }, 202 | "fieldConfig": { 203 | "defaults": { 204 | "color": { 205 | "mode": "thresholds" 206 | }, 207 | "mappings": [], 208 | "thresholds": { 209 | "mode": "absolute", 210 | "steps": [ 211 | { 212 | "color": "green", 213 | "value": null 214 | }, 215 | { 216 | "color": "red", 217 | "value": 80 218 | } 219 | ] 220 | }, 221 | "unit": "binbps" 222 | }, 223 | "overrides": [] 224 | }, 225 | "gridPos": { 226 | "h": 22, 227 | "w": 24, 228 | "x": 0, 229 | "y": 11 230 | }, 231 | "id": 2, 232 | "options": { 233 | "color": "blue", 234 | "iteration": 7, 235 | "labelSize": 15, 236 | "monochrome": false, 237 | "nodeColor": "grey", 238 | "nodePadding": 20, 239 | "nodeWidth": 10 240 | }, 241 | "pluginVersion": "1.1.3", 242 | "targets": [ 243 | { 244 | "editorMode": "code", 245 | "format": "table", 246 | "rawQuery": true, 247 | "rawSql": "select\n coalesce(src_ifidx::text, 'Other') as in_port,\n coalesce(dev_ip::text, 'Other') as router,\n coalesce(dst_ifidx::text, 'Other') as out_port,\n sum(octets) as value\nfrom \"${mo}-ingress_routerp\"\nwhere $__timeFilter(time)\ngroup by in_port, router, out_port\norder by value desc\nlimit 15", 248 | "refId": "A", 249 | "sql": { 250 | "columns": [ 251 | { 252 | "parameters": [], 253 | "type": "function" 254 | } 255 | ], 256 | "groupBy": [ 257 | { 258 | "property": { 259 | "type": "string" 260 | }, 261 | "type": "groupBy" 262 | } 263 | ], 264 | "limit": 50 265 | }, 266 | "datasource": { 267 | "type": "grafana-postgresql-datasource", 268 | "uid": "${DS_GRAFANA-POSTGRESQL-DATASOURCE}" 269 | } 270 | } 271 | ], 272 | "title": "Routers and ports", 273 | "type": "netsage-sankey-panel" 274 | } 275 | ], 276 | "refresh": "", 277 | "schemaVersion": 40, 278 | "tags": [], 279 | "templating": { 280 | "list": [ 281 | { 282 | "allowCustomValue": false, 283 | "current": {}, 284 | "definition": "select left(tablename, length(tablename) - length('ingress_total') - 1) from pg_catalog.pg_tables where tablename like '%ingress_total'", 285 | "name": "mo", 286 | "options": [], 287 | "query": "select left(tablename, length(tablename) - length('ingress_total') - 1) from pg_catalog.pg_tables where tablename like '%ingress_total'", 288 | "refresh": 1, 289 | "regex": "", 290 | "type": "query" 291 | } 292 | ] 293 | }, 294 | "time": { 295 | "from": "now-6h", 296 | "to": "now" 297 | }, 298 | "timepicker": {}, 299 | "timezone": "browser", 300 | "title": "Routers", 301 | "uid": "cecfusxl0gb28d", 302 | "version": 8, 303 | "weekStart": "" 304 | } 305 | -------------------------------------------------------------------------------- /lxc/grafana/Routers6.json: -------------------------------------------------------------------------------- 1 | { 2 | "__inputs": [ 3 | { 4 | "name": "DS_GRAFANA-POSTGRESQL-DATASOURCE", 5 | "label": "grafana-postgresql-datasource", 6 | "description": "", 7 | "type": "datasource", 8 | "pluginId": "grafana-postgresql-datasource", 9 | "pluginName": "PostgreSQL" 10 | } 11 | ], 12 | "__elements": {}, 13 | "__requires": [ 14 | { 15 | "type": "grafana", 16 | "id": "grafana", 17 | "name": "Grafana", 18 | "version": "11.5.1" 19 | }, 20 | { 21 | "type": "datasource", 22 | "id": "grafana-postgresql-datasource", 23 | "name": "PostgreSQL", 24 | "version": "1.0.0" 25 | }, 26 | { 27 | "type": "panel", 28 | "id": "netsage-sankey-panel", 29 | "name": "Sankey Panel", 30 | "version": "1.1.3" 31 | }, 32 | { 33 | "type": "panel", 34 | "id": "timeseries", 35 | "name": "Time series", 36 | "version": "" 37 | } 38 | ], 39 | "annotations": { 40 | "list": [ 41 | { 42 | "builtIn": 1, 43 | "datasource": { 44 | "type": "grafana", 45 | "uid": "-- Grafana --" 46 | }, 47 | "enable": true, 48 | "hide": true, 49 | "iconColor": "rgba(0, 211, 255, 1)", 50 | "name": "Annotations & Alerts", 51 | "type": "dashboard" 52 | } 53 | ] 54 | }, 55 | "editable": true, 56 | "fiscalYearStartMonth": 0, 57 | "graphTooltip": 0, 58 | "id": null, 59 | "links": [ 60 | { 61 | "asDropdown": true, 62 | "icon": "external link", 63 | "includeVars": true, 64 | "keepTime": true, 65 | "tags": [], 66 | "targetBlank": false, 67 | "title": "", 68 | "tooltip": "", 69 | "type": "dashboards", 70 | "url": "" 71 | } 72 | ], 73 | "panels": [ 74 | { 75 | "datasource": { 76 | "type": "grafana-postgresql-datasource", 77 | "uid": "${DS_GRAFANA-POSTGRESQL-DATASOURCE}" 78 | }, 79 | "fieldConfig": { 80 | "defaults": { 81 | "color": { 82 | "mode": "palette-classic" 83 | }, 84 | "custom": { 85 | "axisBorderShow": false, 86 | "axisCenteredZero": false, 87 | "axisColorMode": "text", 88 | "axisLabel": "", 89 | "axisPlacement": "auto", 90 | "barAlignment": 0, 91 | "barWidthFactor": 0.9, 92 | "drawStyle": "bars", 93 | "fillOpacity": 50, 94 | "gradientMode": "none", 95 | "hideFrom": { 96 | "legend": false, 97 | "tooltip": false, 98 | "viz": false 99 | }, 100 | "insertNulls": false, 101 | "lineInterpolation": "linear", 102 | "lineWidth": 1, 103 | "pointSize": 5, 104 | "scaleDistribution": { 105 | "type": "linear" 106 | }, 107 | "showPoints": "auto", 108 | "spanNulls": false, 109 | "stacking": { 110 | "group": "A", 111 | "mode": "normal" 112 | }, 113 | "thresholdsStyle": { 114 | "mode": "off" 115 | } 116 | }, 117 | "mappings": [], 118 | "thresholds": { 119 | "mode": "absolute", 120 | "steps": [ 121 | { 122 | "color": "green", 123 | "value": null 124 | }, 125 | { 126 | "color": "red", 127 | "value": 80 128 | } 129 | ] 130 | }, 131 | "unit": "binbps" 132 | }, 133 | "overrides": [] 134 | }, 135 | "gridPos": { 136 | "h": 11, 137 | "w": 24, 138 | "x": 0, 139 | "y": 0 140 | }, 141 | "id": 1, 142 | "options": { 143 | "legend": { 144 | "calcs": [], 145 | "displayMode": "list", 146 | "placement": "bottom", 147 | "showLegend": true 148 | }, 149 | "tooltip": { 150 | "hideZeros": false, 151 | "mode": "single", 152 | "sort": "none" 153 | } 154 | }, 155 | "pluginVersion": "11.5.1", 156 | "targets": [ 157 | { 158 | "datasource": { 159 | "type": "grafana-postgresql-datasource", 160 | "uid": "${DS_GRAFANA-POSTGRESQL-DATASOURCE}" 161 | }, 162 | "editorMode": "code", 163 | "format": "table", 164 | "rawQuery": true, 165 | "rawSql": "select tm as time, val as \"router\", name from xe_rep('\"${mo}-ingress6_router\"', 'dev_ip', 'octets', '/15*8', $$ $__timeFilter(time) $$, 20);", 166 | "refId": "A", 167 | "sql": { 168 | "columns": [ 169 | { 170 | "parameters": [], 171 | "type": "function" 172 | } 173 | ], 174 | "groupBy": [ 175 | { 176 | "property": { 177 | "type": "string" 178 | }, 179 | "type": "groupBy" 180 | } 181 | ], 182 | "limit": 50 183 | } 184 | } 185 | ], 186 | "title": "Routers", 187 | "transformations": [ 188 | { 189 | "id": "prepareTimeSeries", 190 | "options": { 191 | "format": "multi" 192 | } 193 | } 194 | ], 195 | "type": "timeseries" 196 | }, 197 | { 198 | "datasource": { 199 | "type": "grafana-postgresql-datasource", 200 | "uid": "${DS_GRAFANA-POSTGRESQL-DATASOURCE}" 201 | }, 202 | "fieldConfig": { 203 | "defaults": { 204 | "color": { 205 | "mode": "thresholds" 206 | }, 207 | "mappings": [], 208 | "thresholds": { 209 | "mode": "absolute", 210 | "steps": [ 211 | { 212 | "color": "green", 213 | "value": null 214 | }, 215 | { 216 | "color": "red", 217 | "value": 80 218 | } 219 | ] 220 | }, 221 | "unit": "binbps" 222 | }, 223 | "overrides": [] 224 | }, 225 | "gridPos": { 226 | "h": 22, 227 | "w": 24, 228 | "x": 0, 229 | "y": 11 230 | }, 231 | "id": 2, 232 | "options": { 233 | "color": "blue", 234 | "iteration": 7, 235 | "labelSize": 15, 236 | "monochrome": false, 237 | "nodeColor": "grey", 238 | "nodePadding": 20, 239 | "nodeWidth": 10 240 | }, 241 | "pluginVersion": "1.1.3", 242 | "targets": [ 243 | { 244 | "editorMode": "code", 245 | "format": "table", 246 | "rawQuery": true, 247 | "rawSql": "select\n coalesce(src_ifidx::text, 'Other') as in_port,\n coalesce(dev_ip::text, 'Other') as router,\n coalesce(dst_ifidx::text, 'Other') as out_port,\n sum(octets) as value\nfrom \"${mo}-ingress6_routerp\"\nwhere $__timeFilter(time)\ngroup by in_port, router, out_port\norder by value desc\nlimit 15", 248 | "refId": "A", 249 | "sql": { 250 | "columns": [ 251 | { 252 | "parameters": [], 253 | "type": "function" 254 | } 255 | ], 256 | "groupBy": [ 257 | { 258 | "property": { 259 | "type": "string" 260 | }, 261 | "type": "groupBy" 262 | } 263 | ], 264 | "limit": 50 265 | }, 266 | "datasource": { 267 | "type": "grafana-postgresql-datasource", 268 | "uid": "${DS_GRAFANA-POSTGRESQL-DATASOURCE}" 269 | } 270 | } 271 | ], 272 | "title": "Routers and ports", 273 | "type": "netsage-sankey-panel" 274 | } 275 | ], 276 | "refresh": "", 277 | "schemaVersion": 40, 278 | "tags": [], 279 | "templating": { 280 | "list": [ 281 | { 282 | "allowCustomValue": false, 283 | "current": {}, 284 | "definition": "select left(tablename, length(tablename) - length('ingress6_total') - 1) from pg_catalog.pg_tables where tablename like '%ingress6_total'", 285 | "name": "mo", 286 | "options": [], 287 | "query": "select left(tablename, length(tablename) - length('ingress6_total') - 1) from pg_catalog.pg_tables where tablename like '%ingress6_total'", 288 | "refresh": 1, 289 | "regex": "", 290 | "type": "query" 291 | } 292 | ] 293 | }, 294 | "time": { 295 | "from": "now-6h", 296 | "to": "now" 297 | }, 298 | "timepicker": {}, 299 | "timezone": "browser", 300 | "title": "Routers IPv6", 301 | "uid": "aecfusxl0gb28d", 302 | "version": 8, 303 | "weekStart": "" 304 | } 305 | -------------------------------------------------------------------------------- /monit-objects-mavg-under.c: -------------------------------------------------------------------------------- 1 | /* 2 | * xenoeye 3 | * 4 | * Copyright (c) 2024-2025, Vladimir Misyurov 5 | * 6 | * Permission to use, copy, modify, and/or distribute this software for any 7 | * purpose with or without fee is hereby granted, provided that the above 8 | * copyright notice and this permission notice appear in all copies. 9 | * 10 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 11 | * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 12 | * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 13 | * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 14 | * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE 15 | * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 16 | * PERFORMANCE OF THIS SOFTWARE. 17 | */ 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | #include "utils.h" 28 | #include "netflow.h" 29 | #include "monit-objects.h" 30 | #include "monit-objects-common.h" 31 | 32 | #define MAVG_VAL(DATUM, I, SIZE) ((struct mavg_val *)&DATUM[SIZE * I]) 33 | 34 | static int 35 | underlimit_item_check(struct mo_mavg *mavg, uint8_t *key, size_t keysize, 36 | MAVG_TYPE limit, struct mavg_val *val, size_t limit_index, 37 | uint64_t time_ns) 38 | { 39 | TKVDB_RES rc; 40 | tkvdb_datum dtk, dtv; 41 | uint8_t key_with_limit_index[keysize + sizeof(size_t)]; 42 | MAVG_TYPE v = val->val; 43 | /* adjust to value per second */ 44 | v /= mavg->size_secs; 45 | 46 | /*LOG("Underlimit debug: val %f, limit %f", v, limit);*/ 47 | struct mavg_limits *lim_curr = MAVG_LIM_CURR(mavg); 48 | 49 | memcpy(key_with_limit_index, key, keysize); 50 | memcpy(key_with_limit_index + keysize, &limit_index, sizeof(size_t)); 51 | 52 | dtk.data = key_with_limit_index; 53 | dtk.size = keysize + sizeof(size_t); 54 | 55 | /* check if this item is in db */ 56 | rc = mavg->underlm_db->get(mavg->underlm_db, &dtk, &dtv); 57 | if (rc == TKVDB_OK) { 58 | /* in db, update values */ 59 | struct mavg_lim_data *ld = (struct mavg_lim_data *)dtv.data; 60 | 61 | ld->time_last = time_ns; 62 | ld->val = v; 63 | 64 | ld->limit = limit; 65 | ld->back2norm_time_ns 66 | = lim_curr->underlimit[limit_index].back2norm_time_ns; 67 | 68 | if (v < limit) { 69 | if (ld->state == MAVG_LIM_GONE) { 70 | ld->state = MAVG_LIM_NEW; 71 | } 72 | } 73 | } else { 74 | if (v < limit) { 75 | /* not in db and less than limit, add to db */ 76 | struct mavg_lim_data ld; 77 | ld.state = MAVG_LIM_NEW; 78 | ld.time_last = time_ns; 79 | ld.time_dump = 0; 80 | ld.val = val->val; 81 | ld.limit = limit; 82 | ld.back2norm_time_ns 83 | = lim_curr->underlimit[limit_index].back2norm_time_ns; 84 | 85 | dtv.data = &ld; 86 | dtv.size = sizeof(struct mavg_lim_data); 87 | 88 | rc = mavg->underlm_db->put(mavg->underlm_db, &dtk, &dtv); 89 | if (rc != TKVDB_OK) { 90 | LOG("Can't append item to db with "\ 91 | "underlimited records, error code %d", rc); 92 | return 0; 93 | } 94 | } 95 | } 96 | 97 | return 1; 98 | } 99 | 100 | 101 | static int 102 | underlimit_check(struct mo_mavg *mavg, tkvdb_tr *db, uint64_t time_ns, 103 | size_t val_itemsize) 104 | { 105 | tkvdb_cursor *c; 106 | 107 | struct mavg_limits *lim_curr = MAVG_LIM_CURR(mavg); 108 | 109 | c = tkvdb_cursor_create(db); 110 | if (!c) { 111 | LOG("tkvdb_cursor_create() failed"); 112 | return 0; 113 | } 114 | 115 | if (c->first(c) != TKVDB_OK) { 116 | /* empty set */ 117 | goto empty; 118 | } 119 | 120 | do { 121 | size_t i; 122 | uint8_t *key = c->key(c); 123 | uint8_t *pval = c->val(c); 124 | 125 | for (i=0; ifieldset.n_aggr; i++) { 126 | size_t j; 127 | struct mavg_val *val; 128 | 129 | val = MAVG_VAL(pval, i, val_itemsize); 130 | for (j=0; jnunderlimit; j++) { 131 | size_t lidx = j + lim_curr->noverlimit; 132 | MAVG_TYPE limit; 133 | limit = val->limits[lidx]; 134 | 135 | underlimit_item_check(mavg, key, c->keysize(c), 136 | limit, val, j, time_ns); 137 | } 138 | } 139 | } while (c->next(c) == TKVDB_OK); 140 | 141 | empty: 142 | c->free(c); 143 | 144 | return 1; 145 | } 146 | 147 | 148 | static int 149 | mavg_merge(struct mo_mavg *mavg, tkvdb_tr *db, tkvdb_tr *thread_db, 150 | uint64_t time_ns) 151 | { 152 | MAVG_TYPE wndsize = mavg->size_secs * 1e9; 153 | tkvdb_cursor *c; 154 | 155 | struct mavg_limits *lim_curr = MAVG_LIM_CURR(mavg); 156 | 157 | c = tkvdb_cursor_create(thread_db); 158 | if (!c) { 159 | LOG("tkvdb_cursor_create() failed"); 160 | return 0; 161 | } 162 | 163 | if (c->first(c) != TKVDB_OK) { 164 | /* empty set */ 165 | goto empty; 166 | } 167 | 168 | do { 169 | TKVDB_RES rc; 170 | tkvdb_datum dtk, dtv; 171 | size_t i; 172 | 173 | size_t val_itemsize = c->valsize(c); 174 | uint8_t *vals_db = c->val(c); 175 | 176 | /* array of vals */ 177 | uint8_t vals_local[val_itemsize]; 178 | 179 | /* recalculate moving avg values for the current time */ 180 | for (i=0; ifieldset.n_aggr; i++) { 181 | struct mavg_val *val, *val_db; 182 | MAVG_TYPE v, new_v; 183 | uint64_t tmdiff; 184 | size_t j; 185 | 186 | val = MAVG_VAL(vals_local, i, val_itemsize); 187 | val_db = MAVG_VAL(vals_db, i, val_itemsize); 188 | 189 | v = atomic_load_explicit(&val_db->val, 190 | memory_order_relaxed); 191 | tmdiff = time_ns 192 | - atomic_load_explicit(&val_db->time_prev, 193 | memory_order_relaxed); 194 | 195 | if (tmdiff < wndsize) { 196 | new_v = v - tmdiff / wndsize * v; 197 | val->val = new_v; 198 | } else { 199 | val->val = 0.0f; 200 | } 201 | 202 | /* rest of structure */ 203 | val->time_prev = time_ns; 204 | for (j=0; jnoverlimit + lim_curr->nunderlimit; j++) { 205 | v = atomic_load_explicit(&val_db->limits[j], 206 | memory_order_relaxed); 207 | val->limits[j] = v; 208 | } 209 | } 210 | 211 | 212 | dtk.data = c->key(c); 213 | dtk.size = c->keysize(c); 214 | 215 | /* check if item is in db */ 216 | rc = db->get(db, &dtk, &dtv); 217 | if (rc == TKVDB_OK) { 218 | uint8_t *vals = dtv.data; 219 | 220 | /* update data */ 221 | for (i=0; ifieldset.n_aggr; i++) { 222 | struct mavg_val *val_db, *val_local; 223 | 224 | val_db = MAVG_VAL(vals, i, val_itemsize); 225 | val_local = MAVG_VAL(vals_local, i, 226 | val_itemsize); 227 | val_db->val += val_local->val; 228 | } 229 | } else { 230 | /* not found, create new */ 231 | dtv.data = vals_local; 232 | dtv.size = val_itemsize; 233 | 234 | rc = db->put(db, &dtk, &dtv); 235 | if (rc != TKVDB_OK) { 236 | LOG("put() failed, code %d", rc); 237 | break; 238 | } 239 | } 240 | } while (c->next(c) == TKVDB_OK); 241 | 242 | empty: 243 | c->free(c); 244 | 245 | return 1; 246 | } 247 | 248 | 249 | static int 250 | mavg_check_underlimit(struct monit_object *mo, struct mo_mavg *mavg, 251 | size_t nthreads, uint64_t time_ns) 252 | { 253 | tkvdb_tr *db; 254 | size_t i; 255 | 256 | /* temporary database with moving averages */ 257 | db = tkvdb_tr_create(NULL, NULL); 258 | if (!db) { 259 | LOG("Can't create transaction"); 260 | return 0; 261 | } 262 | 263 | db->begin(db); 264 | 265 | for (i=0; ithr_data[i].db, 269 | memory_order_relaxed); 270 | 271 | mavg_merge(mavg, db, thread_db, time_ns); 272 | } 273 | 274 | underlimit_check(mavg, db, time_ns, mavg->thr_data[0].val_itemsize); 275 | 276 | db->free(db); 277 | 278 | act(mavg, mavg->underlm_db, mavg->size_secs * 1e9, mo->name, 0); 279 | 280 | return 1; 281 | } 282 | 283 | static void 284 | underlimit_check_rec(struct xe_data *globl, 285 | struct monit_object *mos, size_t n_mo, uint64_t time_ns) 286 | { 287 | size_t i, j; 288 | 289 | for (i=0; inmavg; j++) { 294 | struct mo_mavg *mavg = &mo->mavgs[j]; 295 | struct mavg_limits *lim_curr = MAVG_LIM_CURR(mavg); 296 | 297 | if (lim_curr->nunderlimit == 0) { 298 | continue; 299 | } 300 | 301 | if (time_ns < (mavg->start_ns + mavg->size_secs*1e9)) { 302 | continue; 303 | } 304 | 305 | mavg_check_underlimit(mo, mavg, globl->nthreads, 306 | time_ns); 307 | } 308 | 309 | if (mo->n_mo) { 310 | underlimit_check_rec(globl, mo->mos, mo->n_mo, time_ns); 311 | } 312 | } 313 | } 314 | 315 | void * 316 | mavg_check_underlimit_thread(void *arg) 317 | { 318 | struct xe_data *globl = (struct xe_data *)arg; 319 | 320 | for (;;) { 321 | struct timespec tmsp; 322 | uint64_t time_ns; 323 | 324 | if (atomic_load_explicit(&globl->stop, memory_order_relaxed)) { 325 | /* stop */ 326 | break; 327 | } 328 | 329 | 330 | if (clock_gettime(CLOCK_REALTIME_COARSE, &tmsp) < 0) { 331 | LOG("clock_gettime() failed: %s", strerror(errno)); 332 | } 333 | time_ns = tmsp.tv_sec * 1e9 + tmsp.tv_nsec; 334 | 335 | underlimit_check_rec(globl, globl->monit_objects, 336 | globl->nmonit_objects, time_ns); 337 | 338 | sleep(1); 339 | } 340 | 341 | return NULL; 342 | } 343 | 344 | --------------------------------------------------------------------------------