├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── docs ├── .gitignore ├── Makefile ├── Pipfile ├── Pipfile.lock ├── README.md ├── mkdocs.yml └── src │ ├── arch.md │ ├── config.md │ ├── expcap.md │ ├── img │ ├── .DS_Store │ ├── exact-capture-arch.png │ ├── exact-capture-eg-topo.png │ ├── exact-capture-expcap-footer.png │ ├── exact-capture-expcap-padding.png │ ├── exact-capture-expcap.png │ └── exact-capture-toolchain.png │ ├── index.md │ ├── install.md │ ├── quick.md │ ├── server.md │ ├── tools │ ├── analyze.md │ ├── extract.md │ ├── match.md │ ├── modify.md │ ├── overview.md │ └── parse.md │ ├── tuning.md │ └── versions.md ├── libs ├── chaste └── libchaste │ ├── 60740670f7763b10d1b022c6d1709434b39c2a5d │ ├── LICENSE │ ├── README │ ├── include │ ├── asm │ │ └── asm.h │ ├── chaste.h │ ├── data_structs │ │ ├── array │ │ │ ├── array.h │ │ │ ├── array_std.h │ │ │ ├── array_typed_declare_template.h │ │ │ └── array_typed_define_template.h │ │ ├── circular_queue │ │ │ └── circular_queue.h │ │ ├── function_hash_map │ │ │ └── function_hash_map.h │ │ ├── hash_map │ │ │ └── hash_map.h │ │ ├── linked_list │ │ │ ├── linked_list.h │ │ │ ├── linked_list_std.h │ │ │ ├── linked_list_typed_declare_template.h │ │ │ └── linked_list_typed_define_template.h │ │ └── vector │ │ │ ├── vector.h │ │ │ ├── vector_std.h │ │ │ ├── vector_typed_declare_template.h │ │ │ └── vector_typed_define_template.h │ ├── hash_functions │ │ └── spooky │ │ │ └── spooky_hash.h │ ├── log │ │ ├── log.h │ │ └── log_levels.h │ ├── makeinclude.sh │ ├── options │ │ ├── options.h │ │ └── options_vectors.h │ ├── parsing │ │ ├── bool_parser.h │ │ ├── numeric_parser.h │ │ └── utils.h │ ├── perf │ │ ├── perf.h │ │ └── perf_mon.h │ ├── scripts │ │ └── camio_prepare_boot.sh │ ├── string │ │ └── string.h │ ├── sys │ │ ├── sys.h │ │ ├── sys_apple.h │ │ └── sys_linux.h │ ├── term_color │ │ └── term_color.h │ ├── timing │ │ └── timestamp.h │ ├── types │ │ └── types.h │ └── utils │ │ ├── debug.h │ │ └── util.h │ └── libchaste.a ├── src ├── data_structs │ ├── eiostream_vec.c │ ├── eiostream_vec.h │ ├── expcap.h │ ├── pcap-structures.h │ ├── pthread_vec.c │ ├── pthread_vec.h │ └── timespecps.h ├── exact-capture-listener.c ├── exact-capture-listener.h ├── exact-capture-writer.c ├── exact-capture-writer.h ├── exact-capture.c ├── exact-capture.h ├── exactio │ ├── exactio.c │ ├── exactio.h │ ├── exactio_bring.c │ ├── exactio_bring.h │ ├── exactio_dummy.c │ ├── exactio_dummy.h │ ├── exactio_exanic.c │ ├── exactio_exanic.h │ ├── exactio_file.c │ ├── exactio_file.h │ ├── exactio_stream.h │ ├── exactio_timing.c │ └── exactio_timing.h ├── utils.c └── utils.h └── tools ├── checksum.h ├── data_structs ├── buff.c ├── buff.h ├── fusion_hpt.h ├── pcap_buff.c ├── pcap_buff.h ├── pseudo_ip_header.h ├── timespecps.h └── vlan_ethhdr.h ├── exact-pcap-analyze.c ├── exact-pcap-extract.c ├── exact-pcap-match.c ├── exact-pcap-modify.c ├── exact-pcap-parse.c ├── utils.c └── utils.h /.gitignore: -------------------------------------------------------------------------------- 1 | bin/* 2 | 3 | # Prerequisites 4 | *.d 5 | 6 | # Object files 7 | *.o 8 | *.ko 9 | *.obj 10 | *.elf 11 | 12 | # Linker output 13 | *.ilk 14 | *.map 15 | *.exp 16 | 17 | # Precompiled Headers 18 | *.gch 19 | *.pch 20 | 21 | # Libraries 22 | *.lib 23 | *.la 24 | *.lo 25 | 26 | # Shared objects (inc. Windows DLLs) 27 | *.dll 28 | *.so 29 | *.so.* 30 | *.dylib 31 | 32 | # Executables 33 | *.exe 34 | *.out 35 | *.app 36 | *.i*86 37 | *.x86_64 38 | *.hex 39 | 40 | # Debug files 41 | *.dSYM/ 42 | *.su 43 | *.idb 44 | *.pdb 45 | 46 | # Kernel Module Compile Results 47 | *.mod* 48 | *.cmd 49 | .tmp_versions/ 50 | modules.order 51 | Module.symvers 52 | Mkfile.old 53 | dkms.conf 54 | 55 | #Build outputs 56 | bin/* 57 | 58 | #Eclipse settings files 59 | .settings/* 60 | .project 61 | .cproject 62 | 63 | # Emacs temp files 64 | *~ 65 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PREFIX=/usr/local 2 | CC=cc 3 | INCLUDES=-Ilibs -Isrc -I. 4 | LDFLAGS=libs/libchaste/libchaste.a 5 | GLOBAL_CFLAGS=-g -std=c99 -D_GNU_SOURCE -D_DEFAULT_SOURCE -D_XOPEN_SOURCE=700 -fPIC -Wno-missing-field-initializers -Wno-missing-braces 6 | RELEASE_CFLAGS=$(INCLUDES) $(GLOBAL_CFLAGS) -O3 -Wall -DNDEBUG -DNOIFASSERT 7 | ASSERT_CFLAGS=$(INCLUDES) $(GLOBAL_CFLAGS) -O3 -Wall -DNDEBUG 8 | DEBUG_CFLAGS=$(INCLUDES) $(GLOBAL_CFLAGS) -Werror -Wall -Wextra -pedantic 9 | BIN=bin/exact-capture 10 | TOOLS=bin/exact-pcap-extract bin/exact-pcap-parse bin/exact-pcap-match bin/exact-pcap-modify bin/exact-pcap-analyze 11 | 12 | EXACTCAP_SRCS=$(wildcard src/*.c) $(wildcard src/**/*.c) 13 | EXACTCAP_HDRS=$(wildcard src/*.h) $(wildcard src/**/*.h) 14 | LIBCHASTE_HDRS=$(wildcard libs/chaste/*.h) $(wildcard libs/chaste/**/*.h) 15 | BUFF_SRC=tools/data_structs/buff.c tools/data_structs/pcap_buff.c 16 | BUFF_HDRS=tools/data_structs/buff.h tools/data_structs/pcap_buff.h 17 | 18 | all: CFLAGS = $(RELEASE_CFLAGS) 19 | all: $(BIN) $(TOOLS) 20 | 21 | assert: CFLAGS = $(ASSERT_CFLAGS) 22 | assert: $(BIN) $(TOOLS) 23 | 24 | debug: CFLAGS = $(DEBUG_CFLAGS) 25 | debug: $(BIN) $(TOOLS) 26 | 27 | tools: CFLAGS= $(RELEASE_CFLAGS) 28 | tools: $(TOOLS) 29 | 30 | bin/exact-capture: $(EXACTCAP_SRCS) $(EXACTCAP_HDRS) $(LIBCASHTE_HDRS) 31 | mkdir -p bin 32 | $(CC) $(CFLAGS) $(EXACTCAP_SRCS) $(LDFLAGS) -lm -lexanic -lpthread -lrt -o $@ 33 | 34 | bin/exact-pcap-parse: $(BUFF_SRC) $(BUFF_HDRS) tools/exact-pcap-parse.c $(EXACTCAP_HDRS) $(LIBCAHSTE_HDRS) 35 | mkdir -p bin 36 | $(CC) $(CFLAGS) $(BUFF_SRC) $(BUFF_HDRS) tools/exact-pcap-parse.c $(LDFLAGS) -o $@ 37 | 38 | bin/exact-pcap-match: $(BUFF_SRC) $(BUFF_HDRS) tools/exact-pcap-match.c $(EXACTCAP_HDRS) $(LIBCAHSTE_HDRS) 39 | $(CC) $(CFLAGS) $(BUFF_SRC) $(BUFF_HDRS) tools/exact-pcap-match.c $(LDFLAGS) -o $@ 40 | 41 | bin/exact-pcap-extract: $(BUFF_SRC) $(BUFF_HDRS) tools/exact-pcap-extract.c $(EXACTCAP_HDRS) $(LIBCAHSTE_HDRS) 42 | $(CC) $(CFLAGS) $(BUFF_SRC) $(BUFF_HDRS) tools/exact-pcap-extract.c $(LDFLAGS) -o $@ 43 | 44 | bin/exact-pcap-analyze: $(BUFF_SRC) $(BUFF_HDRS) tools/exact-pcap-analyze.c tools/utils.c $(EXACTCAP_HDRS) $(LIBCAHSTE_HDRS) 45 | $(CC) $(CFLAGS) $(BUFF_SRC) $(BUFF_HDRS) tools/exact-pcap-analyze.c tools/utils.c $(LDFLAGS) -o $@ 46 | 47 | bin/exact-pcap-modify: tools/exact-pcap-modify.c $(EXACTCAP_HDRS) $(LIBCAHSTE_HDRS) 48 | $(CC) $(CFLAGS) tools/exact-pcap-modify.c $(LDFLAGS) -o $@ 49 | 50 | install: all install_tools 51 | install -d $(PREFIX)/bin 52 | install -m 0755 -D $(BIN) $(PREFIX)/bin 53 | 54 | install_tools: tools 55 | install -d $(PREFIX)/bin 56 | install -m 0755 -D $(TOOLS) $(PREFIX)/bin 57 | 58 | uninstall: uninstall_tools 59 | rm -f $(foreach file,$(BIN),$(PREFIX)/bin/$(file)) 60 | 61 | uninstall_tools: 62 | rm -f $(foreach file,$(TOOLS),$(PREFIX)/bin/$(file)) 63 | 64 | .PHONY: docs 65 | docs: 66 | $(MAKE) -C docs/ 67 | 68 | clean: 69 | rm -rf bin/* 70 | $(MAKE) -C docs/ clean 71 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Exact Capture 2 | ## Lossless Packet Capture for ExaNICs 3 | 4 | Exact Capture is a high-rate, lossless packet capture solution for ExaNIC network adapters. 5 | The system is fully open source and designed for performance as well as ease of configuration. 6 | It can be used with any ExaNIC network card, and is optimised for use with ExaDisk high speed flash drives. 7 | The system can be deployed on any suitably powerful server system. 8 | 9 | Full online documentation is available [here](https://www.cisco.com/c/en/us/td/docs/dcn/nexus3550/exact-capture/sw/user-guide/exact-capture-user-guide/install.html) 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | #ignore the generated site 2 | site 3 | 4 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | env = pipenv run 2 | 3 | 4 | all: docs 5 | .PHONY all: 6 | 7 | 8 | test: 9 | $(env) mkdocs serve 10 | 11 | publish: 12 | $(env) mkdocs gh-deploy 13 | 14 | docs: 15 | $(env) mkdocs build --clean 16 | 17 | clean: 18 | rm -rf site 19 | 20 | -------------------------------------------------------------------------------- /docs/Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | name = "pypi" 3 | url = "https://pypi.org/simple" 4 | verify_ssl = true 5 | 6 | [dev-packages] 7 | 8 | [packages] 9 | mkdocs = "*" 10 | 11 | [requires] 12 | python_version = "3" 13 | -------------------------------------------------------------------------------- /docs/Pipfile.lock: -------------------------------------------------------------------------------- 1 | { 2 | "_meta": { 3 | "hash": { 4 | "sha256": "53f0dad17ad7f48f8daa50a9063cd498a5c01c1f1c808fec48ae82c2d031bd0b" 5 | }, 6 | "pipfile-spec": 6, 7 | "requires": { 8 | "python_version": "3.6" 9 | }, 10 | "sources": [ 11 | { 12 | "name": "pypi", 13 | "url": "https://pypi.org/simple", 14 | "verify_ssl": true 15 | } 16 | ] 17 | }, 18 | "default": { 19 | "click": { 20 | "hashes": [ 21 | "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", 22 | "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7" 23 | ], 24 | "version": "==7.0" 25 | }, 26 | "jinja2": { 27 | "hashes": [ 28 | "sha256:065c4f02ebe7f7cf559e49ee5a95fb800a9e4528727aec6f24402a5374c65013", 29 | "sha256:14dd6caf1527abb21f08f86c784eac40853ba93edb79552aa1e4b8aef1b61c7b" 30 | ], 31 | "version": "==2.10.1" 32 | }, 33 | "livereload": { 34 | "hashes": [ 35 | "sha256:78d55f2c268a8823ba499305dcac64e28ddeb9a92571e12d543cd304faf5817b", 36 | "sha256:89254f78d7529d7ea0a3417d224c34287ebfe266b05e67e51facaf82c27f0f66" 37 | ], 38 | "version": "==2.6.1" 39 | }, 40 | "markdown": { 41 | "hashes": [ 42 | "sha256:2e50876bcdd74517e7b71f3e7a76102050edec255b3983403f1a63e7c8a41e7a", 43 | "sha256:56a46ac655704b91e5b7e6326ce43d5ef72411376588afa1dd90e881b83c7e8c" 44 | ], 45 | "version": "==3.1.1" 46 | }, 47 | "markupsafe": { 48 | "hashes": [ 49 | "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", 50 | "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161", 51 | "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235", 52 | "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5", 53 | "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff", 54 | "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b", 55 | "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1", 56 | "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e", 57 | "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183", 58 | "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66", 59 | "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1", 60 | "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1", 61 | "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e", 62 | "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b", 63 | "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905", 64 | "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735", 65 | "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d", 66 | "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e", 67 | "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d", 68 | "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c", 69 | "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21", 70 | "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2", 71 | "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5", 72 | "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b", 73 | "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6", 74 | "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f", 75 | "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f", 76 | "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7" 77 | ], 78 | "version": "==1.1.1" 79 | }, 80 | "mkdocs": { 81 | "hashes": [ 82 | "sha256:17d34329aad75d5de604b9ed4e31df3a4d235afefdc46ce7b1964fddb2e1e939", 83 | "sha256:8cc8b38325456b9e942c981a209eaeb1e9f3f77b493ad755bfef889b9c8d356a" 84 | ], 85 | "index": "pypi", 86 | "version": "==1.0.4" 87 | }, 88 | "pyyaml": { 89 | "hashes": [ 90 | "sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9", 91 | "sha256:01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4", 92 | "sha256:5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8", 93 | "sha256:5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696", 94 | "sha256:7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34", 95 | "sha256:7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9", 96 | "sha256:87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73", 97 | "sha256:9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299", 98 | "sha256:a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b", 99 | "sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae", 100 | "sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681", 101 | "sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41", 102 | "sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8" 103 | ], 104 | "version": "==5.1.2" 105 | }, 106 | "six": { 107 | "hashes": [ 108 | "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", 109 | "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" 110 | ], 111 | "version": "==1.12.0" 112 | }, 113 | "tornado": { 114 | "hashes": [ 115 | "sha256:349884248c36801afa19e342a77cc4458caca694b0eda633f5878e458a44cb2c", 116 | "sha256:398e0d35e086ba38a0427c3b37f4337327231942e731edaa6e9fd1865bbd6f60", 117 | "sha256:4e73ef678b1a859f0cb29e1d895526a20ea64b5ffd510a2307b5998c7df24281", 118 | "sha256:559bce3d31484b665259f50cd94c5c28b961b09315ccd838f284687245f416e5", 119 | "sha256:abbe53a39734ef4aba061fca54e30c6b4639d3e1f59653f0da37a0003de148c7", 120 | "sha256:c845db36ba616912074c5b1ee897f8e0124df269468f25e4fe21fe72f6edd7a9", 121 | "sha256:c9399267c926a4e7c418baa5cbe91c7d1cf362d505a1ef898fde44a07c9dd8a5" 122 | ], 123 | "version": "==6.0.3" 124 | } 125 | }, 126 | "develop": {} 127 | } 128 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Exact Capture Documentation 2 | ----------------------------- 3 | 4 | ## mkdocs 5 | Exact capture documentation is written using (mkdocs)[https://www.mkdocs.org/]. 6 | All of the documentation pages can be found in markdown format in the `src` directory. 7 | Configuration for mkdocs is found in `mkdocs.yml`. 8 | 9 | ## Requirements 10 | - python3 11 | - pipenv 12 | - mkdocs 13 | 14 | Python2 is now deprecated, with end of support as of 2019. 15 | Python3 has been around for over 10 years, so it's time to move on to Python3. 16 | 17 | To ensure a consistent output, the build process uses `pipenv`. 18 | Settings for pipenv can be found in the `Pipfile` and `Pipfile.lock` 19 | 20 | To install `python3`, run (on Fedora/RedHat/Centos systems) 21 | ``` 22 | $ sudo yum install python3 23 | ``` 24 | 25 | or (on Debian/Ubuntu systems) 26 | 27 | ``` 28 | $ sudo apt install python3 29 | ``` 30 | 31 | 32 | To install `pip`, run 33 | ``` 34 | $ curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py 35 | $ sudo python get-pip.py 36 | ``` 37 | 38 | To install `pipenv` run 39 | ``` 40 | $ sudo pip install pipenv 41 | ``` 42 | **Note** 43 | ----------------- 44 | The following error is common on Ubuntu based systems: 45 | 46 | ``` 47 | import distutils.spawn 48 | ModuleNotFoundError: No module named 'distutils.spawn' 49 | 50 | Failed to create virtual environment. 51 | ``` 52 | 53 | To resolve you need to install `python3-distuilts`. 54 | You can do this as follows: 55 | ``` 56 | $ sudp apt-get install python3-distutils 57 | ``` 58 | ----------------- 59 | 60 | 61 | To install `mkdocs` run (inside the exact-capture documentation directory) 62 | ``` 63 | ~/exact-capture/docs $ sudo pipenv install mkdocs 64 | ``` 65 | 66 | 67 | ## Building 68 | Pipenv is called directly from the makefile. To build simply run: 69 | 70 | ``` 71 | $ make 72 | pipenv run mkdocs build --clean 73 | INFO - Cleaning site directory 74 | INFO - Building documentation to directory: /home/exact-cap/exact-capture/docs/site 75 | ``` 76 | 77 | ## Testing 78 | mkdocs ships with a built in server for local testing and viewing of the rendered output. 79 | To test, run: 80 | 81 | ``` 82 | $ make test 83 | pipenv run mkdocs serve 84 | INFO - Building documentation... 85 | INFO - Cleaning site directory 86 | [I 191002 17:56:22 server:296] Serving on http://127.0.0.1:8000 87 | [I 191002 17:56:22 handlers:62] Start watching changes 88 | [I 191002 17:56:22 handlers:64] Start detecting changes 89 | ``` 90 | 91 | Connecting your browser to `127.0.0.1:8000` should show the fully rendered page. 92 | -------------------------------------------------------------------------------- /docs/mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: Exact Capture Documentation 2 | site_url: http://exact-capture.io 3 | theme: 4 | name: readthedocs 5 | highlightjs: true 6 | 7 | docs_dir: src 8 | 9 | nav: 10 | - Exact Capture: 11 | - Introduction: index.md 12 | - Installation: install.md 13 | - Quick Start: quick.md 14 | - Server Requirements: server.md 15 | - Configuration Guide: config.md 16 | - Performance Tuning: tuning.md 17 | - Output Format (expcap): expcap.md 18 | - Internal Architecture: arch.md 19 | - Tools: 20 | - Overview: tools/overview.md 21 | - Exact PCAP Extract: tools/extract.md 22 | - Exact PCAP Modify: tools/modify.md 23 | - Exact PCAP Match: tools/match.md 24 | - Exact PCAP Parse: tools/parse.md 25 | - Exact PCAP Analyze: tools/analyze.md 26 | - Version History: versions.md 27 | 28 | markdown_extensions: 29 | - markdown.extensions.admonition 30 | - markdown.extensions.fenced_code 31 | 32 | plugins: 33 | - search 34 | -------------------------------------------------------------------------------- /docs/src/arch.md: -------------------------------------------------------------------------------- 1 | The Exact Capture software system comprises 4 major internal components: 2 | 3 | 1. One or more “hot” threads - to read packets from the ExaNIC(s) into memory 4 | 2. One or more “cold” threads - to write packets from memory to disk(s) 5 | 3. One or more shared memory queues - to connect the hot and cold threads to each other 6 | 4. One management thread - responsible for control and statistics collection / reporting 7 | 8 | These basic architectural components are illustrated below, with the addition of the ExaNIC and ExaDisk resources. 9 | The head of each column highlights the performance limiting resource for that component: 10 | 11 | ![Exact Capture Architecture](img/exact-capture-arch.png) 12 | 13 | -------------------------------------------------------------------------------- /docs/src/expcap.md: -------------------------------------------------------------------------------- 1 | Exact Capture outputs packet captures to a modified `pcap` format called `expcap`. 2 | The `expcap` format is a backwards compatible extension to standard `pcap` format. 3 | A number of [tools and utilities](tools/overview.md) are included for operating on these files, including converting them into standard `pcap` format if necessary. 4 | 5 | For reference, standard `pcap` files are formatted as follows (more details can be found on the [Wireshark](https://wiki.wireshark.org/Development/LibpcapFileFormat) website): 6 | 7 | ![PCAP file format](img/exact-capture-expcap.png) 8 | 9 | 10 | ## Padding Packets 11 | As a performance optimisation, Exact Capture occasionally needs to insert padding packets (e.g. to align to a 4k boundary) into the output file. 12 | These packets are easily recognizable because `pcap` header wire length field (as above) is set to 0B, with the on disk length field (again as above) is set to the size of the padding. 13 | 14 | Although setting these header fields precisely captures the semantics of the padding packets (i.e bytes written to disk that were never found on the wire), this is technically a `pcap` file specification violation. 15 | Presumably the writers never envisaged this kind of use case. 16 | Nevertheless, standard tools like `Wireshark` operate correctly and ignore these packets as they should. 17 | 18 | The following figure depicts a padding packet directly after the file header. 19 | This will be found in all Exact Capture output files. 20 | 21 | ![EXPCAP file padding](img/exact-capture-expcap-padding.png) 22 | 23 | 24 | ## Packet Footers 25 | Each captured packet is extended with a packet footer. 26 | This footer contains a variety of extra fields, not available in the standard pcap format. When the footer is added, the standard pcap disk bytes field is updated to reflect the extra length on disk. Once again, this means that the byes on disk value may exceed the bytes on the wire value (though not always. e.g. when a snaplength is set). The addition of a footer adds bytes to the disk that were never found on the wire is again, technically a PCAP specification violation. However, once again, standard pcap processing tools like Wireshark operate correctly and ignore these extra bytes as they should. The above figure shows a representation of expcap packet footers added to the first packet. 27 | 28 | ![EXPCAP file padding](img/exact-capture-expcap-footer.png) 29 | 30 | The additional expcap footer fields are described in detail in the table below. They borrow the spirit of some of the fields found in the ERF format. 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 46 | 47 | 48 | 49 | 50 | 53 | 54 | 55 | 56 | 57 | 67 | 68 | 69 | 70 | 71 | 75 | 76 | 77 | 78 | 79 | 83 | 84 | 85 | 86 | 87 | 91 | 92 | 93 | 94 | 95 | 98 | 99 |
FieldWidth (bits)Description
Time (seconds)32 44 | Time in seconds since the epoch 45 |
Time (picoseconds)40 51 | Time in picoseconds since the last second boundary 52 |
Flags8 58 | The following flags bits are currently supported: 59 | 60 |
    61 |
  1. New CRC Calculated (The CRC field contains a new new value including the footer)
    2. 62 |
  2. Frame aborted - this frame was aborted on the wire by the sender.
    3. 63 |
  3. Frame corrupt - the hardware CRC checker detected an error with this frame.
    4. 64 |
  4. Frame truncated - this packet was longer than the snap length and has been truncated.
    5. 65 |
66 |
Device ID Number8 72 | The ID of the device that captured these packets. 73 | For example, when capturing on the exanic3:7 interface, the device number would be 3. 74 |
Port ID Number8 80 | The port on the device that was used to capture the packet. 81 | For example, capturing on exanic3:7 interface, the port number would be 7. 82 |
CRC top16 88 | If the new CRC flag is not set, contains the number of packets dropped between this packet and the previous packet. 89 | Otherwise this is the top 16 bits of the new CRC. 90 |
CRC bottom16 96 | If the new CRC flag is set, contains the bottom 16 bits of the new CRC. Otherwise, unused. 97 |
100 | -------------------------------------------------------------------------------- /docs/src/img/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cisco/exact-capture/2a7b74e66cbbafcec7c39947dea63bec03972685/docs/src/img/.DS_Store -------------------------------------------------------------------------------- /docs/src/img/exact-capture-arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cisco/exact-capture/2a7b74e66cbbafcec7c39947dea63bec03972685/docs/src/img/exact-capture-arch.png -------------------------------------------------------------------------------- /docs/src/img/exact-capture-eg-topo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cisco/exact-capture/2a7b74e66cbbafcec7c39947dea63bec03972685/docs/src/img/exact-capture-eg-topo.png -------------------------------------------------------------------------------- /docs/src/img/exact-capture-expcap-footer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cisco/exact-capture/2a7b74e66cbbafcec7c39947dea63bec03972685/docs/src/img/exact-capture-expcap-footer.png -------------------------------------------------------------------------------- /docs/src/img/exact-capture-expcap-padding.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cisco/exact-capture/2a7b74e66cbbafcec7c39947dea63bec03972685/docs/src/img/exact-capture-expcap-padding.png -------------------------------------------------------------------------------- /docs/src/img/exact-capture-expcap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cisco/exact-capture/2a7b74e66cbbafcec7c39947dea63bec03972685/docs/src/img/exact-capture-expcap.png -------------------------------------------------------------------------------- /docs/src/img/exact-capture-toolchain.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cisco/exact-capture/2a7b74e66cbbafcec7c39947dea63bec03972685/docs/src/img/exact-capture-toolchain.png -------------------------------------------------------------------------------- /docs/src/index.md: -------------------------------------------------------------------------------- 1 | Exact Capture is a high-rate, lossless packet capture solution for ExaNIC network adapters. 2 | The system is fully open source and designed for performance as well as ease of configuration. 3 | It can be used with any ExaNIC network adapter, and is optimised for use with ExaDISK high speed NVMe SSDs. 4 | The system can be deployed on any suitably powerful server system. 5 | 6 | This manual covers full details on configuring and operating Exact Capture including: 7 | 8 | * [Quick start guide](quick.md) 9 | * [Installing and building](install) 10 | * [Hardware requirements](server.md) 11 | * [Configuration options](config.md) 12 | * [Version history](versions.md) 13 | -------------------------------------------------------------------------------- /docs/src/install.md: -------------------------------------------------------------------------------- 1 | ## Source Code and Licensing 2 | Exact Capture is available from [github.com](https://github.com/exablaze-oss/exact-capture) as an open source project. 3 | If you would like to discuss alternative licensing schemes, please contact the Exablaze sales team. 4 | 5 | ## Hardware Requirements 6 | Exact Capture requires a high performance server to operate optimally. 7 | Please read the [Server Requirements](server.md) for more details. 8 | 9 | ## Software Requirements 10 | To build the software, you will need a recent C compiler supporting C99 or higher, and to have installed the ExaNIC software libraries (also available from github.com). 11 | 12 | ##Building 13 | There are 3 build options for Exact Capture: 14 | 15 | 1. Performance build 16 | 2. Error assertions build 17 | 3. Debug build 18 | 19 | By default, Exact Capture is built in performance mode. 20 | In performance mode, unnecessary internal error checking is disabled. 21 | For example, bounds checks on memory access. 22 | To build Exact Capture in performance mode, simply run `make` in the top level. 23 | 24 | To build a version with stricter internal error checking assertions, run `make assert`. 25 | This version is still capable of operating at 10Gb/s on many systems, though will suffer marginal performance degradation, especially on slower CPUs. 26 | 27 | To build a debug version, run `make debug`. 28 | The debug build applies stricter warning checking requirements at build time, and enables detailed debug tracing throughout the application. 29 | This version is unlikely to keep up at high-rate. 30 | 31 | ## (un)Installation 32 | To install Exact Capture, run `make install` as the root user. To uninstall, run `make uninstall` as the root user. 33 | -------------------------------------------------------------------------------- /docs/src/quick.md: -------------------------------------------------------------------------------- 1 | To run Exact Capture on a single 10GbE interface, writing to a single disk slice, the following command line is sufficient: 2 | 3 | ```bash 4 | $ exact-capture --input=exanic0:0 --output=/data0/ --cpus=0:1:2 5 | ``` 6 | 7 | !!! note 8 | Canonical Linux interface names such as “eth1” or “enp0s1” can be freely used in place of Exablaze ExaNIC device names (e.g. “exanic0:0”). 9 | The interface must however be an ExaNIC. 10 | 11 | !!! note 12 | The CPU specification is a colon (“:”) separated list containing the management CPU, the ExaNIC listener thread CPU(s), and the ExaDisk writer thread CPU(s). 13 | It is assumed that CPU cores have been isolated according to the System Configuration instructions above. 14 | For more details see [configuration options](config.md#cpus) 15 | 16 | To run exact capture on a pair of 10GbE interfaces, writing to a two disk slices, using 5 cpu cores (management core = 0, NIC listener cores = 1,2, disk writer cores = 3,4): 17 | 18 | ```bash 19 | $ exact-capture --input=exanic0:0 --input=exanic0:1 --output=/data0/ --output=/data1/ --cpus=0:1,2:3,4 20 | ``` 21 | -------------------------------------------------------------------------------- /docs/src/server.md: -------------------------------------------------------------------------------- 1 | You are free to run Exact Capture on any system hardware. 2 | The following table describes our recommendations for the minimum system hardware requirements. 3 | For general guidance, we have successfully run the system on suitably configured Dell R230 and R730 machines. 4 | Any of the Dell R*30 and R*40 machines are likely to be excellent candidates. 5 | 6 | ## CPUs 7 |

Core Count

8 | The number of CPU cores required depends on: 9 | 10 | 1. The number of interfaces that you wish to capture on 11 | 2. The type and speed of the disk drives 12 | 3. The maximum capture rate you need to sustain 13 | 14 | Assuming that **ExaDisks** are the target drives, each drive slice is capable of writing at a sustained rate of 10Gb/s. 15 | 16 | As an example, a minimal 10Gb/s Exact Capture installation will require 3 CPU cores. 17 | One core for a (hot) listener thread, one core for a (cold) disk writer thread and one management core. 18 | The management core is low priority and can be safely shared with other general purpose system cores. 19 | The listener thread should not be shared with any other process (i.e. be sure to make use of the isolcpus kernel parameter). 20 | 21 | In general, for n line-rate 10G ports, the system requires 2n + 1 CPU cores. e.g a 4x10G capture system will require 9 cores in total. 22 | 23 | !!! warning 24 | CPU core counts are based on actual cores, rather than hyperthreads. 25 | In general, we recommend disabling hyperthreads on CPUs that support them. 26 | 27 | !!! tip 28 | We have had good results with Intel Xeon E5-26xx range CPUs with a 3Ghz+ clock speed. For example the Intel Xeon E5-2643. 29 | 30 | 31 |

Speed

32 | The minimum required CPU speed depends on the maximum capture rate required. 33 | For the purposes of this document, we assume that 10G line rate, at minimum sized (64B) frames is the capture rate requirement (i.e. approx. 14 million packets per second ). 34 | 35 | !!! tip 36 | We have found that 3Ghz+ CPUs are sufficient. 37 | 38 | 39 | ##RAM 40 | RAM usage will vary based on the number of NICs and disks that you are using. 41 | By default, each memory queue is organised into 256x 2MB slots for a total memory usage of approximately 512MB per queue. 42 | The total number of memory queues the product of the number of hot (ExaNIC) and cold (ExaDisk) threads. 43 | For a minimal 10Gb/s capture solution, with a single ExaNIC and ExaDisk, only 1 memory queue is required for a total of approximately 512MB of memory. 44 | For 4x10Gbs system, with 4 disks, 4x4 = 16 queues will be required, for a minimum memory usage of ~8GB. 45 | 46 | !!! tip 47 | We recommend at least 16GB of DDR IV RAM in your machine. 48 | 49 | 50 | ##PCIe 51 | For sustained, minimum sized packet capture, each 10Gb/s ExaNIC interface requires approximately 4x PCIe Gen 3 lanes. 52 | The hot threads must run on the CPU socket directly connected to these PCIe lanes. 53 | 54 | For sustained high performance writing, each ExaDisk interface requires 2x PCIe Gen 3 lanes. 55 | The cold threads must run on the CPU socket directly connected to these PCIe lanes. 56 | 57 | !!! tip 58 | For optimal performance, w recommend running PCIe Gen3x8 for all cards connected. 59 | 60 | 61 | ##ExaNIC 62 | All ExaNIC network cards will work with Exact Capture. 63 | Following is a summary of the features, requirements and limitations of each card: 64 | 65 | - **ExaNIC X10 / GM (2x 10GbE)** - these cards can be used without restriction on suitable PCIe Gen 3x8 slots. 66 | Timestamp resolution is 6.2ns. 67 | - **ExaNIC HPT (2x 10GbE)** - these cards can be used without restriction on suitable PCIe Gen 3x8 slots. 68 | Timestamp resolution is 0.25ns (250ps) 69 | - **ExaNIC X40 / VXP (8x 10GbE)** - Only 2 ports can be used at line rate for all packet sizes. Up to 4 ports can be used at larger (average) packet sizes (e.g. 512B+). 70 | Timestamp resolution is 6.2ns. 71 | - **ExaNIC X40 (2x 40GbE)** - Speeds up to 20Gb/s are likely to work out of the box on any single interface (though this is untested). 72 | Load balancing/packet spraying across multiple receive rings is also likely to assist line rate capture, though this is feature is not (yet) implemented. 73 | 74 | !!! tip 75 | ExaNIC X10 and ExaNIC HPT devices are currently optimal. 76 | 77 | 78 | 79 | ##Disk Drives 80 | Exact Capture is tested and optimized to run on *ExaDisk FX1* NVMe SSD drives. 81 | Each ExaDisk is capable of running 40Gb/s sustained write speed to disk in a PCIe Gen 3x 8 slot. 82 | The drives are currently available in 4TB and 8TB capacities. 83 | 84 | The system will (in principle) operate with any disks. 85 | High speed flash drives, especially NVMe disks are highly recommended to keep the number of threads and memory usage down. 86 | For slower disks (e.g. SATA based flash disks) sharing CPU cores for writer threads is likely to reduce the CPU core count requirements without affecting overall performance. 87 | This is untested. 88 | 89 | !!! tip 90 | ExaDISK FX1 (8TB) is the recommended disk drive 91 | -------------------------------------------------------------------------------- /docs/src/tools/analyze.md: -------------------------------------------------------------------------------- 1 | # Exact PCAP Analyze 2 | 3 | Exact PCAP analyze (`exact-pcap-analyze`) is a tool used to analyze `.pcap` files and derive a number of statistics concerning a given capture. 4 | It will determine the total length of the capture (in nanoseconds, microseconds and seconds), the average packet rate (in pps, packets per second), the minimum/maximum inter-frame gaps (IFGs) and the throughput rate (in gigabits per second). 5 | 6 | The following table lists all commands available: 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 22 | 23 | 24 | 25 | 26 | 27 | 30 | 31 | 32 | 33 | 34 | 35 | 38 | 39 | 40 | 41 | 42 | 43 | 46 | 47 | 48 | 49 | 50 | 51 | 54 | 55 | 56 | 57 | 58 | 59 | 62 | 63 |
ShortLongDefaultDescription
iinput(required) 20 | The input .pcap file to analyze. 21 |
fformat(required) 28 | The input file format. Valid values are pcap or expcap. 29 |
rline-rate(required) 36 | The line rate of traffic in the input capture. This value is in Gbps, e.g. 10 = 10Gb/s. 37 |
ooffset0 44 | Start analyzing packets from this packet offset into the input file. 45 |
mmax-1 (no limit) 52 | Limit the number of packets to analyze. 53 |
wwindow100 60 | Analyze packets within this window. Analysis occurs each time this window is exceeded in the input capture. 61 |
64 | -------------------------------------------------------------------------------- /docs/src/tools/extract.md: -------------------------------------------------------------------------------- 1 | # Exact Extract 2 | 3 | Exact Extract is a utility that extracts packets from one or more expcap files and writes them out into a user-specified format. It outputs capture files that are 4 | ordered by the expcap timestamps present in the original capture. 5 | 6 | ```bash 7 | $ exact-pcap-extract -i cap0-0.expcap -w extracted -a -f expcap 8 | ``` 9 | 10 | !!! Note 11 | Exact Extract can take multiple expcap files as its input. It will search for the earliest timestamp in all of the input capture files. 12 | This means that the first packet written out will be the packet with the earliest timestamp across all of the input files. 13 | 14 | The following table lists all commands available: 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 30 | 31 | 32 | 33 | 34 | 35 | 38 | 39 | 40 | 41 | 42 | 43 | 47 | 48 | 49 | 50 | 51 | 52 | 56 | 57 | 58 | 59 | 60 | 61 | 65 | 66 | 67 | 68 | 69 | 70 | 73 | 74 | 75 | 76 | 77 | 78 | 81 | 82 | 83 | 84 | 85 | 86 | 89 | 90 | 91 | 92 | 93 | 94 | 99 | 100 | 101 | 102 | 103 | 104 | 107 | 108 | 109 | 110 | 111 | 112 | 115 | 116 | 117 | 118 | 119 | 120 | 123 | 124 | 125 | 126 | 127 | 128 | 132 | 133 | 134 | 135 | 136 | 137 | 143 | 144 | 145 | 146 | 147 | 148 | 165 | 166 |
ShortLongDefaultDescription
iinput(required) 28 | The expcap files to extract packets from. 29 |
wwrite(required) 36 | The filename used for extracted packets. Exact Extract will automatically append .pcap to the filename. 37 |
Wwrite-dir(none) 44 | The directory used to write extracted packets. If used in conjunction with '--write', packets are extracted to {write-dir}/{write}.pcap. 45 | It must be specified if steering options (--steer) are in use. 46 |
pport-1 53 | Only extract packets that were captured on the specified port number. If this is not specified and '--device' is not set, the user must use '--all' to 54 | extract packets that were captured on all ports. 55 |
ddevice-1 62 | Only extract packets that were captured on the specified device number (i.e., 0 corresponds to exanic0). If this is not specified and '--port' is not 63 | set, the user must use '--all' to extract packets that were captured on all ports. 64 |
aall(flag) 71 | Extract packets that were captured on all ports and all devices. This flag must be set if neither '--port' or '--device' are specified. 72 |
fformatexpcap 79 | The file format to use when writing out packets. Valid options are 'pcap' and 'expcap'. 80 |
ccount0 (no limit) 87 | The maximum number of files to write out in total. 88 |
Mmaxfile0 (128MB) 95 | The maximum file size for output files, in MB. Output files will never exceed 128MB, regardless of the value specified with this option. 96 | If the output must be spread over multiple files, Exact Extract will append _n to the filename, where n is the current segment being written out 97 | (starting from 1). 98 |
uusepcap(flag) 105 | If set, Exact Extract will write pcap timestamps in the microsecond format. 106 |
Ssnaplen1518 113 | The limit in bytes for each individual packet in the output file. 114 |
rskip-runts(flag) 121 | If this flag is set, runt frames will not be written to the output 122 |
Dallow-duplicates(flag) 129 | By default, Exact Extract will write over files which have the same name as specified by '--write'. If this flag is set, it will instead create a file 130 | with a duplicate name, and append __n to the duplicate file, where n is the current number of files with the same name. 131 |
thpt-trailer(flag) 138 | If this flag is set, Exact Extract will assume that each packet terminates in a Fusion HPT timestamp trailer. It will extract the timestamp from this 139 | trailer and write it to the pcap header (in microsecon/nanosecond format) and to the expcap trailer (in picosecond format) for all output files. 140 |

Note: this option assumes all packets in the input captures have Fusion HPT trailers. If this is not 141 | true, the pcap/expcap timestamp fields will contain invalid values in the output file(s). 142 |
ssteer(null) 149 | --steer accepts one of the following values as valid options: hpt, vlan, expcap. 150 | A directory must be specified when using this option (via '--write-dir') 151 | If this option is set, Exact Extract will steer packets to separate files depending upon their content. 152 |

153 | Specifying '--steer vlan' will cause packets to be steered to separate files depending upon their VLAN ID. For example, packets with VLAN ID 100 will 154 | be steered to the file {write-dir}/{write}_vlan_100.pcap. Packets which do not have an 8021.Q tag will be written to the file named 155 | {write-dir}/{write}.pcap. 156 |

157 | Specifying '--steer hpt' will cause packets to be steered to separate files depending upon their Fusion HPT trailer, using the device ID and port 158 | number. For example, files with device ID 0 and port number 10 will be steered to the file {write-dir}/{write}_device_0_port_10.pcap. 159 |
Note: use of this option assumes that all packets in the input file(s) have a Fusion HPT trailer. 160 |

161 | Specifying '--steer expcap' will cause packets to be steered to separate files depending upon their expcap footer, using the device ID and port 162 | number. For example, files with device ID 0 and port number 10 will be steered to the file {write-dir}/{write}_device_0_port_10.pcap. 163 |
Note: use of this option assumes that all packets in the input file(s) have an expcap footer. 164 |
167 | 168 | !!! Note 169 | Depending upon the options specified by, Exact Extract may attempt to keep more files open simultaneously than is allowed the OS. 170 | If it is unable to override this limit, it will need to close and open outputs each time a packet is written in order to ensure that it does not exceed this limit. 171 | This can degrade the performance of Exact Extract, due to the additional overhead of repeatedly opening and closing output files. 172 | 173 | On Linux, it is possible to override the number of open file descriptors limit by giving Exact Extract the "CAP_SYS_RESOURCE" capability to the binary. 174 | This can be accomplished by running: 175 | 176 | ``` 177 | $ sudo setcap cap_sys_resouce+ep exact-pcap-extract 178 | ``` 179 | 180 | The full list of permissions granted by CAP_SYS_RESOURCE is available on the [capabilities(7)](https://man7.org/linux/man-pages/man7/capabilities.7.html) 181 | man page. Granting Exact Extract this permission should improve its performance, however it is not a requirement. Users will receive the warning "Could not 182 | raise the limit on concurrently open files" if they are exceeding the open file limit. -------------------------------------------------------------------------------- /docs/src/tools/match.md: -------------------------------------------------------------------------------- 1 | # Exact PCAP Match 2 | 3 | Exact PCAP Match (`exact-pcap-match`) is a tool used to match packets in two separate files. 4 | It outputs timestamps, timestamp deltas (latency) and the packet data of matched packets in a `.csv` file. 5 | 6 | Where timestamp deltas are shown, they are produced from `reference_file.timestamp minus input_file.timestamp`. 7 | 8 | The following table lists all commands available: 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 24 | 25 | 26 | 27 | 28 | 29 | 32 | 33 | 34 | 35 | 36 | 37 | 40 | 41 | 42 | 43 | 44 | 45 | 49 | 50 | 51 | 52 | 53 | 54 | 58 | 59 | 60 | 61 | 62 | 63 | 66 | 67 | 68 | 69 | 70 | 71 | 74 | 75 | 76 | 77 | 78 | 79 | 82 | 83 | 84 | 85 | 86 | 87 | 90 | 91 | 92 | 93 | 94 | 95 | 98 | 99 | 100 | 101 | 102 | 103 | 106 | 107 |
ShortLongDefaultDescription
rreference(required) 22 | The reference file to use for matching. 23 |
iinput(required) 30 | The file to compare against the reference file. 31 |
ccsv(required) 38 | The .csv file to write matching packet statistics to. 39 |
Rref-miss(optional) 46 | The .csv file to store reference packet misses. 47 | These are packets which were present in the reference file but were not found in the input file. 48 |
Iinp-miss(optional) 55 | The .csv file to store input packet misses. 56 | These are packets which were present in the input file but were not found in the reference file. 57 |
fformat(required) 64 | The input file format. Valid values are pcap or expcap. 65 |
Ooffset-ref0 72 | Start matching packets in the reference file from this packet offset onwards. 73 |
ooffset-inp0 80 | Start matching packets in the input file from this packet offset onwards. 81 |
Mmax-ref-1 (no limit) 88 | Limit the amount of packets in the reference file to match. 89 |
nnum-chars64 96 | The number of bytes from matched packets to output in the packet data field of the .csv file. 97 |
vverbose(flag) 104 | Enables verbose output printing. 105 |
108 | -------------------------------------------------------------------------------- /docs/src/tools/modify.md: -------------------------------------------------------------------------------- 1 | # Exact PCAP Modify 2 | 3 | Exact PCAP Modify (`exact-pcap-modify`) is a tool which can be used to filter and modify Ethernet packets. 4 | Where modifications occur which would invalidate a checksum, Exact PCAP Modify will recalculate a correct checksum in the output file. 5 | 6 | It can perform modify/filter operations on the following fields: 7 | 8 | * Ethernet header: 9 | * Destination MAC address (DST MAC) 10 | * Souce MAC address (SRC MAC) 11 | * 8021.Q tag: 12 | * VLAN ID 13 | * IPv4 header: 14 | * Source address (SRC IP) 15 | * Destination address (DST IP) 16 | * Time-to-live (IP TTL) 17 | * UDP/TCP headers: 18 | * Source port (SRC PORT) 19 | * Destination port (DST PORT) 20 | 21 | ``` 22 | $ exact-pcap-modify -i capture.pcap -f expcap -w modified -a 1.1.1.1,2.2.2.2 23 | ``` 24 | 25 | # Configuration 26 | 27 | Options which operate on the fields of a header are to be supplied in the form filter,modify. 28 | For example, if Exact PCAP Modify should filter a capture such that only packets with the SRC IP of 1.1.1.1 are present written to the output, it should be invoked with the `--src-ip 1.1.1.1` option. 29 | 30 | If the user wishes to modify all packets which have the SRC IP 1.1.1.1 to produce an output where the SRC IP becomes 2.2.2.2, it should be invoked with the `--src-ip 1.1.1.1,2.2.2.2` option. 31 | 32 | When multiple filters are used, packets will need to match **all** of the specified filters to be present in the output capture. 33 | 34 | The following table lists all commands available: 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 50 | 51 | 52 | 53 | 54 | 55 | 58 | 59 | 60 | 61 | 62 | 63 | 66 | 67 | 68 | 69 | 70 | 71 | 74 | 75 | 76 | 77 | 78 | 79 | 82 | 83 | 84 | 85 | 86 | 87 | 90 | 91 | 92 | 93 | 94 | 95 | 98 | 99 | 100 | 101 | 102 | 103 | 106 | 107 | 108 | 109 | 110 | 111 | 114 | 115 | 116 | 117 | 118 | 119 | 123 | 124 | 125 | 126 | 127 | 128 | 132 | 133 | 134 | 135 | 136 | 137 | 146 | 147 | 148 | 149 | 150 | 151 | 155 | 156 | 157 | 158 | 159 | 160 | 164 | 165 | 166 | 167 | 168 | 169 | 173 | 174 | 175 | 176 | 177 | 178 | 182 | 183 | 184 | 185 | 186 | 187 | 191 | 192 | 193 | 194 | 195 | 196 | 200 | 201 |
ShortLongDefaultDescription
iinput(required) 48 | The .pcap/.expcap file to filter/modify. 49 |
wwrite(required) 56 | The destination file where packets which match the specified filter will be written to. 57 |
Wwrite-filtered(optional) 64 | The destination file where all packets which do not match all specified filters will be written to. 65 |
vverbose(flag) 72 | Enables verbose output printing. 73 |
fformatexpcap 80 | The file format to use when writing out packets. Valid options are .pcap and .expcap. 81 |
ooffset0 88 | Jump to the specified packet offset and start processing filter/modify operations from there. 89 |
ttime0 (Epoch time in ns) 96 | Skip all packets that are older than this time. 97 |
mmax0 (no limit) 104 | The maximum number of packets to write out in total. 105 |
nnum-chars64 112 | For use with --verbose. Limit the amount of characters written when dumping packets. 113 |
e--dst-mac(null) 120 | Filter/modify based on the DST MAC. 121 | Accepts arguments in the form 0x001122334455 for filtering and 0x001122334455,0xAABBCCDDEEFF for filtering and modifying. 122 |
E--src-mac(null) 129 | Fitler/modify based on the SRC MAC. 130 | Accepts arguments in the form 0x001122334455 for filtering and 0x001122334455,0xAABBCCDDEEFF for filtering and modifying. 131 |
lvlan(null) 138 | Filter/modify based on the VLAN ID. 139 | Accepts arguments in the form 100 for filtering and 100,200 for filtering and modifying. 140 |

141 | The --vlan option also allows users to strip and add 8021.Q tags in addition to filtering and modifying based on these tags. 142 | For example if --vlan 0,100 is specified, that will cause Exact PCAP Modify to add an 8021.Q header with a VLAN ID of 100 to all non-8021.Q frames. 143 |

144 | The inverse usage, --vlan 100,0 will cause Exact PCAP Modify to strip all 8021.Q frames which have VLAN ID 100 and set the Ethertype to IPv4. 145 |
asrc-ip(null) 152 | Fitler/modify based on the SRC IP. 153 | Accepts arguments in the form 1.1.1.1 for filtering and 1.1.1.1,2.2.2.2 for filtering and modifying. 154 |
Adst-ip(null) 161 | Fitler/modify based on the DST IP. 162 | Accepts arguments in the form 1.1.1.1 for filtering and 1.1.1.1,2.2.2.2 for filtering and modifying. 163 |
Tip-ttl(null) 170 | Fitler/modify based on the IP TTL. 171 | Accepts arguments in the form 10 for filtering and 10,64 for filtering and modifying. 172 |
psrc-port(null) 179 | Fitler/modify based on the SRC PORT of the TCP/UDP header. 180 | Accepts arguments in the form 1000 for filtering and 1000,2000 for filtering and modifying. 181 |
Pdst-port(null) 188 | Fitler/modify based on the DST PORT of the TCP/UDP header. 189 | Accepts arguments in the form 1000 for filtering and 1000,2000 for filtering and modifying. 190 |
ddevice-typenexus3548 197 | When modifying frames, emulate the behaviour of the specified device. 198 | Valid values are: nexus3548, fusion, triton, arista7150. 199 |
202 | -------------------------------------------------------------------------------- /docs/src/tools/parse.md: -------------------------------------------------------------------------------- 1 | # Exact PCAP Parse 2 | 3 | Exact PCAP Parse (`exact-pcap-parse`) is a tool used to create ASCII dumps from `.pcap` and `.expcap` files. It outputs timestamps and packet data in `.csv` files. 4 | 5 | The following table lists all commands available: 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 21 | 22 | 23 | 24 | 25 | 26 | 29 | 30 | 31 | 32 | 33 | 34 | 37 | 38 | 39 | 40 | 41 | 42 | 45 | 46 | 47 | 48 | 49 | 50 | 53 | 54 | 55 | 56 | 57 | 58 | 61 | 62 | 63 | 64 | 65 | 66 | 69 | 70 |
ShortLongDefaultDescription
iinput(required) 19 | The input `.pcap` file to parse and dump. 20 |
ccsv(required) 27 | The `.csv` file to write out the dumped timestamps and packet data. 28 |
vverbose(flag) 35 | Enables verbose output printing. 36 |
fformat(required) 43 | The input file format. Valid values are pcap or expcap. 44 |
ooffset0 51 | Start parsing packets from this packet offset into the input file. 52 |
mmax-1 (no limit) 59 | Limit the number of packets to parse. 60 |
nnum-chars64 67 | The number of bytes parse and output in the packet data field of the .csv file. 68 |
71 | -------------------------------------------------------------------------------- /docs/src/versions.md: -------------------------------------------------------------------------------- 1 | ## Version 1.2.1 2 | **Release date:** 8 March 2021
3 | **Commit ID:** [5025a3682a26179808574e1261be94da93341db3](https://github.com/exablaze-oss/exact-capture/commit/5025a3682a26179808574e1261be94da93341db3)
4 | **Source:** [https://github.com/exablaze-oss/exact-capture/releases/tag/v1.2.1](https://github.com/exablaze-oss/exact-capture/releases/tag/v1.2.1) 5 | 6 | **Release Notes:** 7 | 8 | * Fix non-fatal compiler warning: enumeration value ‘PKT_OVER_SNAPLEN’ not handled in switch 9 | 10 | --------------------------------------- 11 | 12 | ## Version 1.2 13 | **Release date:** 5 March 2021
14 | **Commit ID:** [fdf4d555e768d839eb8d98d66aac70ec4f34d4ad](https://github.com/exablaze-oss/exact-capture/commit/fdf4d555e768d839eb8d98d66aac70ec4f34d4ad)
15 | **Source:** [https://github.com/exablaze-oss/exact-capture/releases/tag/v1.2](https://github.com/exablaze-oss/exact-capture/releases/tag/v1.2) 16 | 17 | **Release Notes:** 18 | 19 | * Moved all utilities into /tools directory 20 | * Moved all utilities to a use common code-base for PCAP read/write operations 21 | * Add exact-pcap-analyze utility for PCAP statistics generation (useful for performance debugging) 22 | * Add exact-pcap-modify utility for easy PCAP search/modify operations (useful for switch benchmarking) 23 | * Improve exact-pcap-extract to understand N3550-F HPT trailer format 24 | * Improve exact-pcap-extract to filter based on H3550-F HPT trailer format or VLAN tag header 25 | * Add performance optimisation and tuning guide 26 | * Add utility and workflow overview documentation 27 | * Add exact-pcap-extract detailed documentation 28 | * Add exact-pcap-match detailed documentation 29 | * Add exact-pacp-analyze detailed documentation 30 | * Add exact-pcap-parse detailed documentation 31 | * Add exact-pcap-modify detailed documentation 32 | 33 | --------------------------------------- 34 | 35 | ## Version 1.1 36 | **Release date:** 30 September 2019
37 | **Commit ID:** [5d337a645987af00fc0d1afa6769ee7ccf62caea](https://github.com/exablaze-oss/exact-capture/commit/5d337a645987af00fc0d1afa6769ee7ccf62caea)
38 | **Source:** [https://github.com/exablaze-oss/exact-capture/tree/v1.1](https://github.com/exablaze-oss/exact-capture/tree/v1.1) 39 | 40 | **Release Notes:** 41 | 42 | * Moved documentation online use [mkdocs](https://www.mkdocs.org/) and [github pages](https://pages.github.com/) 43 | * Removed PDF documentation 44 | * Updated readme 45 | 46 | --------------------------------------- 47 | 48 | ## Version 1.0 49 | **Release date:** 7 March 2018
50 | **Commit ID:** [624a3f97bbaddd3cf827387e2352d5f5c662a7e5](https://github.com/exablaze-oss/exact-capture/commit/624a3f97bbaddd3cf827387e2352d5f5c662a7e5)
51 | **Source:** [https://github.com/exablaze-oss/exact-capture/tree/v1.0](https://github.com/exablaze-oss/exact-capture/tree/v1.0) 52 | 53 | **Release Notes:** 54 | 55 | * Initial release of Exact Capture 56 | * PDF documentation included in repo 57 | 58 | --------------------------------------- 59 | -------------------------------------------------------------------------------- /libs/chaste: -------------------------------------------------------------------------------- 1 | libchaste/include/ -------------------------------------------------------------------------------- /libs/libchaste/60740670f7763b10d1b022c6d1709434b39c2a5d: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cisco/exact-capture/2a7b74e66cbbafcec7c39947dea63bec03972685/libs/libchaste/60740670f7763b10d1b022c6d1709434b39c2a5d -------------------------------------------------------------------------------- /libs/libchaste/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013, Matthew P. Grosvenor 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | * Redistributions of source code must retain the above copyright 7 | notice, this list of conditions and the following disclaimer. 8 | * Redistributions in binary form must reproduce the above copyright 9 | notice, this list of conditions and the following disclaimer in the 10 | documentation and/or other materials provided with the distribution. 11 | * Neither the name of the Matthew P. Grosvenor nor the 12 | names of its contributors may be used to endorse or promote products 13 | derived from this software without specific prior written permission. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 16 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 17 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 | DISCLAIMED. IN NO EVENT SHALL MATTHEW P. GROSVENOR BE LIABLE FOR ANY 19 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 20 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 21 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 22 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 24 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | -------------------------------------------------------------------------------- /libs/libchaste/README: -------------------------------------------------------------------------------- 1 | ------------------------------------------------------------------------------- 2 | Pure, virtuous, unadulterated programming bliss - hastily assembled C programs. 3 | ------------------------------------------------------------------------------- 4 | 5 | The C-Haste Library (libchaste) 6 | =============================== 7 | 8 | License: 9 | -------- 10 | See LICENSE file included. TL;DR - BSD 3 Clause. 11 | 12 | 13 | Main features: 14 | -------------- 15 | Data structures: generic, type safe array, vector and linked list (hash map partially implemented) 16 | 17 | Type safe command line options parser with super simple syntax, usage text etc. 18 | 19 | Powerful logging with runtime and compile time log level tuning, coloured output. 20 | 21 | 22 | Todo/In progress 23 | ----------------- 24 | Architecture agnositc performance monitoring and measuring tools around RDTSC/PMC/PDU 25 | 26 | Yet another string library to efficiently do useful string manipulations using nice macros to make 27 | 28 | 29 | Currently builds on: 30 | --------------------- 31 | - x86 Linux (Ubtuntu 12.10/13.04) (clang 3.2 and gcc 4.6-4.8) 32 | - x86 Darwin (Mac OSX 10.10/10.11) (clang 7.0) 33 | 34 | Targeted to build on: 35 | - Linux/BSD/Darwin (OS X) 36 | - 32bit/64bit (64bit optimized) 37 | - ARM / x86 38 | 39 | 40 | Obtaining the source: 41 | ---------------------- 42 | 43 | The C-Haste library is available at https://github.com/mgrosvenor/libchaste 44 | 45 | - The master branch is a "stable" "release" candidate -- the bleeding edge - guaranteed compile and pass unit tests -- API may vary wildly. 46 | - Release branches are stable, tagged with the release ID. Incremental fixes to a release will be tags added to the release branch. Bug fixes only. No new features. No new APIs. 47 | 48 | 49 | Building 50 | --------- 51 | 52 | Libchaste builds on both gcc 4.6 and clang 3.2 against the c11 standard with pedantic warnings and errors using the Posix 2008 system API. 53 | 54 | It includes a set of unit tests and infrastructure for making them all running and passing valgrind tests. 55 | 56 | - run build.sh to build a debug libcahste.a binary in the bin/ directory. 57 | - run build_release.sh to build a release binary libcahste.tar.bz 58 | 59 | Requirements 60 | ------------- 61 | Cake build system - https://github.com/Zomojo/Cake 62 | Clang 3.2+ / GCC 4.6+ 63 | 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /libs/libchaste/include/asm/asm.h: -------------------------------------------------------------------------------- 1 | // CamIO 2: asm.h 2 | // Copyright (C) 2013: Matthew P. Grosvenor (matthew.grosvenor@cl.cam.ac.uk) 3 | // Licensed under BSD 3 Clause, please see LICENSE for more details. 4 | 5 | #ifndef ASM_H_ 6 | #define ASM_H_ 7 | 8 | 9 | 10 | #endif /* ASM_H_ */ 11 | -------------------------------------------------------------------------------- /libs/libchaste/include/chaste.h: -------------------------------------------------------------------------------- 1 | /* 2 | * libm6.h 3 | * 4 | * Created on: Jun 29, 2013 5 | * Author: mgrosvenor 6 | */ 7 | 8 | #ifndef LIBM6_H_ 9 | #define LIBM6_H_ 10 | 11 | #include "types/types.h" 12 | #include "utils/util.h" 13 | #include "string/string.h" 14 | 15 | #include "options/options.h" 16 | //#include "perf/perf.h" 17 | #include "log/log.h" 18 | //#include "perf/perf_mon.h" 19 | 20 | #include "data_structs/array/array_std.h" 21 | #include "data_structs/vector/vector_std.h" 22 | #include "data_structs/linked_list/linked_list_std.h" 23 | #include "data_structs/hash_map/hash_map.h" 24 | #include "data_structs/function_hash_map/function_hash_map.h" 25 | 26 | #endif /* LIBM6_H_ */ 27 | -------------------------------------------------------------------------------- /libs/libchaste/include/data_structs/array/array.h: -------------------------------------------------------------------------------- 1 | // CamIO 2: array.h 2 | // Copyright (C) 2013: Matthew P. Grosvenor (matthew.grosvenor@cl.cam.ac.uk) 3 | // Licensed under BSD 3 Clause, please see LICENSE for more details. 4 | 5 | #ifndef ARRAY_H_ 6 | #define ARRAY_H_ 7 | 8 | #include "../../types/types.h" 9 | 10 | struct ch_array; 11 | typedef struct ch_array ch_array_t; 12 | 13 | 14 | struct ch_array{ 15 | ch_word size; //Return the max number number of elements in the array list 16 | 17 | void* first; //Pointer to the fist valid entry list. Not valid if first == end 18 | void* last; //Pointer to the last valid element in the array. Not valid if last == end 19 | void* end; //Pointer to the one element beyond the end of the valid elements in array. Do not dereference! 20 | 21 | // Members prefixed with "_" are nominally "private" Don't touch my privates! 22 | int (*_cmp)(const void *, const void *); // Comparator function for find and sort operations 23 | void* _array_backing; //Actual array storage 24 | ch_word _array_backing_size; //Number of elements allocated in the given array 25 | ch_word _element_size; 26 | 27 | }; 28 | 29 | 30 | //Resize the array 31 | void array_resize(ch_array_t* this, ch_word new_size); 32 | 33 | //Check for equality 34 | ch_word array_eq(ch_array_t* this, ch_array_t* that); 35 | 36 | //Return the element at a given offset, with bounds checking 37 | void* array_off(ch_array_t* this, ch_word idx); 38 | 39 | //Step forwards by one entry 40 | void* array_next(ch_array_t* this, void* ptr); 41 | 42 | //Step backwards by one entry 43 | void* array_prev(ch_array_t* this, void* ptr); 44 | 45 | //Step forwards by amount 46 | void* array_forward(ch_array_t* this, void* ptr, ch_word amount); 47 | 48 | //Step backwards by amount 49 | void* array_back(ch_array_t* this, void* ptr, ch_word amount); 50 | 51 | //find the given value using the comparitor function 52 | void* array_find(ch_array_t* this, void* begin, void* end, void* value); 53 | 54 | //return the offset/index of the given item 55 | int array_get_idx(ch_array_t* this, void* value); 56 | 57 | //sort into order given the comparitor function 58 | void array_sort(ch_array_t* this); 59 | 60 | //sort into reverse order given the comparitor function 61 | void array_sort_reverse(ch_array_t* this); 62 | 63 | //Free the resources associated with this array, assumes that individual items have been freed 64 | void array_delete(ch_array_t* this); 65 | 66 | //Set at most count elements to the value in carray starting at offset in this array 67 | void* array_from_carray(ch_array_t* this, void* carray, ch_word count); 68 | 69 | 70 | ch_array_t* ch_array_new(ch_word size, ch_word element_size, int(*cmp)(const void* lhs, const void* rhs)); 71 | 72 | #endif //ARRAY_H_ 73 | -------------------------------------------------------------------------------- /libs/libchaste/include/data_structs/array/array_std.h: -------------------------------------------------------------------------------- 1 | /* 2 | * array_std.h 3 | * 4 | * Created on: Sep 12, 2013 5 | * Author: mgrosvenor 6 | */ 7 | 8 | #ifndef ARRAY_STD_H_ 9 | #define ARRAY_STD_H_ 10 | 11 | #include "array_typed_declare_template.h" 12 | 13 | declare_array(u8, u8) 14 | declare_array(u16, u16) 15 | declare_array(u32, u32) 16 | declare_array(u64, u64) 17 | 18 | declare_array(u8p, u8*) 19 | declare_array(u16p, u16*) 20 | declare_array(u32p, u32*) 21 | declare_array(u64p, u64*) 22 | 23 | declare_array(i8, i8) 24 | declare_array(i16, i16) 25 | declare_array(i32, i32) 26 | declare_array(i64, i64) 27 | 28 | declare_array(i8p, i8*) 29 | declare_array(i16p, i16*) 30 | declare_array(i32p, i32*) 31 | declare_array(i64p, i64*) 32 | 33 | declare_array(machine, ch_machine) 34 | declare_array(word, ch_word) 35 | declare_array(char, ch_char) 36 | declare_array(ch_bool, ch_bool) 37 | declare_array(float, ch_float) 38 | declare_array(string, ch_str) 39 | 40 | 41 | declare_array(machinep, ch_machine*) 42 | declare_array(wordp, ch_word*) 43 | declare_array(charp, ch_char*) 44 | declare_array(boolp, ch_bool*) 45 | declare_array(floatp, ch_float*) 46 | declare_array(stringp, ch_str*) 47 | 48 | declare_array(voidp, void*) 49 | 50 | declare_ch_array_cmp(u8, u8) 51 | declare_ch_array_cmp(u16, u16) 52 | declare_ch_array_cmp(u32, u32) 53 | declare_ch_array_cmp(u64, u64) 54 | 55 | declare_ch_array_cmp(u8p, u8*) 56 | declare_ch_array_cmp(u16p, u16*) 57 | declare_ch_array_cmp(u32p, u32*) 58 | declare_ch_array_cmp(u64p, u64*) 59 | 60 | declare_ch_array_cmp(i8, i8) 61 | declare_ch_array_cmp(i16, i16) 62 | declare_ch_array_cmp(i32, i32) 63 | declare_ch_array_cmp(i64, i64) 64 | 65 | declare_ch_array_cmp(i8p, i8*) 66 | declare_ch_array_cmp(i16p, i16*) 67 | declare_ch_array_cmp(i32p, i32*) 68 | declare_ch_array_cmp(i64p, i64*) 69 | 70 | declare_ch_array_cmp(machine, ch_machine) 71 | declare_ch_array_cmp(word, ch_word) 72 | declare_ch_array_cmp(char, ch_char) 73 | declare_ch_array_cmp(ch_bool, ch_bool) 74 | declare_ch_array_cmp(float, ch_float) 75 | declare_ch_array_cmp(string, ch_str) 76 | declare_ch_array_cmp(cstr, ch_cstr) 77 | 78 | declare_ch_array_cmp(machinep, ch_machine*) 79 | declare_ch_array_cmp(wordp, ch_word*) 80 | declare_ch_array_cmp(charp, ch_char*) 81 | declare_ch_array_cmp(boolp, ch_bool*) 82 | declare_ch_array_cmp(floatp, ch_float*) 83 | declare_ch_array_cmp(stringp, ch_str*) 84 | declare_ch_array_cmp(cstrp, ch_cstr*) 85 | 86 | declare_ch_array_cmp(voidp, void*) 87 | 88 | #endif /* ARRAY_STD_H_ */ 89 | -------------------------------------------------------------------------------- /libs/libchaste/include/data_structs/array/array_typed_declare_template.h: -------------------------------------------------------------------------------- 1 | /* 2 | * array_typed.h 3 | * 4 | * Created on: Sep 12, 2013 5 | * Author: mgrosvenor 6 | */ 7 | 8 | #ifndef ARRAY_TYPED_TEMPLATE_DECLARE_H_ 9 | #define ARRAY_TYPED_TEMPLATE_DECLARE_H_ 10 | 11 | 12 | #include "../../types/types.h" 13 | #include "array.h" 14 | 15 | #define declare_array(NAME,TYPE)\ 16 | \ 17 | struct ch_array_##NAME;\ 18 | typedef struct ch_array_##NAME ch_array_##NAME##_t;\ 19 | \ 20 | struct ch_array_##NAME{\ 21 | ch_word size; /*Return the max number number of elements in the array list*/\ 22 | TYPE* first; /*Pointer to the fist valid entry list. Not valid if first == end*/\ 23 | TYPE* last; /*Pointer to the last valid element in the array. Not valid if last == end*/\ 24 | TYPE* end; /*Pointer to the one element beyond the end of the valid elements in array. Do not dereference! */\ 25 | \ 26 | void (*resize)(ch_array_##NAME##_t* this, ch_word new_size); /*Resize the array*/\ 27 | ch_word (*eq)(ch_array_##NAME##_t* this, ch_array_##NAME##_t* that); /*Check for equality*/\ 28 | \ 29 | TYPE* (*off)(ch_array_##NAME##_t* this, ch_word idx); /*Return the element at a given offset, with bounds checking*/\ 30 | TYPE* (*next)(ch_array_##NAME##_t* this, TYPE* ptr); /*Step forwards by one entry*/\ 31 | TYPE* (*prev)(ch_array_##NAME##_t* this, TYPE* ptr); /*Step backwards by one entry*/\ 32 | TYPE* (*forward)(ch_array_##NAME##_t* this, TYPE* ptr, ch_word amount); /*Step forwards by amount*/\ 33 | TYPE* (*back)(ch_array_##NAME##_t* this, TYPE* ptr, ch_word amount); /*Step backwards by amount*/\ 34 | \ 35 | TYPE* (*find)(ch_array_##NAME##_t* this, TYPE* begin, TYPE* end, TYPE value); /*find the given value using the comparator function*/\ 36 | void (*sort)(ch_array_##NAME##_t* this); /*sort into order given the comparator function*/\ 37 | \ 38 | void (*delete)(ch_array_##NAME##_t* this); /*Free the resources associated with this array, assumes that individual items have been freed*/\ 39 | \ 40 | TYPE* (*from_carray)(ch_array_##NAME##_t* this, TYPE* carray, ch_word count); /*Set at most count elements to the value in carray starting at offset in this array*/\ 41 | \ 42 | ch_array_t* _array;\ 43 | };\ 44 | \ 45 | ch_array_##NAME##_t* ch_array_##NAME##_new(ch_word size, ch_word (*cmp)(TYPE* lhs, TYPE* rhs) );\ 46 | 47 | 48 | #define declare_ch_array_cmp(NAME, TYPE) ch_word ch_array_cmp_##NAME(TYPE* lhs, TYPE* rhs); 49 | 50 | 51 | #define CH_ARRAY(NAME) ch_array_##NAME##_t 52 | #define CH_ARRAY_CMP(NAME) ch_array_cmp_##NAME 53 | #define CH_ARRAY_NEW(NAME, size, cmp) ch_array_##NAME##_new(size, cmp ) 54 | 55 | #endif //ARRAY_TYPED_TEMPLATE_DECLARE_H_ 56 | -------------------------------------------------------------------------------- /libs/libchaste/include/data_structs/array/array_typed_define_template.h: -------------------------------------------------------------------------------- 1 | /* 2 | * array_typed.c 3 | * 4 | * Created on: Sep 12, 2013 5 | * Author: mgrosvenor 6 | */ 7 | 8 | #ifndef ARRAY_TYPED_TEMPLATE_DEFINE_H_ 9 | #define ARRAY_TYPED_TEMPLATE_DEFINE_H_ 10 | 11 | 12 | #include "array.h" 13 | 14 | #include 15 | #include 16 | 17 | #define define_array(NAME,TYPE)\ 18 | \ 19 | static void _update_##NAME(ch_array_##NAME##_t* this)\ 20 | {\ 21 | this->size = this->_array->size;\ 22 | this->first = this->_array->first;\ 23 | this->last = this->_array->last;\ 24 | this->end = this->_array->end;\ 25 | }\ 26 | \ 27 | static void _resize_##NAME(ch_array_##NAME##_t* this, ch_word new_size) { array_resize(this->_array, new_size); _update_##NAME(this); }\ 28 | static ch_word _eq_##NAME(ch_array_##NAME##_t* this, ch_array_##NAME##_t* that) { ch_word result = array_eq(this->_array, that->_array); _update_##NAME(this); return result;}\ 29 | static TYPE* _off_##NAME(ch_array_##NAME##_t* this, ch_word idx) { TYPE* result = (TYPE*)array_off(this->_array, idx); _update_##NAME(this); return result; }\ 30 | static TYPE* _forward_##NAME(ch_array_##NAME##_t* this, TYPE* ptr, ch_word amount) { TYPE* result = (TYPE*)array_forward(this->_array, (void*)ptr, amount); _update_##NAME(this); return result; }\ 31 | static TYPE* _back_##NAME(ch_array_##NAME##_t* this, TYPE* ptr, ch_word amount) { TYPE* result = (TYPE*)array_back(this->_array, (void*)ptr, amount); _update_##NAME(this); return result; }\ 32 | static TYPE* _next_##NAME(ch_array_##NAME##_t* this, TYPE* ptr) { TYPE* result = (TYPE*)_forward_##NAME(this, ptr, 1); _update_##NAME(this); return result; }\ 33 | static TYPE* _prev_##NAME(ch_array_##NAME##_t* this, TYPE* ptr) { TYPE* result = (TYPE*)_back_##NAME(this, ptr, 1); _update_##NAME(this); return result; }\ 34 | static TYPE* _find_##NAME(ch_array_##NAME##_t* this, TYPE* begin, TYPE* end, TYPE value) { TYPE* result = (TYPE*) array_find(this->_array, (void*)begin, (void*)end, &value); _update_##NAME(this); return result; }\ 35 | static void _sort_##NAME(ch_array_##NAME##_t* this) { array_sort(this->_array); _update_##NAME(this); }\ 36 | static TYPE* _from_carray_##NAME(ch_array_##NAME##_t* this, TYPE* carray, ch_word count) { TYPE* result = array_from_carray(this->_array, (void*)carray, count); _update_##NAME(this); return result; }\ 37 | \ 38 | static void _delete_##NAME(ch_array_##NAME##_t* this)\ 39 | {\ 40 | if(this->_array){\ 41 | array_delete(this->_array);\ 42 | }\ 43 | \ 44 | free(this);\ 45 | }\ 46 | \ 47 | \ 48 | ch_array_##NAME##_t* ch_array_##NAME##_new(ch_word size, ch_word (*cmp)(TYPE* lhs, TYPE* rhs) )\ 49 | {\ 50 | \ 51 | ch_array_##NAME##_t* result = (ch_array_##NAME##_t*)calloc(1,sizeof(ch_array_##NAME##_t));\ 52 | if(!result){\ 53 | printf("Could not allocate memory for new array structure. Giving upn");\ 54 | return ((void *)0);\ 55 | }\ 56 | \ 57 | result->_array = ch_array_new(size, sizeof(TYPE), (cmp_void_f)cmp );\ 58 | \ 59 | \ 60 | /*We have memory to play with, now do all the other assignments*/\ 61 | result->resize = _resize_##NAME;\ 62 | result->off = _off_##NAME;\ 63 | result->next = _next_##NAME;\ 64 | result->prev = _prev_##NAME;\ 65 | result->forward = _forward_##NAME;\ 66 | result->back = _back_##NAME;\ 67 | result->from_carray = _from_carray_##NAME;\ 68 | result->delete = _delete_##NAME;\ 69 | \ 70 | /*Fail hard and early if the compare function is NULL*/\ 71 | if(cmp){\ 72 | result->eq = _eq_##NAME;\ 73 | result->find = _find_##NAME;\ 74 | result->sort = _sort_##NAME;\ 75 | }\ 76 | \ 77 | _update_##NAME(result);\ 78 | \ 79 | return result;\ 80 | } 81 | 82 | //Regular comparison function 83 | #define define_ch_array_cmp(NAME, TYPE) \ 84 | ch_word ch_array_cmp_##NAME(TYPE* lhs, TYPE* rhs)\ 85 | { \ 86 | return ( *lhs == *rhs ? 0 : *lhs < *rhs ? -1 : 1); \ 87 | } 88 | 89 | //Pointer comparison function 90 | #define define_ch_array_cmpp(NAME, TYPE) ch_word ch_array_cmp_##NAME##p(TYPE* lhs, TYPE* rhs)\ 91 | { \ 92 | return ch_array_cmp_##NAME(*lhs, *rhs);\ 93 | } 94 | 95 | #define define_ch_array_compare(NAME, TYPE) ch_word ch_array_cmp_##NAME(TYPE* lhs, TYPE* rhs) 96 | 97 | 98 | #endif //ARRAY_TYPED_TEMPLATE_DEFINE_H_ 99 | -------------------------------------------------------------------------------- /libs/libchaste/include/data_structs/circular_queue/circular_queue.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This is the implementation of a circular buffer based queue data structure. The data structure provides a fixed sized 3 | * queue with constant time push and pop operations (much like a linked listed) but with out runtime memory allocation 4 | * costs. The structure provides the ability to push items on to the queue, to "use" items from that queue and to release 5 | * items from the queue. All operations are O(c) time with the size of the queue. The structure also provides the ability to 6 | * push an array of times on to the queue which is O(n) with the size of the array. 7 | * 8 | * cbq.c 9 | * 10 | * Created on: July 22, 2015 11 | * Author: mgrosvenor 12 | */ 13 | 14 | 15 | #ifndef CBQ_H_ 16 | #define CBQ_H_ 17 | 18 | #include "../../types/types.h" 19 | #include "../array/array.h" 20 | 21 | struct ch_cbq; 22 | typedef struct ch_cbq ch_cbq_t; 23 | 24 | struct ch_cbq{ 25 | ch_word size; //Return the max number number of elements in the cbq list 26 | ch_word count; //Return the actual number of elements in the cbq 27 | ch_word in_use; //Return the number of elements that are currently in use. in_use <= count <= size. 28 | 29 | // Members prefixed with "_" are nominally "private" Don't touch my privates! 30 | ch_word _add_next_index; 31 | ch_word _release_index; 32 | ch_word _use_next_index; 33 | 34 | ch_array_t* _array; //Actual CBQ storage back end 35 | }; 36 | 37 | //Get access to an element at the front of the list. Return a pointer to the element. 38 | void* cbq_peek_front(ch_cbq_t* this); 39 | 40 | //Pull an element from the front of the cbq. Return 0 on success, -1 on failure. 41 | int cbq_pop_front(ch_cbq_t* this); 42 | 43 | //Mark an item as "used" an item from the front of the queue. This marks the item as "in use" and returns a pointer to it. 44 | //This is useful to use the queue for in-place storage while working on producing a result. 45 | void* cbq_use_front(ch_cbq_t* this); 46 | 47 | //Unmark the last item as used. This is useful if an operation failed on an item marked as used. 48 | void cbq_unuse_front(ch_cbq_t* this); 49 | 50 | // Put an element at the back of the circular buffer queue 51 | void* cbq_push_back(ch_cbq_t* this, void* value); 52 | 53 | //Push back count elements the C cbq to the back cbq-list 54 | void* cbq_push_back_carray(ch_cbq_t* this, void* ccbq, ch_word* count); 55 | 56 | //Free the resources associated with this cbq, assumes that individual items have been freed 57 | void cbq_delete(ch_cbq_t* this); 58 | 59 | //Create a new circualr buffer queue structure with size queue slots in it, each with element_size storage. 60 | ch_cbq_t* ch_cbq_new(ch_word size, ch_word element_size); 61 | 62 | #endif // CBQ_H_ 63 | -------------------------------------------------------------------------------- /libs/libchaste/include/data_structs/function_hash_map/function_hash_map.h: -------------------------------------------------------------------------------- 1 | /* 2 | * function_hash_map.h 3 | * 4 | * Created on: Sep 23, 2013 5 | * Author: mgrosvenor 6 | */ 7 | 8 | #ifndef FUNCTION_HASH_MAP_H_ 9 | #define FUNCTION_HASH_MAP_H_ 10 | 11 | #include "../../types/types.h" 12 | #include "../linked_list/linked_list.h" 13 | #include "../array/array.h" 14 | 15 | 16 | struct ch_function_hash_map_t; 17 | typedef struct ch_function_hash_map_t ch_function_hash_map; 18 | 19 | typedef struct { 20 | ch_llist_t* list; 21 | ch_word offset; 22 | ch_word key_size; 23 | ch_word key_int; //For keys less than or equal to 8bytes, just assign them 24 | void* key_ptr; //For keys that are longer, and not static, alloc memory and copy them here 25 | void* key_ptr_unsafe; //For keys that are in mapped memory for the life of the program, just keep the pointer here 26 | ch_word value; 27 | ch_word index; 28 | } ch_function_hash_map_node; 29 | 30 | typedef struct { 31 | //These state variables are private 32 | ch_function_hash_map_node* _node; 33 | ch_llist_it item; 34 | 35 | //This is public 36 | void* key; 37 | ch_word key_size; 38 | ch_word value; 39 | } ch_function_hash_map_it; 40 | 41 | 42 | struct ch_function_hash_map_t{ 43 | ch_word count; //Return the actual number of elements in the function_hash_map 44 | 45 | // Members prefixed with "_" are nominally "private" Don't touch my privates! 46 | ch_word (*_cmp)(void* lhs, void* rhs); // Comparator function for find and sort operations 47 | ch_array_t* _backing_array; 48 | ch_word _element_size; 49 | ch_word (*_func)(ch_word value, void* key, ch_word key_size, void* data, ch_word index); 50 | }; 51 | 52 | 53 | 54 | //Return the element at a given offset, with bounds checking 55 | ch_function_hash_map_it function_hash_map_off(ch_function_hash_map* this, ch_word idx); 56 | 57 | //Get the first entry 58 | ch_function_hash_map_it function_hash_map_first(ch_function_hash_map* this); 59 | //Get the last entry 60 | ch_function_hash_map_it function_hash_map_last(ch_function_hash_map* this); 61 | //Get the end 62 | ch_function_hash_map_it function_hash_map_end(ch_function_hash_map* this); 63 | 64 | //Step forwards by one entry 65 | void function_hash_map_next (ch_function_hash_map* this, ch_function_hash_map_it* it); 66 | //Step backwards by one entry 67 | void function_hash_map_prev(ch_function_hash_map* this, ch_function_hash_map_it*); 68 | //Step forwards by amount 69 | void function_hash_map_forward(ch_function_hash_map* this, ch_function_hash_map_it* it, ch_word amount); 70 | //Step backwards by amount 71 | void function_hash_map_back(ch_function_hash_map* this, ch_function_hash_map_it* it, ch_word amount); 72 | 73 | // Put an element into the table, 74 | ch_function_hash_map_it function_hash_map_push_unsafe_ptr(ch_function_hash_map* this, void* key, ch_word key_size, void* value); 75 | ch_function_hash_map_it function_hash_map_push(ch_function_hash_map* this, void* key, ch_word key_size, void* value); 76 | //Remove the given ptr 77 | ch_function_hash_map_it function_hash_map_remove(ch_function_hash_map* this, ch_function_hash_map_it* itr); 78 | 79 | //Free the resources associated with this function_hash_map, assumes that individual items have been freed 80 | void function_hash_map_delete(ch_function_hash_map* this); 81 | 82 | //Push back count elements the C function_hash_map to the back function_hash_map-list 83 | ch_function_hash_map_it function_hash_map_push_back_carray(ch_function_hash_map* this, const void* keys, ch_word* key_sizes, const void* carray, ch_word count); 84 | 85 | //Check for equality 86 | ch_word function_hash_map_eq(ch_function_hash_map* this, ch_function_hash_map* that); 87 | 88 | //Return the value associated with key using the comparator function 89 | ch_function_hash_map_it function_hash_map_get_first(ch_function_hash_map* this, void* key, ch_word key_size); 90 | 91 | //Get the next value with the given key 92 | ch_function_hash_map_it function_hash_map_get_next(ch_function_hash_map_it it); 93 | 94 | //Find the key of the given value 95 | ch_function_hash_map_it function_hash_map_find(ch_function_hash_map* this, ch_function_hash_map_it* begin, ch_function_hash_map_it* end, void* value); 96 | 97 | ch_function_hash_map* ch_function_hash_map_new( ch_word size, ch_word (*func)(ch_word value, void* key, ch_word key_size, void* data, ch_word index) ); 98 | 99 | #endif // FUNCTION_HASH_MAP_H_ 100 | 101 | -------------------------------------------------------------------------------- /libs/libchaste/include/data_structs/hash_map/hash_map.h: -------------------------------------------------------------------------------- 1 | /* 2 | * hash_map.h 3 | * 4 | * Created on: Sep 23, 2013 5 | * Author: mgrosvenor 6 | */ 7 | 8 | #ifndef HASH_MAP_H_ 9 | #define HASH_MAP_H_ 10 | 11 | #include "../../types/types.h" 12 | #include "../linked_list/linked_list.h" 13 | #include "../array/array.h" 14 | 15 | 16 | struct ch_hash_map_t; 17 | typedef struct ch_hash_map_t ch_hash_map; 18 | 19 | typedef struct { 20 | ch_llist_t* list; 21 | ch_word offset; 22 | ch_word key_size; 23 | ch_word key_int; //For keys less than or equal to 8bytes, just assign them 24 | void* key_ptr; //For keys that are longer, and not static, alloc memory and copy them here 25 | void* key_ptr_unsafe; //For keys that are in mapped memory for the life of the program, just keep the pointer here 26 | } ch_hash_map_node; 27 | 28 | typedef struct { 29 | //These state variables are private 30 | ch_hash_map_node* _node; 31 | ch_llist_it item; 32 | 33 | //This is public 34 | void* key; 35 | ch_word key_size; 36 | void* value; 37 | } ch_hash_map_it; 38 | 39 | 40 | struct ch_hash_map_t{ 41 | ch_word count; //Return the actual number of elements in the hash_map 42 | 43 | // Members prefixed with "_" are nominally "private" Don't touch my privates! 44 | cmp_void_f _cmp; // Comparator function for find and sort operations 45 | ch_array_t* _backing_array; 46 | ch_word _element_size; 47 | }; 48 | 49 | 50 | 51 | //Return the element at a given offset, with bounds checking 52 | ch_hash_map_it hash_map_off(ch_hash_map* this, ch_word idx); 53 | 54 | //Get the first entry 55 | ch_hash_map_it hash_map_first(ch_hash_map* this); 56 | //Get the last entry 57 | ch_hash_map_it hash_map_last(ch_hash_map* this); 58 | //Get the end 59 | ch_hash_map_it hash_map_end(ch_hash_map* this); 60 | 61 | //Step forwards by one entry 62 | void hash_map_next (ch_hash_map* this, ch_hash_map_it* it); 63 | //Step backwards by one entry 64 | void hash_map_prev(ch_hash_map* this, ch_hash_map_it*); 65 | //Step forwards by amount 66 | void hash_map_forward(ch_hash_map* this, ch_hash_map_it* it, ch_word amount); 67 | //Step backwards by amount 68 | void hash_map_back(ch_hash_map* this, ch_hash_map_it* it, ch_word amount); 69 | 70 | // Put an element into the table, 71 | ch_hash_map_it hash_map_push_unsafe_ptr(ch_hash_map* this, void* key, ch_word key_size, void* value); 72 | ch_hash_map_it hash_map_push(ch_hash_map* this, void* key, ch_word key_size, void* value); 73 | //Remove the given ptr 74 | ch_hash_map_it hash_map_remove(ch_hash_map* this, ch_hash_map_it* itr); 75 | 76 | //Free the resources associated with this hash_map, assumes that individual items have been freed 77 | void hash_map_delete(ch_hash_map* this); 78 | 79 | //Push back count elements the C hash_map to the back hash_map-list 80 | ch_hash_map_it hash_map_push_back_carray(ch_hash_map* this, const void* keys, ch_word* key_sizes, const void* carray, ch_word count); 81 | 82 | //Check for equality 83 | ch_word hash_map_eq(ch_hash_map* this, ch_hash_map* that); 84 | 85 | //Return the value associated with key using the comparator function 86 | ch_hash_map_it hash_map_get_first(ch_hash_map* this, void* key, ch_word key_size); 87 | 88 | //Get the next value with the given key 89 | ch_hash_map_it hash_map_get_next(ch_hash_map_it it); 90 | 91 | //Find the key of the given value 92 | ch_hash_map_it hash_map_find(ch_hash_map* this, ch_hash_map_it* begin, ch_hash_map_it* end, void* value); 93 | 94 | ch_hash_map* ch_hash_map_new( ch_word size, ch_word element_size, cmp_void_f cmp ); 95 | 96 | #endif // HASH_MAP_H_ 97 | 98 | -------------------------------------------------------------------------------- /libs/libchaste/include/data_structs/linked_list/linked_list.h: -------------------------------------------------------------------------------- 1 | /* 2 | * llist.h 3 | * 4 | * Created on: Sep 12, 2013 5 | * Author: mgrosvenor 6 | */ 7 | 8 | #ifndef LINKEDLIST_H_ 9 | #define LINKEDLIST_H_ 10 | 11 | #include "../../types/types.h" 12 | 13 | struct ch_llist_node; 14 | typedef struct ch_llist_node ch_llist_node_t; 15 | 16 | struct ch_llist_node { 17 | ch_llist_node_t* next; 18 | ch_llist_node_t* prev; 19 | }; 20 | 21 | struct ch_llist; 22 | typedef struct ch_llist ch_llist_t; 23 | 24 | struct ch_llist{ 25 | ch_word count; //Return the actual number of elements in the llist 26 | 27 | // Members prefixed with "_" are nominally "private" Don't touch my privates! 28 | cmp_void_f _cmp; // Comparator function for find and sort operations 29 | ch_llist_node_t* _first; 30 | ch_llist_node_t* _last; 31 | ch_word _element_size; 32 | }; 33 | 34 | typedef struct { 35 | //These state variables are private 36 | ch_llist_node_t* _node; 37 | 38 | //This is public 39 | void* value; 40 | } ch_llist_it; 41 | 42 | 43 | //Return the element at a given offset, with bounds checking [WARN: This is slow in general] 44 | ch_llist_it llist_off(ch_llist_t* this, ch_word idx); 45 | 46 | //Get the first entry 47 | ch_llist_it llist_first(ch_llist_t* this); 48 | //Get the last entry 49 | ch_llist_it llist_last(ch_llist_t* this); 50 | //Get the end 51 | ch_llist_it llist_end(ch_llist_t* this); 52 | 53 | //Step forwards by one entry 54 | void llist_next (ch_llist_t* this, ch_llist_it* it); 55 | //Step backwards by one entry 56 | void llist_prev(ch_llist_t* this, ch_llist_it*); 57 | //Step forwards by amount 58 | void llist_forward(ch_llist_t* this, ch_llist_it* it, ch_word amount); 59 | //Step backwards by amount 60 | void llist_back(ch_llist_t* this, ch_llist_it* it, ch_word amount); 61 | 62 | // Put an element at the front of the llist list values, 63 | ch_llist_it llist_push_front(ch_llist_t* this, const void* value); 64 | // Push an element off the front of the llist list values, 65 | ch_llist_it llist_pop_front(ch_llist_t* this); 66 | // Put an element at the back of the llist values 67 | ch_llist_it llist_push_back(ch_llist_t* this, const void* value); 68 | // Push an element at the back of the llist values 69 | ch_llist_it llist_pop_back(ch_llist_t* this); 70 | 71 | // Insert an element after the element given by ptr 72 | ch_llist_it* llist_insert_after(ch_llist_t* this, ch_llist_it* itr, const void* value); 73 | // Insert an element before the element giver by ptr 74 | ch_llist_it* llist_insert_before(ch_llist_t* this, ch_llist_it* it, const void* value); 75 | //Remove the given iterator 76 | ch_llist_it llist_remove_it(ch_llist_t* this, ch_llist_it* itr); 77 | //Remove all items with the given value 78 | ch_llist_it llist_remove_all(ch_llist_t* this, void* value); 79 | 80 | //Remove all elememts from the list 81 | void llist_pop_all(ch_llist_t* this); 82 | 83 | //Free the resources associated with this llist, assumes that individual items have been freed 84 | void llist_delete(ch_llist_t* this); 85 | 86 | //Push back count elements the C llist to the back llist-list 87 | ch_llist_it llist_push_back_carray(ch_llist_t* this, const void* carray, ch_word count); 88 | 89 | //Check for equality 90 | ch_word llist_eq(ch_llist_t* this, ch_llist_t* that); 91 | //find the given value using the comparator function 92 | ch_llist_it llist_find(ch_llist_t* this, ch_llist_it* begin, ch_llist_it* end, void* value); 93 | ch_llist_it llist_find_first(ch_llist_t* this, void* value); 94 | ch_llist_it llist_find_next(ch_llist_t* this, ch_llist_it* begin, void* value); 95 | ch_llist_it llist_insert_inorder(ch_llist_t* this, void* value); 96 | ch_llist_it llist_insert_carray_ordered(ch_llist_t* this, void* carray, ch_word count); 97 | //sort into order given the comparator function 98 | void llist_sort(ch_llist_t* this); 99 | 100 | ch_llist_t* ch_llist_new( ch_word element_size, cmp_void_f cmp ); 101 | ch_llist_t* ch_llist_init( ch_llist_t* this, ch_word element_size, cmp_void_f cmp ); 102 | 103 | #endif // LINKEDLIST_H_ 104 | -------------------------------------------------------------------------------- /libs/libchaste/include/data_structs/linked_list/linked_list_std.h: -------------------------------------------------------------------------------- 1 | /* 2 | * linked_list_std.h 3 | * 4 | * Created on: Sep 23, 2013 5 | * Author: mgrosvenor 6 | */ 7 | 8 | #ifndef LINKED_LIST_STD_H_ 9 | #define LINKED_LIST_STD_H_ 10 | 11 | #include "linked_list_typed_declare_template.h" 12 | 13 | declare_ch_llist(u8, u8) 14 | declare_ch_llist(u16, u16) 15 | declare_ch_llist(u32, u32) 16 | declare_ch_llist(u64, u64) 17 | 18 | declare_ch_llist(u8p, u8*) 19 | declare_ch_llist(u16p, u16*) 20 | declare_ch_llist(u32p, u32*) 21 | declare_ch_llist(u64p, u64*) 22 | 23 | declare_ch_llist(i8, i8) 24 | declare_ch_llist(i16, i16) 25 | declare_ch_llist(i32, i32) 26 | declare_ch_llist(i64, i64) 27 | 28 | declare_ch_llist(i8p, i8*) 29 | declare_ch_llist(i16p, i16*) 30 | declare_ch_llist(i32p, i32*) 31 | declare_ch_llist(i64p, i64*) 32 | 33 | declare_ch_llist(machine, ch_machine) 34 | declare_ch_llist(word, ch_word) 35 | declare_ch_llist(char, ch_char) 36 | declare_ch_llist(ch_bool, ch_bool) 37 | declare_ch_llist(float, ch_float) 38 | declare_ch_llist(string, ch_str) 39 | declare_ch_llist(cstr, ch_cstr) 40 | 41 | declare_ch_llist(machinep, ch_machine*) 42 | declare_ch_llist(wordp, ch_word*) 43 | declare_ch_llist(charp, ch_char*) 44 | declare_ch_llist(boolp, ch_bool*) 45 | declare_ch_llist(floatp, ch_float*) 46 | declare_ch_llist(stringp, ch_str*) 47 | declare_ch_llist(cstrp, ch_cstr*) 48 | 49 | declare_ch_llist(voidp, void*) 50 | 51 | declare_ch_llist_cmp(u8, u8); 52 | declare_ch_llist_cmp(u16, u16); 53 | declare_ch_llist_cmp(u32, u32); 54 | declare_ch_llist_cmp(u64, u64); 55 | 56 | declare_ch_llist_cmp(u8p, u8*); 57 | declare_ch_llist_cmp(u16p, u16*); 58 | declare_ch_llist_cmp(u32p, u32*); 59 | declare_ch_llist_cmp(u64p, u64*); 60 | 61 | declare_ch_llist_cmp(i8, i8); 62 | declare_ch_llist_cmp(i16, i16); 63 | declare_ch_llist_cmp(i32, i32); 64 | declare_ch_llist_cmp(i64, i64); 65 | 66 | declare_ch_llist_cmp(i8p, i8*); 67 | declare_ch_llist_cmp(i16p, i16*); 68 | declare_ch_llist_cmp(i32p, i32*); 69 | declare_ch_llist_cmp(i64p, i64*); 70 | 71 | declare_ch_llist_cmp(machine, ch_machine); 72 | declare_ch_llist_cmp(word, ch_word); 73 | declare_ch_llist_cmp(char, ch_char); 74 | declare_ch_llist_cmp(ch_bool, ch_bool); 75 | declare_ch_llist_cmp(float, ch_float); 76 | declare_ch_llist_cmp(string, ch_str); 77 | declare_ch_llist_cmp(cstr, ch_cstr); 78 | 79 | declare_ch_llist_cmp(machinep, ch_machine*); 80 | declare_ch_llist_cmp(wordp, ch_word*); 81 | declare_ch_llist_cmp(charp, ch_char*); 82 | declare_ch_llist_cmp(boolp, ch_bool*); 83 | declare_ch_llist_cmp(floatp, ch_float*); 84 | declare_ch_llist_cmp(stringp, ch_str*); 85 | declare_ch_llist_cmp(cstrp, ch_cstr*); 86 | 87 | declare_ch_llist_cmp(voidp, void*); 88 | 89 | 90 | declare_ch_llist(UINT64, u64) 91 | declare_ch_llist_cmp(UINT64, u64); 92 | 93 | #endif /* LINKED_LIST_STD_H_ */ 94 | -------------------------------------------------------------------------------- /libs/libchaste/include/data_structs/linked_list/linked_list_typed_declare_template.h: -------------------------------------------------------------------------------- 1 | #ifndef LINKED_LIST_TYPED_TEMPLATE_DECLARE_H_ 2 | #define LINKED_LIST_TYPED_TEMPLATE_DECLARE_H_ 3 | 4 | 5 | #include "../../types/types.h" 6 | #include "linked_list.h" 7 | 8 | #define declare_ch_llist(NAME,TYPE) \ 9 | \ 10 | struct ch_llist_##NAME;\ 11 | struct ch_llist_##NAME##_it_t;\ 12 | typedef struct ch_llist_##NAME ch_llist_##NAME##_t;\ 13 | typedef struct ch_llist_##NAME##_it_t ch_llist_##NAME##_it;\ 14 | \ 15 | struct ch_llist_##NAME##_it_t{ \ 16 | ch_llist_node_t* _node; \ 17 | TYPE* value;\ 18 | };\ 19 | \ 20 | \ 21 | struct ch_llist_##NAME{\ 22 | ch_word count; /*Return the actual number of elements in the llist*/ \ 23 | \ 24 | ch_llist_##NAME##_it (*off)(ch_llist_##NAME##_t* this, ch_word idx); /* Return the element at a given offset, with bounds checking [WARN: This is slow in general] */\ 25 | \ 26 | ch_llist_##NAME##_it (*first)(ch_llist_##NAME##_t* this); /*Get the first entry*/\ 27 | ch_llist_##NAME##_it (*last)(ch_llist_##NAME##_t* this); /*Get the last entry*/\ 28 | ch_llist_##NAME##_it (*end)(ch_llist_##NAME##_t* this); /*Get the end*/\ 29 | \ 30 | void (*next)(ch_llist_##NAME##_t* this, ch_llist_##NAME##_it* it); /*Step forwards by one entry*/\ 31 | void (*prev)(ch_llist_##NAME##_t* this, ch_llist_##NAME##_it*); /*Step backwards by one entry*/\ 32 | void (*forward)(ch_llist_##NAME##_t* this, ch_llist_##NAME##_it* it, ch_word amount); /*Step forwards by amount*/\ 33 | void (*back)(ch_llist_##NAME##_t* this, ch_llist_##NAME##_it* it, ch_word amount); /*Step backwards by amount*/\ 34 | \ 35 | ch_llist_##NAME##_it (*push_front)(ch_llist_##NAME##_t* this, TYPE value); /* Put an element at the front of the linked list values,*/\ 36 | ch_llist_##NAME##_it (*pop_front)(ch_llist_##NAME##_t* this);/* Push an element off the front of the linked list values,*/\ 37 | ch_llist_##NAME##_it (*push_back)(ch_llist_##NAME##_t* this, TYPE value); /* Put an element at the back of the linked list values*/\ 38 | ch_llist_##NAME##_it (*pop_back)(ch_llist_##NAME##_t* this); /* Push an element at the back of the linked list values*/\ 39 | \ 40 | ch_llist_##NAME##_it* (*insert_after)(ch_llist_##NAME##_t* this, ch_llist_##NAME##_it* itr, TYPE value); /* Insert an element after the element given by ptr*/\ 41 | ch_llist_##NAME##_it* (*insert_before)(ch_llist_##NAME##_t* this, ch_llist_##NAME##_it* it, TYPE value); /* Insert an element before the element giver by ptr*/\ 42 | ch_llist_##NAME##_it (*remove_it)(ch_llist_##NAME##_t* this, ch_llist_##NAME##_it* itr); /*Remove the given ptr*/\ 43 | ch_llist_##NAME##_it (*remove_all)(ch_llist_##NAME##_t* this, TYPE value); /*Remove the given ptr*/\ 44 | \ 45 | void (*delete)(ch_llist_##NAME##_t* this); /*Free the resources associated with this llist, assumes that individual items have been freed*/\ 46 | \ 47 | ch_llist_##NAME##_it (*push_back_carray)(ch_llist_##NAME##_t* this, const TYPE* carray, ch_word count); /*Push back count elements the C llist to the back llist-list*/\ 48 | \ 49 | ch_word (*eq)(ch_llist_##NAME##_t* this, ch_llist_##NAME##_t* that); /*Check for equality*/\ 50 | ch_llist_##NAME##_it (*find)(ch_llist_##NAME##_t* this, ch_llist_##NAME##_it* begin, ch_llist_##NAME##_it* end, TYPE value); /*find the given value using the comparator function*/\ 51 | ch_llist_##NAME##_it (*find_first)(ch_llist_##NAME##_t* this, TYPE value); /*find the given value using the comparator function*/\ 52 | ch_llist_##NAME##_it (*find_next)(ch_llist_##NAME##_t* this, ch_llist_##NAME##_it* begin, TYPE value); /*find the given value using the comparator function*/\ 53 | ch_llist_##NAME##_it (*insert_inorder)(ch_llist_##NAME##_t* this, TYPE value); /* Insert an element before the element giver by ptr*/\ 54 | ch_llist_##NAME##_it (*insert_carray_ordered)(ch_llist_##NAME##_t* this, const TYPE* carray, ch_word count); /*Insert count element in order*/\ 55 | void (*sort)(ch_llist_##NAME##_t* this); /*sort into order given the comparator function*/\ 56 | \ 57 | /* Members prefixed with "_" are nominally "private" Don't touch my privates!*/\ 58 | ch_llist_t* _llist; /*Actual llist storage*/\ 59 | \ 60 | };\ 61 | \ 62 | \ 63 | ch_llist_##NAME##_t* ch_llist_##NAME##_new(ch_word(*cmp)(TYPE* lhs, TYPE* rhs) ); 64 | 65 | 66 | #define declare_ch_llist_cmp(NAME, TYPE) ch_word ch_llist_cmp_##NAME(TYPE* lhs, TYPE* rhs) 67 | 68 | 69 | //********************************************************************************************************************** 70 | //Shortcuts to make things more accessible 71 | #define CH_LIST(NAME) ch_llist_##NAME##_t 72 | #define CH_LIST_IT(NAME) ch_llist_##NAME##_it 73 | #define CH_LIST_NEW(NAME, cmp) ch_llist_##NAME##_new(cmp ) 74 | #define CH_LIST_CMP(NAME) ch_llist_cmp_##NAME 75 | #define CH_LIST_FOREACH(TYPE_NAME, LIST_NAME,IT_NAME) \ 76 | for(CH_LIST_IT(TYPE_NAME) IT_NAME = LIST_NAME->first(LIST_NAME); IT_NAME.value; LIST_NAME->next(LIST_NAME,&IT_NAME)) 77 | 78 | #endif /*LINKED_LIST_TYPED_TEMPLATE_DECLARE_H_*/ 79 | 80 | -------------------------------------------------------------------------------- /libs/libchaste/include/data_structs/vector/vector.h: -------------------------------------------------------------------------------- 1 | /* 2 | * vector.h 3 | * 4 | * Created on: Sep 12, 2013 5 | * Author: mgrosvenor 6 | */ 7 | 8 | #ifndef VECTOR_H_ 9 | #define VECTOR_H_ 10 | 11 | #include "../../types/types.h" 12 | #include "../array/array.h" 13 | 14 | struct ch_vector; 15 | typedef struct ch_vector ch_vector_t; 16 | 17 | 18 | struct ch_vector{ 19 | ch_word size; //Return the max number number of elements in the vector list 20 | ch_word count; //Return the actual number of elements in the vector 21 | 22 | void* first; //Pointer to the fist valid entry list. Not valid if first == end 23 | void* last; //Pointer to the last valid element in the list. Not valid if last == end 24 | void* end; //Pointer to the one element beyond the end of the valid elements in list. Do not dereference! 25 | 26 | // Members prefixed with "_" are nominally "private" Don't touch my privates! 27 | cmp_void_f _cmp; // Comparator function for find and sort operations 28 | ch_array_t* _array; //Actual vector storage 29 | ch_word _array_count; //NUmber of elements currently in the vector 30 | }; 31 | 32 | 33 | //Resize the vector 34 | void vector_resize(ch_vector_t* this, ch_word new_size); 35 | 36 | //Return the element at a given offset, with bounds checking 37 | void* vector_off(ch_vector_t* this, ch_word idx); 38 | 39 | //Step forwards by one entry 40 | void* vector_next (ch_vector_t* this, void* ptr); 41 | //Step backwards by one entry 42 | void* vector_prev(ch_vector_t* this, void* ptr); 43 | //Step forwards by amount 44 | void* vector_forward(ch_vector_t* this, void* ptr, ch_word amount); 45 | //Step backwards by amount 46 | void* vector_back(ch_vector_t* this, void* ptr, ch_word amount); 47 | 48 | // Put an element at the front of the vector list values, [WARN: In general this is very expensive for an vector] 49 | void* vector_push_front(ch_vector_t* this, void* value); 50 | // Push an element off the front of the vector list values, [WARN: In general this is very expensive for an vector] 51 | void vector_pop_front(ch_vector_t* this); 52 | // Put an element at the back of the vector values 53 | void* vector_push_back(ch_vector_t* this, void* value); 54 | // Push an element at the back of the vector values 55 | void vector_pop_back(ch_vector_t* this); 56 | 57 | // Insert an element after the element given by ptr 58 | void* vector_insert_after(ch_vector_t* this, void* ptr, void* value); 59 | // Insert an element before the element giver by ptr [WARN: In general this is very expensive for an vector] 60 | void* vector_insert_before(ch_vector_t* this, void* ptr, void* value); 61 | //Remove the given ptr [WARN: In general this is very expensive] 62 | void* vector_remove(ch_vector_t* this, void* ptr); 63 | //Pop everything out of the vector 64 | void vector_clear(ch_vector_t* this); 65 | 66 | //Free the resources associated with this vector, assumes that individual items have been freed 67 | void vector_delete(ch_vector_t* this); 68 | 69 | //Push back count elements the C vector to the back vector-list 70 | void* vector_push_back_carray(ch_vector_t* this, void* cvector, ch_word count); 71 | 72 | //Check for equality 73 | ch_word vector_eq(ch_vector_t* this, ch_vector_t* that); 74 | //find the given value using the comparator function 75 | void* vector_find(ch_vector_t* this, void* begin, void* end, void* value); 76 | //return the index of the value 77 | int vector_get_idx(ch_vector_t* this, void* value); 78 | //sort into order given the comparator function 79 | void vector_sort(ch_vector_t* this); 80 | 81 | ch_vector_t* ch_vector_new(ch_word size, ch_word element_size, cmp_void_f cmp ); 82 | 83 | #endif // VECTOR_H_ 84 | -------------------------------------------------------------------------------- /libs/libchaste/include/data_structs/vector/vector_std.h: -------------------------------------------------------------------------------- 1 | // CamIO 2: vector_std_int.h 2 | // Copyright (C) 2013: Matthew P. Grosvenor (matthew.grosvenor@cl.cam.ac.uk) 3 | // Licensed under BSD 3 Clause, please see LICENSE for more details. 4 | 5 | #ifndef VECTOR_STD_INT_H_ 6 | #define VECTOR_STD_INT_H_ 7 | 8 | #include 9 | 10 | #include "../../types/types.h" 11 | #include "vector_typed_declare_template.h" 12 | 13 | declare_ch_vector(u8, u8) 14 | declare_ch_vector(u16, u16) 15 | declare_ch_vector(u32, u32) 16 | declare_ch_vector(u64, u64) 17 | 18 | declare_ch_vector(u8p, u8*) 19 | declare_ch_vector(u16p, u16*) 20 | declare_ch_vector(u32p, u32*) 21 | declare_ch_vector(u64p, u64*) 22 | 23 | declare_ch_vector(i8, i8) 24 | declare_ch_vector(i16, i16) 25 | declare_ch_vector(i32, i32) 26 | declare_ch_vector(i64, i64) 27 | 28 | declare_ch_vector(i8p, i8*) 29 | declare_ch_vector(i16p, i16*) 30 | declare_ch_vector(i32p, i32*) 31 | declare_ch_vector(i64p, i64*) 32 | 33 | declare_ch_vector(machine, ch_machine) 34 | declare_ch_vector(word, ch_word) 35 | declare_ch_vector(char, ch_char) 36 | declare_ch_vector(ch_bool, ch_bool) 37 | declare_ch_vector(float, ch_float) 38 | declare_ch_vector(string, ch_str) 39 | declare_ch_vector(cstr, ch_cstr) 40 | 41 | declare_ch_vector(machinep, ch_machine*) 42 | declare_ch_vector(wordp, ch_word*) 43 | declare_ch_vector(charp, ch_char*) 44 | declare_ch_vector(boolp, ch_bool*) 45 | declare_ch_vector(floatp, ch_float*) 46 | declare_ch_vector(stringp, ch_str*) 47 | declare_ch_vector(cstrp, ch_cstr*) 48 | 49 | declare_ch_vector(voidp, void*) 50 | 51 | declare_ch_vector_cmp(u8, u8) 52 | declare_ch_vector_cmp(u16, u16) 53 | declare_ch_vector_cmp(u32, u32) 54 | declare_ch_vector_cmp(u64, u64) 55 | 56 | declare_ch_vector_cmp(u8p, u8*) 57 | declare_ch_vector_cmp(u16p, u16*) 58 | declare_ch_vector_cmp(u32p, u32*) 59 | declare_ch_vector_cmp(u64p, u64*) 60 | 61 | declare_ch_vector_cmp(i8, i8) 62 | declare_ch_vector_cmp(i16, i16) 63 | declare_ch_vector_cmp(i32, i32) 64 | declare_ch_vector_cmp(i64, i64) 65 | 66 | declare_ch_vector_cmp(i8p, i8*) 67 | declare_ch_vector_cmp(i16p, i16*) 68 | declare_ch_vector_cmp(i32p, i32*) 69 | declare_ch_vector_cmp(i64p, i64*) 70 | 71 | declare_ch_vector_cmp(machine, ch_machine) 72 | declare_ch_vector_cmp(word, ch_word) 73 | declare_ch_vector_cmp(char, ch_char) 74 | declare_ch_vector_cmp(ch_bool, ch_bool) 75 | declare_ch_vector_cmp(float, ch_float) 76 | declare_ch_vector_cmp(string, ch_str) 77 | declare_ch_vector_cmp(cstr, ch_cstr) 78 | 79 | declare_ch_vector_cmp(machinep, ch_machine*) 80 | declare_ch_vector_cmp(wordp, ch_word*) 81 | declare_ch_vector_cmp(charp, ch_char*) 82 | declare_ch_vector_cmp(boolp, ch_bool*) 83 | declare_ch_vector_cmp(floatp, ch_float*) 84 | declare_ch_vector_cmp(stringp, ch_str*) 85 | declare_ch_vector_cmp(cstrp, ch_cstr*) 86 | 87 | declare_ch_vector_cmp(voidp, void*) 88 | 89 | 90 | 91 | 92 | #endif /* ARRAY_STD_INT_H_ */ 93 | -------------------------------------------------------------------------------- /libs/libchaste/include/data_structs/vector/vector_typed_declare_template.h: -------------------------------------------------------------------------------- 1 | #ifndef VECTOR_TYPED_TEMPLATE_DECLARE_H_ 2 | #define VECTOR_TYPED_TEMPLATE_DECLARE_H_ 3 | 4 | 5 | #include "../../types/types.h" 6 | #include "vector.h" 7 | 8 | 9 | #define declare_ch_vector(NAME,TYPE) \ 10 | \ 11 | struct ch_vector_##NAME;\ 12 | typedef struct ch_vector_##NAME ch_vector_##NAME##_t;\ 13 | \ 14 | \ 15 | struct ch_vector_##NAME{\ 16 | ch_word size; /*Return the max number number of elements in the vector list*/\ 17 | ch_word count; /*Return the actual number of elements in the vector*/\ 18 | TYPE* first; /*Pointer to the fist valid entry list. Not valid if first == end*/\ 19 | TYPE* last; /*Pointer to the last valid element in the list. Not valid if last == end*/\ 20 | TYPE* end; /*Pointer to the one element beyond the end of the valid elements in list. Do not dereference! */\ 21 | \ 22 | \ 23 | void (*resize)(ch_vector_##NAME##_t* this, ch_word new_size); /*Resize the vector*/\ 24 | TYPE* (*off)(ch_vector_##NAME##_t* this, ch_word idx); /*Return the element at a given offset, with bounds checking*/\ 25 | \ 26 | TYPE* (*next)(ch_vector_##NAME##_t* this, TYPE* ptr); /*Step forwards by one entry*/\ 27 | TYPE* (*prev)(ch_vector_##NAME##_t* this, TYPE* ptr); /*Step backwards by one entry*/\ 28 | TYPE* (*forward)(ch_vector_##NAME##_t* this, TYPE* ptr, ch_word amount); /*Step forwards by amount*/\ 29 | TYPE* (*back)(ch_vector_##NAME##_t* this, TYPE* ptr, ch_word amount); /*Step backwards by amount*/\ 30 | \ 31 | TYPE* (*push_front)(ch_vector_##NAME##_t* this, TYPE value); /* Put an element at the front of the vector list values, [WARN: In general this is very expensive for a vector] */\ 32 | void (*pop_front)(ch_vector_##NAME##_t* this); /* Push an element off the front of the vector list values, [WARN: In general this is very expensive for a vector] */\ 33 | TYPE* (*push_back)(ch_vector_##NAME##_t* this, TYPE value); /* Put an element at the back of the vector values*/\ 34 | void (*pop_back)(ch_vector_##NAME##_t* this); /* Push an element at the back of the vector values*/\ 35 | void (*clear)(ch_vector_##NAME##_t* this); /*Pop everything out of the vector*/\ 36 | \ 37 | TYPE* (*insert_after)(ch_vector_##NAME##_t* this, TYPE* ptr, TYPE value); /* Insert an element after the element given by ptr*/\ 38 | TYPE* (*insert_before)(ch_vector_##NAME##_t* this, TYPE* ptr, TYPE value); /* Insert an element before the element giver by ptr [WARN: In general this is very expensive for a vector] */\ 39 | TYPE* (*remove)(ch_vector_##NAME##_t* this, TYPE* ptr); /*Remove the given ptr [WARN: In general this is very expensive] */\ 40 | void (*delete)(ch_vector_##NAME##_t* this); /*Free the resources associated with this vector, assumes that individual items have been freed*/\ 41 | \ 42 | TYPE* (*find)(ch_vector_##NAME##_t* this, TYPE* begin, TYPE* end, TYPE value); /*find the given value using the comparator function*/\ 43 | int (*get_idx)(ch_vector_##NAME##_t* this, TYPE* value); /*Convert the iterator into an index for use with off() above*/\ 44 | void (*sort)(ch_vector_##NAME##_t* this); /*sort into order given the comparator function*/\ 45 | ch_word (*eq)(ch_vector_##NAME##_t* this, ch_vector_##NAME##_t* that); /*Check for equality*/\ 46 | \ 47 | TYPE* (*push_back_carray)(ch_vector_##NAME##_t* this, TYPE* carray, ch_word count); /*Push back count elements the C vector to the back vector-list*/\ 48 | \ 49 | /* Members prefixed with "_" are "private" Don't touch my privates!*/\ 50 | ch_vector_t* _vector; /*Actual vector storage*/\ 51 | \ 52 | };\ 53 | \ 54 | \ 55 | ch_vector_##NAME##_t* ch_vector_##NAME##_new(ch_word size, ch_word(*cmp)(TYPE* lhs, TYPE* rhs) ); 56 | 57 | #define declare_ch_vector_cmp(NAME, TYPE) ch_word ch_vector_cmp_##NAME(TYPE* lhs, TYPE* rhs); 58 | 59 | #define CH_VECTOR(NAME) ch_vector_##NAME##_t 60 | #define CH_VECTOR_NEW(NAME, size, cmp) ch_vector_##NAME##_new(size, cmp ) 61 | #define CH_VECTOR_CMP(NAME) ch_vector_cmp_##NAME 62 | 63 | #endif //VECTOR_TYPED_TEMPLATE_DECLARE_H_ 64 | 65 | -------------------------------------------------------------------------------- /libs/libchaste/include/data_structs/vector/vector_typed_define_template.h: -------------------------------------------------------------------------------- 1 | #ifndef VECTOR_TYPED_TEMPLATE_DEFINE_H_ 2 | #define VECTOR_TYPED_TEMPLATE_DEFINE_H_ 3 | 4 | #include "vector.h" 5 | #include "vector_std.h" 6 | 7 | #include "../../utils/util.h" 8 | #include "../../types/types.h" 9 | 10 | #include 11 | #include 12 | #include 13 | 14 | #define define_ch_vector(NAME,TYPE)\ 15 | \ 16 | static void _update_##NAME(ch_vector_##NAME##_t* this)\ 17 | {\ 18 | this->size = this->_vector->size;\ 19 | this->first = this->_vector->first;\ 20 | this->last = this->_vector->last;\ 21 | this->end = this->_vector->end;\ 22 | this->count = this->_vector->count;\ 23 | }\ 24 | \ 25 | static void _resize_##NAME(ch_vector_##NAME##_t* this, ch_word new_size) { vector_resize(this->_vector, new_size); _update_##NAME(this); }\ 26 | static ch_word _eq_##NAME(ch_vector_##NAME##_t* this, ch_vector_##NAME##_t* that) { ch_word result = vector_eq(this->_vector, that->_vector); _update_##NAME(this); return result;}\ 27 | static TYPE* _off_##NAME(ch_vector_##NAME##_t* this, ch_word idx) { TYPE* result = (TYPE*)vector_off(this->_vector, idx); _update_##NAME(this); return result; }\ 28 | static TYPE* _forward_##NAME(ch_vector_##NAME##_t* this, TYPE* ptr, ch_word amount) { TYPE* result = (TYPE*)vector_forward(this->_vector, (void*)ptr, amount); _update_##NAME(this); return result; }\ 29 | static TYPE* _back_##NAME(ch_vector_##NAME##_t* this, TYPE* ptr, ch_word amount) { TYPE* result = (TYPE*)vector_back(this->_vector, (void*)ptr, amount); _update_##NAME(this); return result; }\ 30 | static TYPE* _next_##NAME(ch_vector_##NAME##_t* this, TYPE* ptr) { TYPE* result = (TYPE*)_forward_##NAME(this, ptr, 1); _update_##NAME(this); return result; }\ 31 | static TYPE* _prev_##NAME(ch_vector_##NAME##_t* this, TYPE* ptr) { TYPE* result = (TYPE*)_back_##NAME(this, ptr, 1); _update_##NAME(this); return result; }\ 32 | static TYPE* _find_##NAME(ch_vector_##NAME##_t* this, TYPE* begin, TYPE* end, TYPE value) { TYPE* result = (TYPE*) vector_find(this->_vector, (void*)begin, (void*)end, &value); _update_##NAME(this); return result; }\ 33 | static int _get_idx_##NAME(ch_vector_##NAME##_t* this, TYPE* value) { return vector_get_idx(this->_vector, value); }\ 34 | static void _sort_##NAME(ch_vector_##NAME##_t* this) { vector_sort(this->_vector); _update_##NAME(this); }\ 35 | static TYPE* _push_front_##NAME(ch_vector_##NAME##_t* this, TYPE value) { TYPE* result = (TYPE*) vector_push_front(this->_vector, &value); _update_##NAME(this); return result; }\ 36 | static TYPE* _push_back_##NAME(ch_vector_##NAME##_t* this, TYPE value) { TYPE* result = (TYPE*) vector_push_back(this->_vector, &value); _update_##NAME(this); return result; }\ 37 | static TYPE* _insert_after_##NAME(ch_vector_##NAME##_t* this, TYPE* ptr, TYPE value) { TYPE* result = (TYPE*) vector_insert_after(this->_vector, ptr, &value); _update_##NAME(this); return result; }\ 38 | static TYPE* _insert_before_##NAME(ch_vector_##NAME##_t* this, TYPE* ptr, TYPE value) { TYPE* result = (TYPE*) vector_insert_before(this->_vector, ptr, &value); _update_##NAME(this); return result; }\ 39 | static TYPE* _remove_##NAME(ch_vector_##NAME##_t* this, TYPE* ptr) { TYPE* result = (TYPE*) vector_remove(this->_vector, ptr); _update_##NAME(this); return result; }\ 40 | static void _pop_front_##NAME(ch_vector_##NAME##_t* this) { vector_pop_front(this->_vector); _update_##NAME(this); }\ 41 | static void _pop_back_##NAME(ch_vector_##NAME##_t* this) { vector_pop_back(this->_vector); _update_##NAME(this); }\ 42 | static void _clear_##NAME(ch_vector_##NAME##_t* this) { vector_clear(this->_vector); _update_##NAME(this); }\ 43 | static TYPE* _push_back_carray_##NAME(ch_vector_##NAME##_t* this, TYPE* carray, ch_word count) { TYPE* result = vector_push_back_carray(this->_vector, (void*)carray, count); _update_##NAME(this); return result; }\ 44 | \ 45 | static void _delete_##NAME(ch_vector_##NAME##_t* this)\ 46 | {\ 47 | if(this->_vector){\ 48 | vector_delete(this->_vector);\ 49 | }\ 50 | \ 51 | free(this);\ 52 | }\ 53 | \ 54 | \ 55 | ch_vector_##NAME##_t* ch_vector_##NAME##_new(ch_word size, ch_word (*cmp)(TYPE* lhs, TYPE* rhs) )\ 56 | {\ 57 | \ 58 | ch_vector_##NAME##_t* result = (ch_vector_##NAME##_t*)calloc(1,sizeof(ch_vector_##NAME##_t));\ 59 | if(!result){\ 60 | printf("Could not allocate memory for new vector structure. Giving upn");\ 61 | return ((void *)0);\ 62 | }\ 63 | \ 64 | result->_vector = ch_vector_new(size, sizeof(TYPE), (cmp_void_f)cmp );\ 65 | \ 66 | \ 67 | /*We have memory to play with, now do all the other assignments*/\ 68 | result->resize = _resize_##NAME;\ 69 | result->eq = _eq_##NAME;\ 70 | result->off = _off_##NAME;\ 71 | result->get_idx = _get_idx_##NAME;\ 72 | result->next = _next_##NAME;\ 73 | result->prev = _prev_##NAME;\ 74 | result->forward = _forward_##NAME;\ 75 | result->back = _back_##NAME;\ 76 | result->find = _find_##NAME;\ 77 | result->sort = _sort_##NAME;\ 78 | result->push_front = _push_front_##NAME;\ 79 | result->pop_front = _pop_front_##NAME;\ 80 | result->push_back = _push_back_##NAME;\ 81 | result->pop_back = _pop_back_##NAME;\ 82 | result->insert_after = _insert_after_##NAME;\ 83 | result->insert_before = _insert_before_##NAME;\ 84 | result->remove = _remove_##NAME;\ 85 | result->push_back_carray = _push_back_carray_##NAME;\ 86 | result->clear = _clear_##NAME;\ 87 | result->delete = _delete_##NAME;\ 88 | \ 89 | \ 90 | \ 91 | /*Fail hard and early if the compare function is NULL*/\ 92 | if(cmp){\ 93 | result->eq = _eq_##NAME;\ 94 | result->find = _find_##NAME;\ 95 | result->sort = _sort_##NAME;\ 96 | }\ 97 | \ 98 | _update_##NAME(result);\ 99 | \ 100 | return result;\ 101 | } 102 | 103 | //Regular comparison function 104 | #define define_ch_vector_cmp(NAME, TYPE) \ 105 | ch_word ch_vector_cmp_##NAME(TYPE* lhs, TYPE* rhs)\ 106 | { \ 107 | return ( *lhs == *rhs ? 0 : *lhs < *rhs ? -1 : 1); \ 108 | } 109 | 110 | //Pointer comparison function 111 | #define define_ch_vector_cmpp(NAME, TYPE) ch_word ch_vector_cmp_##NAME##p(TYPE* lhs, TYPE* rhs)\ 112 | { \ 113 | return ch_vector_cmp_##NAME(*lhs, *rhs);\ 114 | } 115 | 116 | #define define_ch_vector_compare(NAME, TYPE) ch_word ch_vector_cmp_##NAME(TYPE* lhs, TYPE* rhs) 117 | 118 | #endif //VECTOR_TYPED_TEMPLATE_DEFINE_H_ 119 | -------------------------------------------------------------------------------- /libs/libchaste/include/hash_functions/spooky/spooky_hash.h: -------------------------------------------------------------------------------- 1 | /* 2 | * spooky_hash.h 3 | * 4 | * Created on: Sep 23, 2013 5 | * Author: mgrosvenor 6 | */ 7 | 8 | #ifndef SPOOKY_HASH_H_ 9 | #define SPOOKY_HASH_H_ 10 | 11 | /* 12 | * This is a port of Bob Jenkins Spooky Hash from C++ to C. Port by M. P. Grosvenor. 13 | * http://burtleburtle.net/bob/hash/spooky.html 14 | */ 15 | 16 | // SpookyHash: a 128-bit noncryptographic hash function 17 | // By Bob Jenkins, public domain 18 | // Oct 31 2010: alpha, framework + SpookyHash::Mix appears right 19 | // Oct 31 2011: alpha again, Mix only good to 2^^69 but rest appears right 20 | // Dec 31 2011: beta, improved Mix, tested it for 2-bit deltas 21 | // Feb 2 2012: production, same bits as beta 22 | // Feb 5 2012: adjusted definitions of uint* to be more portable 23 | // Mar 30 2012: 3 bytes/cycle, not 4. Alpha was 4 but wasn't thorough enough. 24 | // August 5 2012: SpookyV2 (different results) 25 | // 26 | // Up to 3 bytes/cycle for long messages. Reasonably fast for short messages. 27 | // All 1 or 2 bit deltas achieve avalanche within 1% bias per output bit. 28 | // 29 | // This was developed for and tested on 64-bit x86-compatible processors. 30 | // It assumes the processor is little-endian. There is a macro 31 | // controlling whether unaligned reads are allowed (by default they are). 32 | // This should be an equally good hash on big-endian machines, but it will 33 | // compute different results on them than on little-endian machines. 34 | // 35 | // Google's CityHash has similar specs to SpookyHash, and CityHash is faster 36 | // on new Intel boxes. MD4 and MD5 also have similar specs, but they are orders 37 | // of magnitude slower. CRCs are two or more times slower, but unlike 38 | // SpookyHash, they have nice math for combining the CRCs of pieces to form 39 | // the CRCs of wholes. There are also cryptographic hashes, but those are even 40 | // slower than MD5. 41 | // 42 | 43 | #include 44 | #include 45 | 46 | #define INLINE inline 47 | typedef uint64_t uint64; 48 | typedef uint32_t uint32; 49 | typedef uint16_t uint16; 50 | typedef uint8_t uint8; 51 | 52 | 53 | #define sc_numVars 12 54 | #define sc_blockSize (sc_numVars * 8) 55 | #define sc_bufSize (sc_blockSize * 2) 56 | #define sc_const ( 0xdeadb0efdeadbeefLL ) 57 | #define SPOOKY_HASH_INIT_STATE = { .sc_numVars = sc_numVars, .sc_blockSize = sc_blockSize, .sc_bufSize = sc_bufSize, .sc_const = sc_const, 0 } 58 | 59 | 60 | typedef struct { 61 | uint64 m_data[2*sc_numVars]; // unhashed data, for partial messages 62 | uint64 m_state[sc_numVars]; // internal state of the hash 63 | size_t m_length; // total length of the input so far 64 | uint8 m_remainder; // length of unhashed data stashed in m_data 65 | } spooky_hash_state; 66 | 67 | 68 | 69 | // 70 | // SpookyHash: hash a single message in one call, produce 128-bit output 71 | // 72 | // message to hash 73 | // length of message in bytes 74 | // in/out: in seed 1, out hash value 1 75 | // in/out: in seed 2, out hash value 2 76 | void spooky_Hash128( const void *message, size_t length, uint64 *hash1, uint64 *hash2); 77 | uint64 spooky_Hash64( const void *message, size_t length, uint64 seed); 78 | uint32 spooky_Hash32( const void *message, size_t length, uint32 seed); 79 | 80 | // 81 | // Init: initialize the context of a SpookyHash 82 | // 83 | // any 64-bit value will do, including 0 84 | // different seeds produce independent hashes 85 | void spooky_Init(spooky_hash_state* s, uint64 seed1, uint64 seed2); 86 | 87 | // 88 | // Update: add a piece of a message to a SpookyHash state 89 | // 90 | // message fragment 91 | // length of message fragment in bytes 92 | void spooky_Update(spooky_hash_state* s, const void *message, size_t length); 93 | 94 | 95 | // 96 | // Final: compute the hash for the current SpookyHash state 97 | // 98 | // This does not modify the state; you can keep updating it afterward 99 | // 100 | // The result is the same as if SpookyHash() had been called with 101 | // all the pieces concatenated into one message. 102 | // 103 | // out only: first 64 bits of hash value. 104 | // out only: second 64 bits of hash value. 105 | void spooky_Final(spooky_hash_state* s, uint64 *hash1, uint64 *hash2); 106 | 107 | 108 | 109 | 110 | #endif /* SPOOKY_HASH_H_ */ 111 | -------------------------------------------------------------------------------- /libs/libchaste/include/log/log_levels.h: -------------------------------------------------------------------------------- 1 | /* 2 | * log_levels.h 3 | * 4 | * Created on: Jul 2, 2013 5 | * Author: mgrosvenor 6 | */ 7 | 8 | 9 | //Logging levels - The higher the number, the more this is on the critical path 10 | #define CH_LOG_LVL_FATAL 0 //0: Fatal errors that cause the program to terminate 11 | #define CH_LOG_LVL_ERROR 1 //1: Errors that a user can detect and recover from with substantial effort 12 | #define CH_LOG_LVL_WARN 2 //2: Warnings that do not hinder operation, but may lead to unexpected results 13 | #define CH_LOG_LVL_INFO 3 //3: Generally useful output information 14 | #define CH_LOG_LVL_DEBUG1 4 //4: Shallow depth debug logs 15 | #define CH_LOG_LVL_DEBUG2 5 //5: Medium depth debug logs 16 | #define CH_LOG_LVL_DEBUG3 6 //6: Very deep debug logs 17 | #define CH_LOG_LVL_COUNT 7 18 | 19 | 20 | //Options for output 21 | #define CH_LOG_OUT_STDOUT 0 22 | #define CH_LOG_OUT_STDERR 1 23 | #define CH_LOG_OUT_FILE 2 24 | #define CH_LOG_OUT_SYSLOG 3 25 | -------------------------------------------------------------------------------- /libs/libchaste/include/makeinclude.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | tmp=$1 4 | 5 | rm -rf $tmp/include 6 | find . -type d | grep -v "\.$" | grep -v ".\/include" | grep -v "\/bin" | grep -v "/\.git" | grep -v ".\/dag" | grep -v ".\/netmap" | grep -v "build" > $tmp/.tmp.dirs 7 | find . | grep ".*.h$" | grep -v ".\/include" | grep -v "\/bin" | grep -v "/\.git" | grep -v ".\/dag" | grep -v ".\/netmap" | grep -v "build" > $tmp/.tmp.headers 8 | 9 | python makeinclude.py $tmp 10 | 11 | rm $tmp/.tmp.headers 12 | rm $tmp/.tmp.dirs 13 | 14 | -------------------------------------------------------------------------------- /libs/libchaste/include/options/options.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) Matthew P. Grosvenor, 2012, All Rights Reserved 3 | * 4 | * Tools to parse and print command line options 5 | */ 6 | 7 | #ifndef CH_OPTIONS_H_ 8 | #define CH_OPTIONS_H_ 9 | 10 | #include "../types/types.h" 11 | #include "../log/log.h" //Include this here, so users don't have to 12 | #include "options_vectors.h" 13 | #include "../data_structs/vector/vector_std.h" 14 | 15 | 16 | typedef struct { 17 | char* short_description; 18 | char* long_description; 19 | CH_VECTOR(opts)* opt_defs; 20 | uint64_t count; 21 | ch_bool help; 22 | int unlimted_set; 23 | ch_bool done_init; 24 | } ch_options_t; 25 | 26 | 27 | //Declare all the options parsers for non vector types, with initializers 28 | #define ch_opt_add_declare_i(ch_type_name, c_type_name, short_name, long_name)\ 29 | int ch_opt_add##short_name##i(ch_options_mode_e mode, char short_str, char* long_str, char* descr, c_type_name* result_out, c_type_name default_val) 30 | ch_opt_add_declare_i(CH_BOOL, ch_bool, b, "Boolean"); 31 | ch_opt_add_declare_i(CH_UINT64, u64, u, "Unsigned"); 32 | ch_opt_add_declare_i(CH_INT64, ch_word, i, "Integer"); 33 | ch_opt_add_declare_i(CH_STRING, ch_cstr, s, "String"); 34 | ch_opt_add_declare_i(CH_DOUBLE, ch_float, f, "Float"); 35 | ch_opt_add_declare_i(CH_HEX, u64, x, "Unsigned"); 36 | 37 | //Declare all the options parsers for non vector types, untantalized 38 | #define ch_opt_add_declare_u(ch_type_name, c_type_name, short_name, long_name)\ 39 | int ch_opt_add##short_name##u(ch_options_mode_e mode, char short_str, char* long_str, char* descr, c_type_name* result_out) 40 | ch_opt_add_declare_u(CH_BOOL, ch_bool, b, "boolean"); 41 | ch_opt_add_declare_u(CH_UINT64, u64, u, "unsigned"); 42 | ch_opt_add_declare_u(CH_INT64, ch_word, i, "integer"); 43 | ch_opt_add_declare_u(CH_STRING, ch_cstr, s, "string"); 44 | ch_opt_add_declare_u(CH_DOUBLE, ch_float, f, "float"); 45 | ch_opt_add_declare_u(CH_HEX, u64, x, "Unsigned"); 46 | 47 | //Declare all the options parsers for vector types, with initializers 48 | #define ch_opt_add_declare_VI(ch_type_name, vector_name, c_type_name_default, short_name, long_name)\ 49 | int ch_opt_add##short_name##I(ch_options_mode_e mode, char short_str, char* long_str, char* descr, CH_VECTOR(vector_name)** result_out, c_type_name_default default_val) 50 | ch_opt_add_declare_VI(CH_BOOLS, ch_bool, ch_bool, B, "Boolean List"); 51 | ch_opt_add_declare_VI(CH_UINT64S, u64, u64, U, "Unsigned List"); 52 | ch_opt_add_declare_VI(CH_INT64S, word, ch_word, I, "Integer List"); 53 | ch_opt_add_declare_VI(CH_STRINGS, cstr, ch_cstr, S, "String List"); 54 | ch_opt_add_declare_VI(CH_DOUBLES, float, ch_float, F, "Float List"); 55 | ch_opt_add_declare_VI(CH_HEXS, u64, u64, X, "Unsigned List"); 56 | 57 | //Declare all the options parsers for vector types, with uninitialized 58 | #define ch_opt_add_declare_VU(ch_type_name, vector_name, short_name, long_name)\ 59 | int ch_opt_add##short_name##U(ch_options_mode_e mode, char short_str, char* long_str, char* descr, CH_VECTOR(vector_name)** result_out) 60 | ch_opt_add_declare_VU(CH_BOOLS, ch_bool, B, "Boolean List"); 61 | ch_opt_add_declare_VU(CH_UINT64S, u64, U, "Unsigned List"); 62 | ch_opt_add_declare_VU(CH_INT64S, word, I, "Integer List"); 63 | ch_opt_add_declare_VU(CH_STRINGS, cstr, S, "String List"); 64 | ch_opt_add_declare_VU(CH_DOUBLES, float, F, "Float List"); 65 | ch_opt_add_declare_VU(CH_HEXS, u64, X, "Unsigned List"); 66 | 67 | #define Vector_geti(type,vector,i) ((type*)vector.mem)[i] 68 | 69 | 70 | #define USE_CH_OPTIONS \ 71 | ch_options_t opts = { 0 } 72 | 73 | int ch_opt_name(char* description); 74 | int ch_opt_tail(char* description); 75 | int ch_opt_short_description(char* description); 76 | int ch_opt_long_description(char* description); 77 | int ch_opt_parse(int argc, char** argv); 78 | 79 | 80 | #endif /* CH_OPTIONS_H_ */ 81 | -------------------------------------------------------------------------------- /libs/libchaste/include/options/options_vectors.h: -------------------------------------------------------------------------------- 1 | /* 2 | * options_vectors.h 3 | * 4 | * Created on: 16 Sep 2013 5 | * Author: mpg39 6 | */ 7 | 8 | #ifndef OPTIONS_VECTORS_H_ 9 | #define OPTIONS_VECTORS_H_ 10 | 11 | #include "../data_structs/vector/vector_typed_declare_template.h" 12 | 13 | typedef enum { 14 | CH_OPTION_REQUIRED, //This option is required for the program to run 15 | CH_OPTION_OPTIONAL, //This option is not required but has a default value 16 | CH_OPTION_UNLIMTED, //This option produces a list of outputs 17 | CH_OPTION_FLAG, //This option is optional, and has no arguments 18 | } ch_options_mode_e; 19 | 20 | 21 | typedef struct ch_options_opt{ 22 | ch_options_mode_e mode; 23 | ch_types_e type; 24 | void* var; 25 | char short_str; 26 | char* long_str; 27 | char* descr; 28 | int found; 29 | } ch_options_opt_t; 30 | 31 | 32 | declare_ch_vector(opts, ch_options_opt_t) 33 | 34 | 35 | 36 | #endif /* OPTIONS_VECTORS_H_ */ 37 | -------------------------------------------------------------------------------- /libs/libchaste/include/parsing/bool_parser.h: -------------------------------------------------------------------------------- 1 | /* 2 | * bool_parser.h 3 | * 4 | * Created on: Dec 20, 2012 5 | * Author: mgrosvenor 6 | */ 7 | 8 | #ifndef BOOL_PARSER_H_ 9 | #define BOOL_PARSER_H_ 10 | #include 11 | 12 | #include "numeric_parser.h" 13 | 14 | num_result_t parse_bool(const char* c, size_t n, size_t i); 15 | 16 | 17 | #endif /* BOOL_PARSER_H_ */ 18 | -------------------------------------------------------------------------------- /libs/libchaste/include/parsing/numeric_parser.h: -------------------------------------------------------------------------------- 1 | /* 2 | * numeric_parser.h 3 | * 4 | * Created on: Dec 20, 2012 5 | * Author: mgrosvenor 6 | */ 7 | 8 | #ifndef NUMERIC_PARSER_H_ 9 | #define NUMERIC_PARSER_H_ 10 | 11 | #include 12 | #include 13 | 14 | #include "../types/types.h" 15 | 16 | typedef struct{ 17 | ch_types_e type; 18 | union{ 19 | uint64_t val_uint; 20 | int64_t val_int; 21 | double val_dble; 22 | }; 23 | ch_word index; //If the type returned is a valid number, return the index of the first char directly after the number 24 | } num_result_t; 25 | 26 | 27 | num_result_t parse_number(const char* c, size_t i); 28 | num_result_t parse_nnumber(const char* c, size_t i, size_t n); 29 | 30 | 31 | #endif /* NUMERIC_PARSER_H_ */ 32 | -------------------------------------------------------------------------------- /libs/libchaste/include/parsing/utils.h: -------------------------------------------------------------------------------- 1 | /* 2 | * utils.h 3 | * 4 | * Created on: Dec 20, 2012 5 | * Author: mgrosvenor 6 | */ 7 | 8 | #ifndef UTILS_H_ 9 | #define UTILS_H_ 10 | 11 | 12 | /* 13 | * utils.c 14 | * 15 | * Created on: Dec 20, 2012 16 | * Author: mgrosvenor 17 | */ 18 | 19 | #include "utils.h" 20 | 21 | int isodigit(char c); 22 | int isbdigit(char c); 23 | uint8_t getxdigit(char c); 24 | int iswhite(char c); 25 | int isnwhite(char c); 26 | 27 | 28 | #endif /* UTILS_H_ */ 29 | -------------------------------------------------------------------------------- /libs/libchaste/include/perf/perf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) Matthew P. Grosvenor, 2012, All Rights Reserved 3 | * 4 | * Input stream definition 5 | * 6 | */ 7 | 8 | #ifndef CH_PERF_H_ 9 | #define CH_PERF_H_ 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | #include "../log/log.h" //Include this here, so users don't have to 16 | #include "../utils/util.h" 17 | 18 | //#warning "This is all broken and arch specifc. Need to make the logic and impl separate" 19 | #define uint64_t unsigned long long 20 | 21 | typedef struct { 22 | uint64_t ts; //Time the event was logged 23 | uint32_t event_id; //ID of the event, used to tie start and stop operations together 24 | uint32_t cond_id; //ID of the event, used to differentiate different start/stop conditions for the same ID. 25 | uint64_t data; 26 | } ch_perf_event_t; 27 | 28 | 29 | typedef struct { 30 | //Perf logging control 31 | u64 event_count; 32 | u64 event_index; 33 | u64 max_events; 34 | ch_perf_event_t* events; 35 | 36 | //Output control 37 | int fd; 38 | 39 | //Stop watch temps 40 | u64 timer_start; 41 | u64 timer_stop; 42 | 43 | //Get tsc 44 | u64 tsc; 45 | 46 | 47 | } ch_perf_t; 48 | 49 | 50 | #define USE_CH_PERF(events_max) \ 51 | ch_perf_event_t ch_perf_events[events_max]; \ 52 | ch_perf_t ch_perf = { \ 53 | .event_count = 0, \ 54 | .event_index = 0, \ 55 | .max_events = events_max, \ 56 | .events = ch_perf_events, \ 57 | .fd = -1, \ 58 | .timer_start = ~0, \ 59 | .timer_stop = ~0, \ 60 | .tsc = ~0 \ 61 | } 62 | 63 | extern ch_perf_t ch_perf; 64 | 65 | //Borrowed from linux/arch/x86/include/asm/msr.h 66 | #define EAX_EDX_VAL(low, high) ((low) | ((uint64_t)(high) << 32)) 67 | #define EAX_EDX_RET(low, high) "=a" (low), "=d" (high) 68 | #define DECLARE_ARGS(low, high) uint32_t low, high 69 | 70 | //Notes: 71 | //- This function is written as a macro to ensure that it is inlined 72 | //- Generally calls to rdtsc are prepended by a call to cpuid. This is done so that the 73 | // pipeline is flushed and that there is determinism about the moment when rdtsc is called. 74 | // Flushing the pipeline is an expensive call and not something that we want to do too much 75 | // on the critical path. For this reason I've decided to trade a little accuracy for 76 | // better overall performance. 77 | #ifndef NDEBUG 78 | #define ch_perf_event_start(ch_word_event_id, ch_word_cond_id, ch_word_data) \ 79 | if(likely(ch_perf.event_index < ch_perf.max_events)){ \ 80 | ch_perf.events[ch_perf.event_index].event_id = (ch_word_event_id); \ 81 | ch_perf.events[ch_perf.event_index].cond_id = (ch_word_cond_id); \ 82 | ch_perf.events[ch_perf.event_index].data = (ch_word_data); \ 83 | DECLARE_ARGS(lo, hi); \ 84 | __asm__ __volatile__("rdtsc" : EAX_EDX_RET(lo, hi)); \ 85 | ch_perf.events[ch_perf.event_index].ts = EAX_EDX_VAL(lo, hi); \ 86 | ch_perf.event_index++; \ 87 | } \ 88 | ch_perf.event_count++; 89 | #else 90 | #define ch_perf_event_start(ch_word_event_id, ch_word_cond_id, ch_word_data) 91 | #endif 92 | 93 | 94 | 95 | //Notes: 96 | //- This function is written as a macro to ensure that it is inlined 97 | //- Generally calls to rdtsc are prepended by a call to cpuid. This is done so that the 98 | // pipeline is flushed and that there is determinism about the moment when rdtsc is called. 99 | // Flushing the pipeline is an expensive call and not something that we want to do too much 100 | // on the critical path. For this reason I've decided to trade a little accuracy for 101 | // better overall performance. 102 | #ifndef NDEBUG 103 | #define ch_perf_event_stop(ch_word_event_id, ch_word_cond_id, ch_word_data) \ 104 | if(likely(ch_perf.event_index < ch_perf.max_events)){ \ 105 | DECLARE_ARGS(lo, hi); \ 106 | __asm__ __volatile__("rdtsc" : EAX_EDX_RET(lo, hi)); \ 107 | ch_perf.events[ch_perf.event_index].ts = EAX_EDX_VAL(lo, hi); \ 108 | ch_perf.events[ch_perf.event_index].event_id = (ch_word_event_id) | (1<<31); \ 109 | ch_perf.events[ch_perf.event_index].cond_id = (ch_word_cond_id); \ 110 | ch_perf.events[ch_perf.event_index].data = (ch_word_data); \ 111 | ch_perf.event_index++; \ 112 | } \ 113 | ch_perf.event_count++; 114 | #else 115 | #define ch_perf_event_stop(ch_word_event_id, ch_word_cond_id, ch_word_data) 116 | #endif 117 | 118 | typedef enum { 119 | ch_perf_output_tostdout, 120 | ch_perf_output_tostderr, 121 | ch_perf_output_tofile, 122 | //ch_perf_http, -- Not yet implemented 123 | } ch_perf_output_e; 124 | 125 | 126 | typedef enum { 127 | ch_perf_format_csv, //Output CSV -- Easy to read text format. 128 | ch_perf_format_ssv, //Output SSV -- Easy to read inside GNU plot, useless for everything else 129 | ch_perf_format_binary, //Output in binary format -- Nice and compact, easy to store, anaylse later. 130 | //ch_perf_format_http, -- Not yet implemented 131 | } ch_perf_format_e; 132 | 133 | 134 | void ch_perf_finish_(ch_perf_output_e output, ch_perf_format_e format, const char* filename); 135 | 136 | #ifndef NDEBU 137 | #define ch_perf_finish(ch_perf_output, ch_perf_format, filename)\ 138 | ch_perf_finish_(ch_perf_output, ch_perf_format, filename); 139 | #else 140 | #define ch_perf_finish(ch_perf_output, ch_perf_format, filename) 141 | #endif 142 | 143 | 144 | #define ch_perf_timer_start { \ 145 | DECLARE_ARGS(lo, hi); \ 146 | __asm__ __volatile__("rdtsc" : EAX_EDX_RET(lo, hi)); \ 147 | ch_perf.timer_start = EAX_EDX_VAL(lo, hi); \ 148 | } 149 | 150 | 151 | #define ch_perf_timer_stop { \ 152 | DECLARE_ARGS(lo, hi); \ 153 | __asm__ __volatile__("rdtsc" : EAX_EDX_RET(lo, hi)); \ 154 | ch_perf.timer_stop = EAX_EDX_VAL(lo, hi); \ 155 | } 156 | 157 | #define ch_perf_timer_reset { \ 158 | ch_perf.timer_start = ~0; \ 159 | ch_perf.timer_stop = ~0; \ 160 | } 161 | 162 | 163 | #define ch_perf_sample_tsc { \ 164 | DECLARE_ARGS(lo, hi); \ 165 | __asm__ __volatile__("rdtsc" : EAX_EDX_RET(lo, hi)); \ 166 | ch_perf.tsc = EAX_EDX_VAL(lo, hi); \ 167 | } 168 | 169 | #define ch_perf_cycles_now(result) { \ 170 | DECLARE_ARGS(lo, hi); \ 171 | __asm__ __volatile__("rdtsc" : EAX_EDX_RET(lo, hi)); \ 172 | result = EAX_EDX_VAL(lo, hi); \ 173 | } 174 | 175 | 176 | u64 ch_perf_get_watch_ticks(); 177 | u64 ch_perf_get_tsc_sample(); 178 | 179 | 180 | #endif /* CH_PERF_H_ */ 181 | -------------------------------------------------------------------------------- /libs/libchaste/include/perf/perf_mon.h: -------------------------------------------------------------------------------- 1 | // CamIO 2: perf_mon.h 2 | // Copyright (C) 2013: Matthew P. Grosvenor (matthew.grosvenor@cl.cam.ac.uk) 3 | // Licensed under BSD 3 Clause, please see LICENSE for more details. 4 | 5 | #ifndef PERF_MON_H_ 6 | #define PERF_MON_H_ 7 | 8 | #include "../types/types.h" 9 | #include 10 | 11 | /** 12 | * The TSC counter structure is used for both time and cycle based profiling/performance monitoring. 13 | * The intention is that 1 or more of these structures will be used in an array. Multiple instances 14 | * are used to monitor multiple stop conditions. i.e. If there are multiple paths to exit a function, 15 | * then an idea for which paths are taken most often and how much they cost can be gathered in real- 16 | * time or compiled out. The strong intention is that performance monitoring should be light weight 17 | * enough that it can be run in production without affecting performance (too much). At a minimum, 18 | * counter based performance monitoring is offered which only adds an additional increment operator 19 | * as it's cost but provides useful insight into code performance 20 | */ 21 | 22 | typedef struct { 23 | //Sample using just incrementing a counter 24 | u64 start_count; //Number of times start counter was called. Only used for counter 0 25 | u64 end_count; //Number of times end counter was called 26 | 27 | //Sampling using timestamp counter 28 | u64 nanos_tmp; //Only used for counter 0 29 | u64 nanos_total; 30 | u64 nanos_max; 31 | u64 nanos_min; 32 | double nanos_avg; 33 | 34 | //Sampling using cycle counter 35 | u64 cycles_tmp; //Only used for counter 0 36 | u64 cycles_total; 37 | u64 cycles_max; 38 | u64 cycles_min; 39 | double cycles_avg; 40 | } TSC; 41 | 42 | 43 | /** 44 | * Per module performance monitoring. Use only once per file scope. Never in headers. 45 | */ 46 | 47 | struct perf_mod_generic; 48 | typedef struct perf_mod_generic perf_mod_generic_t; 49 | 50 | struct perf_mod_generic{ 51 | ch_word id; 52 | ch_cstr descr; 53 | ch_word descr_len; 54 | }; 55 | 56 | 57 | #define make_perf_module(PERFS) \ 58 | static struct perf_mod_t { \ 59 | ch_word id; \ 60 | ch_cstr descr; \ 61 | ch_word descr_len; \ 62 | PERFS \ 63 | } perf_mod = { .id = 0, .descr = #PERFS, . descr_len = strlen(#PERFS) } 64 | 65 | 66 | #define get_perf_mod_ref (perf_mod_generic_t*)&perf_mod 67 | 68 | #define perf_mod_start( NAME ) \ 69 | perf_mod.NAME[0].start_count += 1; \ 70 | perf_mod.NAME[0].nanos_tmp = get_nanos_now(); 71 | 72 | 73 | #define perf_mod_end( NAME, ID ) \ 74 | perf_mod.NAME[ID].end_count += 1; \ 75 | perf_mod.NAME[0].nanos_tmp = (get_nanos_now() - perf_mod.NAME[0].nanos_tmp); \ 76 | perf_mod.NAME[ID].nanos_total += perf_mod.NAME[0].nanos_tmp; \ 77 | perf_mod.NAME[ID].nanos_min = perf_mod.NAME[ID].nanos_min == 0 ? perf_mod.NAME[0].nanos_tmp : perf_mod.NAME[0].nanos_tmp < perf_mod.NAME[ID].nanos_min ? perf_mod.NAME[0].nanos_tmp : perf_mod.NAME[ID].nanos_min ; \ 78 | perf_mod.NAME[ID].nanos_max = perf_mod.NAME[0].nanos_tmp > perf_mod.NAME[ID].nanos_max ? perf_mod.NAME[0].nanos_tmp : perf_mod.NAME[ID].nanos_max ; 79 | 80 | 81 | void print_perf_stats(perf_mod_generic_t* perf_mod); 82 | u64 get_nanos_now(); 83 | 84 | #endif /* PERF_MON_H_ */ 85 | -------------------------------------------------------------------------------- /libs/libchaste/include/scripts/camio_prepare_boot.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | #This scrpit kicks off the install process, it checks that a sufficiently new 4 | #version of python is available assuming that it is, starts the installer script 5 | 6 | 7 | #Try to find python or accept the first command line arg as the ptyhon path 8 | python_dir=$(which python) 9 | 10 | if [ $# -gt 0 ] 11 | then 12 | python_dir="$1" 13 | fi 14 | 15 | if [ ! -f "$python_dir" ] 16 | then 17 | echo "Could not find python in \"$python_dir\", Python is not installed." 18 | echo "Please install it before continuing," 19 | fi 20 | 21 | cd scripts 22 | python camio_prepare_boot.py 23 | 24 | 25 | -------------------------------------------------------------------------------- /libs/libchaste/include/string/string.h: -------------------------------------------------------------------------------- 1 | /* 2 | * string_utils.h 3 | * 4 | * Created on: Jul 6, 2013 5 | * Author: mgrosvenor 6 | */ 7 | 8 | #ifndef STRING_UTILS_H_ 9 | #define STRING_UTILS_H_ 10 | 11 | #include "../types/types.h" 12 | #include 13 | 14 | 15 | void ch_str_free(ch_str* s); //Safe free function 16 | i64 ch_str_resize(ch_str* rhs,i64 newsize); // 17 | ch_str ch_str_new(const char* s,i64 size, ch_bool is_char); //Try not to use this, use the macro functions below which are better optimised for useful situations 18 | ch_str ch_str_cat(ch_str* lhs, ch_str rhs); 19 | ch_str ch_str_cat_cstr(ch_str* lhs, const char* cstr); 20 | ch_str ch_str_cat_char(ch_str* lhs, const char ch); 21 | ch_str ch_str_trunc(ch_str* lhs, ch_word count); 22 | ch_word ch_str_eq(ch_str lhs, ch_str rhs); 23 | 24 | 25 | 26 | #define CH_STR_CONST_NEW(name, s) ch_str name = CH_STR_CONST_LIT(s) 27 | #define CH_STR_CONST_LIT(s) { .cstr = (char*)s, .is_const = 1, .slen = strlen(s), .mlen = strlen(s) } 28 | #define CH_STR_CONST_LIT_NSTRL(s) { .cstr = (char*)s, .is_const = 1, .slen = -1, .mlen = -1 } 29 | #define CH_STR_FUNC(s) { .cstr = (char*)s, .is_const = -1, .slen = -1, .mlen = -1 } 30 | #define CH_STR_NULL { .cstr = NULL, .is_const = 1, .slen = 0, .mlen = 0 } 31 | #define CH_STR(s,sz) ch_str_new(s, sz, 0) 32 | 33 | #define CH_STR_CAT(lhs,rhs) ch_str_cat(lhs, rhs) 34 | #define CH_STR_CAT_CSTR(lhs,rhs) ch_str_cat_cstr(lhs, rhs) 35 | #define CH_STR_CAT_CHAR(lhs,rhs) ch_str_cat_char(lhs, rhs) 36 | 37 | #define CH_STR_CSTR(ch_str) ( (ch_str).cstr) 38 | #define CH_STR_CSTR_END(ch_str) ( (ch_str).cstr + (ch_str).slen ) 39 | #define CH_STR_AVAIL(ch_str) ( (ch_str).mlen -1 - (ch_str).slen ) 40 | #define CH_STR_LEN(ch_str) ( (ch_str).slen) 41 | 42 | 43 | #endif /* STRING_UTILS_H_ */ 44 | 45 | 46 | -------------------------------------------------------------------------------- /libs/libchaste/include/sys/sys.h: -------------------------------------------------------------------------------- 1 | /* 2 | * sys.h 3 | * 4 | * Created on: Apr 23, 2015 5 | * Author: mgrosvenor 6 | */ 7 | 8 | #ifndef SYS_H_ 9 | #define SYS_H_ 10 | 11 | #ifdef __APPLE__ 12 | # include "sys_apple.h" 13 | #else 14 | # include "sys_linux.h" 15 | #endif 16 | //#else 17 | //# warning "LibChaste Needs to be ported to this system" 18 | //#endif 19 | 20 | 21 | #endif /* SYS_H_ */ 22 | -------------------------------------------------------------------------------- /libs/libchaste/include/sys/sys_apple.h: -------------------------------------------------------------------------------- 1 | /* 2 | * sys_apple.h 3 | * 4 | * Created on: Apr 23, 2015 5 | * Author: mgrosvenor 6 | */ 7 | 8 | #ifndef SYS_APPLE_H_ 9 | #define SYS_APPLE_H_ 10 | 11 | #include 12 | #include 13 | 14 | #ifdef __APPLE__ //These definitions are only required for apple/darwin 15 | 16 | #define CLOCK_REALTIME 0 17 | void clock_gettime(int ignored, struct timespec *ts); 18 | 19 | #endif 20 | 21 | #endif /* SYS_APPLE_H_ */ 22 | -------------------------------------------------------------------------------- /libs/libchaste/include/sys/sys_linux.h: -------------------------------------------------------------------------------- 1 | /* 2 | * sys_linux.h 3 | * 4 | * Created on: Apr 23, 2015 5 | * Author: mgrosvenor 6 | */ 7 | 8 | #ifndef SYS_LINUX_H_ 9 | #define SYS_LINUX_H_ 10 | 11 | 12 | #endif /* SYS_LINUX_H_ */ 13 | -------------------------------------------------------------------------------- /libs/libchaste/include/term_color/term_color.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ch_term_colors.h 3 | * 4 | * Created on: Jul 2, 2013 5 | * Author: mgrosvenor 6 | */ 7 | #include 8 | 9 | #include "../types/types.h" 10 | #include "../string/string.h" 11 | 12 | 13 | #ifndef CH_TERM_COLORS_H_ 14 | #define CH_TERM_COLORS_H_ 15 | 16 | //Private definitions, don't use directly 17 | #define CH_TERM_ATTR_RAW_RESET "0" 18 | #define CH_TERM_ATTR_RAW_BRIGHT "1" 19 | #define CH_TERM_ATTR_RAW_DIM "2" 20 | #define CH_TERM_ATTR_RAW_UNDERLINE "3" 21 | #define CH_TERM_ATTR_RAW_BLINK "4" 22 | #define CH_TERM_ATTR_RAW_REVERSE "7" 23 | #define CH_TERM_ATTR_RAW_HIDDEN "8" 24 | 25 | //Private definitions, don't use directly 26 | #define CH_TERM_COL_RAW_BLACK "0" 27 | #define CH_TERM_COL_RAW_RED "1" 28 | #define CH_TERM_COL_RAW_GREEN "2" 29 | #define CH_TERM_COL_RAW_YELLOW "3" 30 | #define CH_TERM_COL_RAW_BLUE "4" 31 | #define CH_TERM_COL_RAW_MAGENTA "5" 32 | #define CH_TERM_COL_RAW_CYAN "6" 33 | #define CH_TERM_COL_RAW_WHITE "7" 34 | 35 | //Private definitions, don't use directly 36 | #define CH_TERM_COLOR(col) CH_STR_CONST_LIT_NSTRL("\x1B[3"col"m") 37 | #define CH_TERM_COLOR_ATTR(col,attr) CH_STR_CONST_LIT_NSTRL("\x1B["attr";3"col"m") 38 | #define CH_TERM_ATTR(attr) CH_STR_CONST_LIT_NSTRL("\x1B["attr"m") 39 | 40 | //Colors to be used by users - Please use directly! 41 | #define CH_TERM_COL_NONE CH_TERM_ATTR(CH_TERM_ATTR_RAW_RESET) 42 | #define CH_TERM_COL_BLACK CH_TERM_COLOR(CH_TERM_COL_RAW_BLACK) 43 | #define CH_TERM_COL_BLUE CH_TERM_COLOR(CH_TERM_COL_RAW_BLUE) 44 | #define CH_TERM_COL_CYAN CH_TERM_COLOR(CH_TERM_COL_RAW_CYAN) 45 | #define CH_TERM_COL_GREEN CH_TERM_COLOR(CH_TERM_COL_RAW_GREEN) 46 | #define CH_TERM_COL_MAGENTA CH_TERM_COLOR(CH_TERM_COL_RAW_MAGENTA) 47 | #define CH_TERM_COL_RED CH_TERM_COLOR(CH_TERM_COL_RAW_RED) 48 | #define CH_TERM_COL_WHITE CH_TERM_COLOR(CH_TERM_COL_RAW_WHITE) 49 | #define CH_TERM_COL_YELLOW CH_TERM_COLOR(CH_TERM_COL_RAW_YELLOW) 50 | 51 | #define CH_TERM_COL_BRIGHT_BLACK CH_TERM_COLOR_ATTR(CH_TERM_COL_RAW_BLACK, CH_TERM_ATTR_RAW_BRIGHT) 52 | #define CH_TERM_COL_BRIGHT_BLUE CH_TERM_COLOR_ATTR(CH_TERM_COL_RAW_BLUE, CH_TERM_ATTR_RAW_BRIGHT) 53 | #define CH_TERM_COL_BRIGHT_CYAN CH_TERM_COLOR_ATTR(CH_TERM_COL_RAW_CYAN, CH_TERM_ATTR_RAW_BRIGHT) 54 | #define CH_TERM_COL_BRIGHT_GREEN CH_TERM_COLOR_ATTR(CH_TERM_COL_RAW_GREEN, CH_TERM_ATTR_RAW_BRIGHT) 55 | #define CH_TERM_COL_BRIGHT_MAGENTA CH_TERM_COLOR_ATTR(CH_TERM_COL_RAW_MAGENTA, CH_TERM_ATTR_RAW_BRIGHT) 56 | #define CH_TERM_COL_BRIGHT_RED CH_TERM_COLOR_ATTR(CH_TERM_COL_RAW_RED, CH_TERM_ATTR_RAW_BRIGHT) 57 | #define CH_TERM_COL_BRIGHT_WHITE CH_TERM_COLOR_ATTR(CH_TERM_COL_RAW_WHITE, CH_TERM_ATTR_RAW_BRIGHT) 58 | #define CH_TERM_COL_BRIGHT_YELLOW CH_TERM_COLOR_ATTR(CH_TERM_COL_RAW_YELLOW, CH_TERM_ATTR_RAW_BRIGHT) 59 | 60 | #define CH_TERM_COL_DIM_BLACK CH_TERM_COLOR_ATTR(CH_TERM_COL_RAW_BLACK, CH_TERM_ATTR_RAW_DIM) 61 | #define CH_TERM_COL_DIM_BLUE CH_TERM_COLOR_ATTR(CH_TERM_COL_RAW_BLUE, CH_TERM_ATTR_RAW_DIM) 62 | #define CH_TERM_COL_DIM_CYAN CH_TERM_COLOR_ATTR(CH_TERM_COL_RAW_CYAN, CH_TERM_ATTR_RAW_DIM) 63 | #define CH_TERM_COL_DIM_GREEN CH_TERM_COLOR_ATTR(CH_TERM_COL_RAW_GREEN, CH_TERM_ATTR_RAW_DIM) 64 | #define CH_TERM_COL_DIM_MAGENTA CH_TERM_COLOR_ATTR(CH_TERM_COL_RAW_MAGENTA, CH_TERM_ATTR_RAW_DIM) 65 | #define CH_TERM_COL_DIM_WHITE CH_TERM_COLOR_ATTR(CH_TERM_COL_RAW_WHITE, CH_TERM_ATTR_RAW_DIM) 66 | #define CH_TERM_COL_DIM_RED CH_TERM_COLOR_ATTR(CH_TERM_COL_RAW_RED, CH_TERM_ATTR_RAW_DIM) 67 | #define CH_TERM_COL_DIM_YELLOW CH_TERM_COLOR_ATTR(CH_TERM_COL_RAW_YELLOW, CH_TERM_ATTR_RAW_DIM) 68 | 69 | #define CH_TERM_COL_EMPH_BLACK CH_TERM_COLOR_ATTR(CH_TERM_COL_RAW_BLACK, CH_TERM_ATTR_RAW_UNDERLINE) 70 | #define CH_TERM_COL_EMPH_BLUE CH_TERM_COLOR_ATTR(CH_TERM_COL_RAW_BLUE, CH_TERM_ATTR_RAW_UNDERLINE) 71 | #define CH_TERM_COL_EMPH_CYAN CH_TERM_COLOR_ATTR(CH_TERM_COL_RAW_CYAN, CH_TERM_ATTR_RAW_UNDERLINE) 72 | #define CH_TERM_COL_EMPH_GREEN CH_TERM_COLOR_ATTR(CH_TERM_COL_RAW_GREEN, CH_TERM_ATTR_RAW_UNDERLINE) 73 | #define CH_TERM_COL_EMPH_MAGENTA CH_TERM_COLOR_ATTR(CH_TERM_COL_RAW_MAGENTA, CH_TERM_ATTR_RAW_UNDERLINE) 74 | #define CH_TERM_COL_EMPH_WHITE CH_TERM_COLOR_ATTR(CH_TERM_COL_RAW_WHITE, CH_TERM_ATTR_RAW_UNDERLINE) 75 | #define CH_TERM_COL_EMPH_RED CH_TERM_COLOR_ATTR(CH_TERM_COL_RAW_RED, CH_TERM_ATTR_RAW_UNDERLINE) 76 | #define CH_TERM_COL_EMPH_YELLOW CH_TERM_COLOR_ATTR(CH_TERM_COL_RAW_YELLOW, CH_TERM_ATTR_RAW_UNDERLINE) 77 | 78 | 79 | typedef ch_str ch_colour_t; 80 | 81 | int dprintf_color_va(int fd, ch_colour_t color, ch_str format, va_list va); 82 | int dprintf_color(int fd, ch_colour_t color, ch_str format, ...); 83 | int printf_color_va( ch_colour_t color, ch_str format, va_list va); 84 | int printf_color( ch_colour_t color, ch_str format, ...); 85 | 86 | 87 | 88 | #endif /* CH_TERM_COLORS_H_ */ 89 | -------------------------------------------------------------------------------- /libs/libchaste/include/timing/timestamp.h: -------------------------------------------------------------------------------- 1 | /* 2 | * timestamp.h 3 | * 4 | * Created on: Jul 3, 2013 5 | * Author: mgrosvenor 6 | */ 7 | 8 | #ifndef TIMESTAMP_H_ 9 | #define TIMESTAMP_H_ 10 | 11 | #include "../types/types.h" 12 | 13 | ch_str generate_iso_timestamp(ch_bool use_gmt, ch_word subseconds, ch_bool incl_tz_offset); 14 | 15 | #endif /* TIMESTAMP_H_ */ 16 | -------------------------------------------------------------------------------- /libs/libchaste/include/types/types.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) Matthew P. Grosvenor, 2012, All Rights Reserved 3 | */ 4 | 5 | 6 | #ifndef CH_TYPES_H_ 7 | #define CH_TYPES_H_ 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | //#include "../deps/libcgen/vector.h" 14 | 15 | //Unsigned types, use these sparingly 16 | typedef uint8_t u8; 17 | typedef uint16_t u16; 18 | typedef uint32_t u32; 19 | //typedef uint64_t u64; //Try to avoid this unless absolutely necessary. Too many bugs are caused by int/uint mixups 20 | typedef unsigned long long u64; //Try to avoid this unless absolutely necessary. Too many bugs are caused by int/uint mixups 21 | 22 | //Signed types, use these sparingly 23 | typedef int8_t i8; 24 | typedef int16_t i16; 25 | typedef int32_t i32; 26 | typedef long long i64; //Try to avoid this, use ch_word. 27 | 28 | 29 | //CH Library Preferred Types 30 | typedef size_t ch_machine; //Use this sparingly, only when it really matters that the int size is the same as the word size 31 | typedef i8 ch_char; 32 | typedef u8 ch_byte; 33 | typedef long long int ch_word; //Singed int chosen specifically to avoid underrun and signed/unsigned comparisons 34 | typedef bool ch_bool; //Give the compiler tons of freedom to decide what to do here 35 | typedef double ch_float; //Again we preference 64bit machines explicitly 36 | 37 | //Please avoid using char*. CH strings are a thin wrapper, that support easy concatenation and other functions 38 | typedef struct { 39 | char* cstr; //Can be NULL or point to a const or non-const memory area 40 | int is_const; //Can be -1 (unknown), 0 (can call free) or 1 (cannot call free) 41 | i64 slen; //A valid string len is always -1 (unknown), or < mlen 42 | i64 mlen; //A valid memory len is always -1 (unknown), or > slen 43 | } ch_str; 44 | 45 | typedef char* ch_cstr; //Use this sparingly, only when it really matters that you are backwards compatible with c-strings 46 | typedef const char* ch_ccstr; //Use this sparingly as well 47 | 48 | //#define cstr(bstr) ((char*)(bstr->data)) 49 | 50 | typedef enum { 51 | CH_NO_TYPE = 0, 52 | CH_BOOL, 53 | CH_UINT64, 54 | CH_HEX, 55 | CH_INT64, 56 | CH_STRING, 57 | CH_DOUBLE, 58 | CH_BOOLS, //Vector types 59 | CH_UINT64S, 60 | CH_INT64S, 61 | CH_HEXS, 62 | CH_STRINGS, 63 | CH_DOUBLES, 64 | } ch_types_e; 65 | 66 | ch_word is_vector(ch_word type); 67 | 68 | typedef int (*cmp_void_f)(const void*, const void*); 69 | 70 | 71 | 72 | #endif /* CH_TYPES_H_ */ 73 | -------------------------------------------------------------------------------- /libs/libchaste/include/utils/debug.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015, All rights reserved. 3 | * See LICENSE.txt for full details. 4 | * 5 | * Created: Jul 30, 2015 6 | * File name: debug.h 7 | * Description: 8 | * 9 | */ 10 | #ifndef DEBUG_H_ 11 | #define DEBUG_H_ 12 | 13 | #include "../types/types.h" 14 | 15 | typedef enum { 16 | DBG, 17 | ERR, 18 | WARN 19 | } ch_dbg_mode_e; 20 | 21 | 22 | ch_word ch_debug_out_( 23 | ch_bool info, 24 | ch_dbg_mode_e mode, 25 | ch_word line_num, 26 | const char* filename, 27 | const char* function, 28 | const char* format, ... ); 29 | #define ERR( /*format, args*/...) camio_err_helper(__VA_ARGS__, "") 30 | #define camio_err_helper(format, ...) ch_debug_out_(true, ERR, __LINE__, __FILE__, __FUNCTION__, format, __VA_ARGS__ ) 31 | #define ERR2( /*format, args*/...) camio_err_helper(__VA_ARGS__, "") 32 | #define camio_err_helper2(format, ...) ch_debug_out_(false, ERR, __LINE__, __FILE__, __FUNCTION__, format, __VA_ARGS__ ) 33 | 34 | 35 | 36 | #ifndef NDEBUG 37 | //ch_word ch_debug_out_(ch_bool info, ch_word line_num, const char* filename, const char* function, const char* format, ... ); 38 | #define DBG( /*format, args*/...) ch_debug_helper(__VA_ARGS__, "") 39 | #define ch_debug_helper(format, ...) ch_debug_out_(true,DBG,__LINE__, __FILE__, __FUNCTION__, format, __VA_ARGS__ ) 40 | #define DBG2( /*format, args*/...) ch_debug_helper2(__VA_ARGS__, "") 41 | #define ch_debug_helper2(format, ...) ch_debug_out_(false,DBG,__LINE__, __FILE__, __FUNCTION__, format, __VA_ARGS__ ) 42 | #define WARN( /*format, args*/...) ch_debug_helper3(__VA_ARGS__, "") 43 | #define ch_debug_helper3(format, ...) ch_debug_out_(true,WARN,__LINE__, __FILE__, __FUNCTION__, format, __VA_ARGS__ ) 44 | #else 45 | #define DBG( /*format, args*/...) 46 | #define DBG2( /*format, args*/...) 47 | #define WARN( /*format, args*/...) 48 | #endif 49 | 50 | void hexdump(const void *data, int size); 51 | 52 | 53 | #endif /* DEBUG_H_ */ 54 | -------------------------------------------------------------------------------- /libs/libchaste/include/utils/util.h: -------------------------------------------------------------------------------- 1 | /* 2 | * util.h 3 | * 4 | * Created on: Nov 18, 2012 5 | * Author: mgrosvenor 6 | */ 7 | 8 | #ifndef CH_UTIL_H_ 9 | #define CH_UTIL_H_ 10 | 11 | #define iflikely(x) if(__builtin_expect((long long)(x),1)) 12 | #define ifunlikely(x) if(__builtin_expect((long long)(x),0)) 13 | #define ifeqlikely(x) if((x)) 14 | 15 | #define MIN(x,y) ( (x) < (y) ? (x) : (y)) 16 | #define MIN3(x,y,z) (MIN(MIN(x,y),z)) 17 | #define MAX(x,y) ( (x) > (y) ? (x) : (y)) 18 | 19 | #include "../types/types.h" 20 | #include "debug.h" 21 | ///** 22 | // * (from Linux kernel source) 23 | // * container_of - cast a member of a structure out to the containing structure 24 | // * @ptr: the pointer to the member. 25 | // * @type: the type of the container struct this is embedded in. 26 | // * @member: the name of the member within the struct. 27 | // * 28 | // */ 29 | #define container_of(ptr, type, member) ({ \ 30 | const typeof( ((type *)0)->member ) *__mptr = (ptr); \ 31 | (type *)( (char *)__mptr - offsetof(type,member) );}) 32 | 33 | 34 | /* 35 | * The below version is borrowed from http://stackoverflow.com/questions/10269685/kernels-container-of-any-way-to-make-it-iso-conforming 36 | * The new one is better because it is iso compatible (apparently) 37 | */ 38 | 39 | #ifdef __GNUC__ 40 | #define member_type(type, member) __typeof__ (((type *)0)->member) 41 | #else 42 | #define member_type(type, member) const void 43 | #endif 44 | 45 | 46 | //#ifndef __clang__ 47 | // #define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) 48 | //#endif 49 | 50 | ch_word next_pow2(ch_word v); 51 | 52 | #define CH_ASSERT(p) do { if(!(p)) { fprintf(stdout, "Error in %s: failed assertion \""#p"\" on line %u\n", __FUNCTION__, __LINE__); result = 0; } } while(0) 53 | 54 | //Uses integer division to round up 55 | #define round_up( value, nearest) ((( value + nearest -1) / nearest ) * nearest ) 56 | 57 | #endif /* UTIL_H_ */ 58 | -------------------------------------------------------------------------------- /libs/libchaste/libchaste.a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cisco/exact-capture/2a7b74e66cbbafcec7c39947dea63bec03972685/libs/libchaste/libchaste.a -------------------------------------------------------------------------------- /src/data_structs/eiostream_vec.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017,2018 All rights reserved. 3 | * See LICENSE.txt for full details. 4 | * 5 | * Created: 7 Jul 2017 6 | * Author: Matthew P. Grosvenor 7 | * Description: 8 | * Implementation of a libchaste vector, containing exactio 9 | * streams 10 | */ 11 | 12 | 13 | #include 14 | #include "eiostream_vec.h" 15 | 16 | define_ch_vector(eiostreamp,eio_stream_t*) 17 | -------------------------------------------------------------------------------- /src/data_structs/eiostream_vec.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017,2018 All rights reserved. 3 | * See LICENSE.txt for full details. 4 | * 5 | * Created: 7 Jul 2017 6 | * Author: Matthew P. Grosvenor 7 | * Description: 8 | * Declaration of a libchaste vector, containing exactio 9 | * streams 10 | */ 11 | 12 | 13 | #ifndef SRC_DATA_STRUCTS_EIOSTREAM_VEC_H_ 14 | #define SRC_DATA_STRUCTS_EIOSTREAM_VEC_H_ 15 | 16 | #include 17 | 18 | #include "../exactio/exactio.h" 19 | 20 | declare_ch_vector(eiostreamp,eio_stream_t*) 21 | 22 | 23 | #endif /* SRC_DATA_STRUCTS_PTHREAD_VEC_H_ */ 24 | -------------------------------------------------------------------------------- /src/data_structs/expcap.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017,2018 All rights reserved. 3 | * See LICENSE.txt for full details. 4 | * 5 | * Created: 5 Mar 2018 6 | * Author: Matthew P. Grosvenor 7 | * Description: 8 | * Declaration of a the expcap footer structure. This structure 9 | * makes it possible to include picosecond timestamps (for high resolution 10 | * capture), port and device IDs (for traceability) and dropped counters and 11 | * flags (for debugging). It is appended to each packet. 12 | */ 13 | 14 | 15 | #ifndef SRC_DATA_STRUCTS_EXPCAP_H_ 16 | #define SRC_DATA_STRUCTS_EXPCAP_H_ 17 | 18 | #include 19 | 20 | enum { 21 | EXPCAP_FLAG_NONE = 0x00, //No flags 22 | EXPCAP_FLAG_HASCRC = 0x01, //New CRC included 23 | EXPCAP_FLAG_ABRT = 0x02, //Frame aborted 24 | EXPCAP_FLAG_CRPT = 0x04, //Frame corrupt 25 | EXPCAP_FLAG_TRNC = 0x08, //Frame truncated 26 | EXPCAP_FLAG_SWOVFL = 0x10, //A software overflow happened 27 | EXPCAP_FLAG_HWOVFL = 0x20, //A hardware overflow happened 28 | }; 29 | 30 | 31 | typedef struct expcap_pktftr { 32 | uint64_t ts_secs : 32; /* 32bit seconds = max 136 years */ 33 | uint64_t ts_psecs :40; /* 40bit picos = max 1.09 seconds */ 34 | uint8_t flags; 35 | uint8_t dev_id; 36 | uint8_t port_id; 37 | union { 38 | struct{ 39 | uint16_t dropped; 40 | uint16_t _reserved; 41 | } extra; 42 | uint32_t new_fcs; 43 | } foot; 44 | } __attribute__((packed)) expcap_pktftr_t; 45 | 46 | 47 | #endif /* SRC_DATA_STRUCTS_EXPCAP_H_ */ 48 | -------------------------------------------------------------------------------- /src/data_structs/pcap-structures.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017,2018 All rights reserved. 3 | * See LICENSE.txt for full details. 4 | * 5 | * Created: 10 July 2017 6 | * Author: Matthew P. Grosvenor 7 | * Description: 8 | * A local declaration of the the PCAP file format headers, to avoid pulling in 9 | * all of libpcap 10 | */ 11 | 12 | 13 | #ifndef SRC_DATA_STRUCTS_PCAP_STRUCTURES_H_ 14 | #define SRC_DATA_STRUCTS_PCAP_STRUCTURES_H_ 15 | 16 | #include 17 | #include 18 | #include "../data_structs/timespecps.h" 19 | 20 | #define TCPDUMP_MAGIC 0xa1b2c3d4 21 | #define NSEC_TCPDUMP_MAGIC 0xa1b23c4d 22 | #define PCAP_VERSION_MAJOR 2 23 | #define PCAP_VERSION_MINOR 4 24 | #define DLT_EN10MB 1 25 | 26 | typedef struct __attribute__ ((packed)) pcap_file_header { 27 | uint32_t magic; 28 | uint16_t version_major; 29 | uint16_t version_minor; 30 | uint32_t thiszone; /* gmt to local correction */ 31 | uint32_t sigfigs; /* accuracy of timestamps */ 32 | uint32_t snaplen; /* max length saved portion of each pkt */ 33 | uint32_t linktype; /* data link type (LINKTYPE_*) */ 34 | } __attribute__ ((packed)) pcap_file_header_t; 35 | 36 | typedef struct __attribute__ ((packed)) pcap_pkthdr { 37 | union { 38 | struct{ 39 | uint32_t ts_sec; /* time stamp */ 40 | uint32_t ts_usec; 41 | } us; 42 | struct{ 43 | uint32_t ts_sec; /* time stamp */ 44 | uint32_t ts_nsec; 45 | } ns; 46 | int64_t raw; 47 | } ts; 48 | 49 | uint32_t caplen; /* length of portion present */ 50 | uint32_t len; /* length this packet (off wire) */ 51 | } __attribute__ ((packed)) pcap_pkthdr_t; 52 | 53 | #endif /* SRC_DATA_STRUCTS_PCAP_STRUCTURES_H_ */ 54 | -------------------------------------------------------------------------------- /src/data_structs/pthread_vec.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017,2018 All rights reserved. 3 | * See LICENSE.txt for full details. 4 | * 5 | * Created: 7 July 2017 6 | * Author: Matthew P. Grosvenor 7 | * Description: 8 | * Implementaton of a libchaste vector containing pthread structures. 9 | */ 10 | #include 11 | #include "pthread_vec.h" 12 | 13 | define_ch_vector(pthread,pthread_t) 14 | 15 | define_ch_vector_cmp(pthread,pthread_t) 16 | -------------------------------------------------------------------------------- /src/data_structs/pthread_vec.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017,2018 All rights reserved. 3 | * See LICENSE.txt for full details. 4 | * 5 | * Created: 7 July 2017 6 | * Author: Matthew P. Grosvenor 7 | * Description: 8 | * Declaration of a libchaste vector containing pthread structures. 9 | */ 10 | 11 | #ifndef SRC_DATA_STRUCTS_PTHREAD_VEC_H_ 12 | #define SRC_DATA_STRUCTS_PTHREAD_VEC_H_ 13 | 14 | #include 15 | #include 16 | 17 | declare_ch_vector(pthread,pthread_t) 18 | 19 | 20 | #endif /* SRC_DATA_STRUCTS_PTHREAD_VEC_H_ */ 21 | -------------------------------------------------------------------------------- /src/data_structs/timespecps.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017,2018 All rights reserved. 3 | * See LICENSE.txt for full details. 4 | * 5 | * Created: 13 July 2017 6 | * Author: Matthew P. Grosvenor 7 | * Description: 8 | * A local definition of a timespec like structure for containing picosecond 9 | * values 10 | */ 11 | 12 | #ifndef SRC_DATA_STRUCTS_TIMESPECPS_H_ 13 | #define SRC_DATA_STRUCTS_TIMESPECPS_H_ 14 | 15 | #include 16 | 17 | typedef struct timespecps { 18 | uint64_t tv_sec; 19 | uint64_t tv_psec; 20 | } __attribute__((packed)) timespecps_t; 21 | 22 | 23 | #endif /* SRC_DATA_STRUCTS_TIMESPECPS_H_ */ 24 | -------------------------------------------------------------------------------- /src/exact-capture-listener.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017,2018 All rights reserved. 3 | * See LICENSE.txt for full details. 4 | * 5 | * Created: 4 Aug 2017 6 | * Author: Matthew P. Grosvenor 7 | * Description: Definition of listener threads 8 | */ 9 | 10 | 11 | #ifndef SRC_EXACT_CAPTURE_LISTENER_C_ 12 | #define SRC_EXACT_CAPTURE_LISTENER_C_ 13 | 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | #include "data_structs/pthread_vec.h" 31 | #include "data_structs/eiostream_vec.h" 32 | #include "data_structs/pcap-structures.h" 33 | 34 | #include "exactio/exactio.h" 35 | #include "exactio/exactio_exanic.h" 36 | #include "exactio/exactio_timing.h" 37 | 38 | #include "exact-capture.h" 39 | 40 | typedef struct 41 | { 42 | char* interface; 43 | CH_VECTOR(cstr)* dests; 44 | volatile bool* stop; 45 | bool dummy_istream; 46 | bool dummy_ostream; 47 | int64_t ltid; /* Listener thread id */ 48 | 49 | exanic_t* nic; 50 | int exanic_port; 51 | int exanic_dev_num; 52 | char exanic_dev[16]; 53 | 54 | bool kernel_bypass; 55 | bool promisc; 56 | 57 | } listener_params_t; 58 | 59 | void* listener_thread (void* params); 60 | 61 | #endif /* SRC_EXACT_CAPTURE_LISTENER_C_ */ 62 | -------------------------------------------------------------------------------- /src/exact-capture-writer.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017,2018 All rights reserved. 3 | * See LICENSE.txt for full details. 4 | * 5 | * Created: 4 Aug 2017 6 | * Author: Matthew P. Grosvenor 7 | * Description: Definition of writer threads 8 | */ 9 | 10 | 11 | #ifndef SRC_EXACT_CAPTURE_WRITER_C_ 12 | #define SRC_EXACT_CAPTURE_WRITER_C_ 13 | 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | #include "data_structs/pthread_vec.h" 31 | #include "data_structs/eiostream_vec.h" 32 | #include "data_structs/pcap-structures.h" 33 | 34 | #include "exactio/exactio.h" 35 | #include "exactio/exactio_exanic.h" 36 | #include "exactio/exactio_timing.h" 37 | 38 | #include "exact-capture.h" 39 | #include "utils.h" 40 | 41 | typedef struct 42 | { 43 | char* destination; 44 | CH_VECTOR(cstr)* interfaces; 45 | int* exanic_port_id; 46 | int* exanic_dev_id; 47 | volatile bool* stop; 48 | bool dummy_istream; 49 | bool dummy_ostream; 50 | int64_t wtid; /* Writer thread id */ 51 | } writer_params_t; 52 | 53 | typedef struct 54 | { 55 | eio_stream_t* istream; 56 | eio_stream_t* exa_istream; 57 | ch_word dev_id; 58 | ch_word port_num; 59 | } istream_state_t; 60 | 61 | void* writer_thread (void* params); 62 | 63 | 64 | #endif /* SRC_EXACT_CAPTURE_WRITER_C_ */ 65 | -------------------------------------------------------------------------------- /src/exact-capture.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017,2018 All rights reserved. 3 | * See LICENSE.txt for full details. 4 | * 5 | * Created: 4 Aug 2017 6 | * Author: Matthew P. Grosvenor 7 | * Description: Useful global definitions for the application, especially stats 8 | */ 9 | 10 | 11 | #ifndef SRC_EXACT_CAPTURE_H_ 12 | #define SRC_EXACT_CAPTURE_H_ 13 | 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | 21 | #define MIN_ETH_PKT (64) 22 | 23 | /* 24 | * High efficiency disk writing uses O_DIRECT, but writing must be aligned and 25 | * sized as a multiple of the disk block size 26 | */ 27 | #define DISK_BLOCK (4096) 28 | 29 | /* 30 | * BRINGs are used join listener and writer threads to each other. They are 31 | * named shared memory rings that reside in /dev/shm. 32 | */ 33 | #define BRING_NAME_LEN (512) 34 | /*Must be a multiple of disk block size. 512 * 4096 = 2MB */ 35 | #define BRING_SLOT_SIZE (512 * DISK_BLOCK) 36 | #define BRING_SLOT_COUNT (128) 37 | 38 | /*Maximum number of input and output threads/cores*/ 39 | #define MAX_OTHREADS (64) 40 | #define MAX_ITHREADS (64) 41 | 42 | typedef struct 43 | { 44 | int64_t swofl; 45 | int64_t hwofl; 46 | int64_t dropped; 47 | int64_t errors; 48 | int64_t spins1_rx; 49 | int64_t spinsP_rx; 50 | int64_t bytes_rx; 51 | int64_t packets_rx; 52 | 53 | } lstats_t __attribute__( ( aligned ( 8 ) ) ); 54 | 55 | 56 | 57 | typedef struct 58 | { 59 | int64_t pcbytes; /* capture packet bytes */ 60 | int64_t plbytes; /* wire length packet bytes */ 61 | int64_t dbytes; /* to disk bytes */ 62 | int64_t packets; 63 | int64_t spins; 64 | 65 | } wstats_t __attribute__( ( aligned ( 8 ) ) ); 66 | 67 | 68 | 69 | #endif /* SRC_EXACT_CAPTURE_H_ */ 70 | -------------------------------------------------------------------------------- /src/exactio/exactio.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017,2018 All rights reserved. 3 | * See LICENSE.txt for full details. 4 | * 5 | * Created: 21 Jun 2017 6 | * Author: Matthew P. Grosvenor 7 | * Description: 8 | * A "factory" function for creating and destroying exactio streams. 9 | */ 10 | 11 | 12 | 13 | #include "exactio.h" 14 | #include "exactio_file.h" 15 | #include "exactio_dummy.h" 16 | #include "exactio_exanic.h" 17 | #include "exactio_bring.h" 18 | 19 | int eio_new(eio_args_t* args, eio_stream_t** result) 20 | { 21 | 22 | switch(args->type){ 23 | case EIO_FILE: return NEW_IOSTREAM(file,result,&args->args.file); 24 | case EIO_DUMMY: return NEW_IOSTREAM(dummy,result,&args->args.dummy); 25 | case EIO_EXA: return NEW_IOSTREAM(exa,result,&args->args.exa); 26 | case EIO_BRING:return NEW_IOSTREAM(bring,result,&args->args.bring); 27 | } 28 | 29 | return -1; 30 | } 31 | 32 | void eio_des(eio_stream_t* this) 33 | { 34 | this->vtable.destroy(this); 35 | } 36 | 37 | 38 | -------------------------------------------------------------------------------- /src/exactio/exactio.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017,2018 All rights reserved. 3 | * See LICENSE.txt for full details. 4 | * 5 | * Created: 19 Jun 2017 6 | * Author: Matthew P. Grosvenor 7 | * Description: 8 | * An abstract IO interface to make it possible to perf test Exact Capture. 9 | * These generic functions make it easier to call exactio functions without 10 | * having to put "this" pointers everywhere. 11 | */ 12 | 13 | 14 | #ifndef EXACTIO_IO_H_ 15 | #define EXACTIO_IO_H_ 16 | 17 | #include "exactio_bring.h" 18 | #include "exactio_file.h" 19 | #include "exactio_stream.h" 20 | #include "exactio_dummy.h" 21 | #include "exactio_exanic.h" 22 | 23 | #include "../data_structs/timespecps.h" 24 | 25 | typedef enum { 26 | EIO_DUMMY, 27 | EIO_FILE, 28 | EIO_EXA, 29 | EIO_BRING, 30 | } exactio_stream_type_t; 31 | 32 | 33 | typedef struct { 34 | exactio_stream_type_t type; 35 | union { 36 | exa_args_t exa; 37 | dummy_args_t dummy; 38 | file_args_t file; 39 | bring_args_t bring; 40 | } args; 41 | } eio_args_t; 42 | 43 | 44 | int eio_new(eio_args_t* args, eio_stream_t** result); 45 | void eio_des(eio_stream_t* this); 46 | 47 | /* Keep these implementations in the header file so that they can (ideally) be 48 | * compiled inline */ 49 | 50 | //Read operations 51 | static inline eio_error_t eio_rd_acq(eio_stream_t* this, char** buffer, int64_t* len, int64_t* ts ) 52 | { 53 | return this->vtable.read_acquire(this, buffer, len, ts); 54 | } 55 | 56 | static inline eio_error_t eio_rd_rel(eio_stream_t* this, int64_t* ts) 57 | { 58 | return this->vtable.read_release(this, ts); 59 | } 60 | 61 | //Write operations 62 | static inline eio_error_t eio_wr_acq(eio_stream_t* this, char** buffer, int64_t* len, int64_t* ts) 63 | { 64 | return this->vtable.write_acquire(this, buffer, len, ts); 65 | } 66 | 67 | 68 | static inline eio_error_t eio_wr_rel(eio_stream_t* this, int64_t len, int64_t* ts) 69 | { 70 | return this->vtable.write_release(this,len, ts); 71 | } 72 | 73 | 74 | #endif /* EXACTIO_IO_H_ */ 75 | -------------------------------------------------------------------------------- /src/exactio/exactio_bring.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018 All rights reserved. 3 | * See LICENSE.txt for full details. 4 | * 5 | * Created: 19 Jun 2017 6 | * Author: Matthew P. Grosvenor 7 | * Description: 8 | * Definition of a memory mapped "blocking ring" or circular queue to join 9 | * listener and writer threads using the exactio abstract IO interface. 10 | */ 11 | 12 | #ifndef EXACTIO_BRING_H_ 13 | #define EXACTIO_BRING_H_ 14 | 15 | #include "exactio_stream.h" 16 | 17 | typedef struct { 18 | char* filename; 19 | uint64_t isserver; 20 | 21 | //These params only required if server 22 | uint64_t slot_size; 23 | uint64_t slot_count; 24 | uint64_t dontexpand; 25 | } bring_args_t; 26 | 27 | NEW_IOSTREAM_DECLARE(bring,bring_args_t); 28 | 29 | #endif /* EXACTIO_BRING_H_ */ 30 | -------------------------------------------------------------------------------- /src/exactio/exactio_dummy.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018 All rights reserved. 3 | * See LICENSE.txt for full details. 4 | * 5 | * Created: 19 Jun 2017 6 | * Author: Matthew P. Grosvenor 7 | * Description: 8 | * Definition of a dummy stream using the exactio abstract IO interface. The 9 | * dummy simply returns a static memory region rather than performing I/O 10 | * operations. 11 | */ 12 | 13 | 14 | 15 | #ifndef EXACTIO_DUMMY_H_ 16 | #define EXACTIO_DUMMY_H_ 17 | 18 | #include "exactio_stream.h" 19 | 20 | 21 | typedef enum 22 | { 23 | DUMMY_MODE_NONE, /* Reads as if there is one big empty buffer */ 24 | DUMMY_MODE_EXPCAP, /* The buffer full of minimum sized expcap records */ 25 | DUMMY_MODE_EXANIC, /* The buffer is full of minimum sized ExaNIC chunks */ 26 | } dummy_read_mode; 27 | 28 | typedef struct { 29 | uint64_t read_buff_size; 30 | uint64_t write_buff_size; 31 | dummy_read_mode rd_mode; 32 | uint64_t expcap_bytes; 33 | uint64_t exanic_pkt_bytes; 34 | } dummy_args_t; 35 | 36 | 37 | NEW_IOSTREAM_DECLARE(dummy, dummy_args_t); 38 | 39 | #endif /* EXACTIO_DUMMY_H_ */ 40 | -------------------------------------------------------------------------------- /src/exactio/exactio_exanic.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018 All rights reserved. 3 | * See LICENSE.txt for full details. 4 | * 5 | * Created: 19 Jun 2017 6 | * Author: Matthew P. Grosvenor 7 | * Description: 8 | * Definition of an ExaNIC reader/writer stream using the exactio abstract 9 | * I/O interface. Some of the speed critical function a pulled into the header 10 | * file to (hopefully) aid compilation optimizations. 11 | */ 12 | 13 | 14 | #ifndef EXACTIO_EXA_H_ 15 | #define EXACTIO_EXA_H_ 16 | 17 | #include "exactio_stream.h" 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | #include "exactio_timing.h" 26 | 27 | 28 | typedef struct { 29 | char* interface_rx; 30 | char* interface_tx; 31 | bool promisc; 32 | bool kernel_bypass; 33 | } exa_args_t; 34 | 35 | NEW_IOSTREAM_DECLARE(exa,exa_args_t); 36 | 37 | /******************************************************************************/ 38 | /* Bring the below code up in to the header to make it easier for the compiler 39 | * to inline the functions properly. 40 | */ 41 | 42 | 43 | typedef struct exa_priv { 44 | int rx_port; 45 | int rx_dev_id; 46 | int tx_port; 47 | int tx_dev_id; 48 | char rx_dev[16]; 49 | char tx_dev[16]; 50 | 51 | exanic_t* tx_nic; 52 | exanic_t* rx_nic; 53 | exanic_tx_t *tx; 54 | exanic_rx_t *rx; 55 | 56 | char* rx_buffer; 57 | int64_t rx_len; 58 | uint32_t chunk_id; 59 | int more_rx_chunks; 60 | 61 | size_t max_tx_mtu; 62 | char* tx_buffer; 63 | int64_t tx_buffer_len; 64 | 65 | 66 | bool closed; 67 | } exa_priv_t; 68 | 69 | 70 | //eio_error_t exa_read_acquire(eio_stream_t* this, char** buffer, int64_t* len, int64_t* ts ); 71 | //eio_error_t exa_read_release(eio_stream_t* this, int64_t* ts); 72 | 73 | //Read operations 74 | static inline eio_error_t exa_read_acquire(eio_stream_t* this, char** buffer, int64_t* len, int64_t* ts ) 75 | { 76 | exa_priv_t* priv = IOSTREAM_GET_PRIVATE(this); 77 | ifassert(priv->closed){ 78 | return EIO_ECLOSED; 79 | } 80 | 81 | ifassert((ssize_t)priv->rx_buffer){ 82 | return EIO_ERELEASE; 83 | } 84 | 85 | struct rx_chunk_info info = {.frame_status =0}; 86 | priv->rx_len = exanic_receive_chunk_inplace_ex( 87 | priv->rx,&priv->rx_buffer,&priv->chunk_id,&priv->more_rx_chunks, 88 | &info); 89 | 90 | ssize_t frame_error = -(info.frame_status & EXANIC_RX_FRAME_ERROR_MASK); 91 | ifunlikely(priv->rx_len < 0){ 92 | frame_error = -EXANIC_RX_FRAME_SWOVFL; 93 | } 94 | 95 | ifunlikely(frame_error){ 96 | switch(frame_error){ 97 | //These are unrecoverable errors, so exit now. 98 | //Translate between ExaNIC error codes and EIO codes 99 | case -EXANIC_RX_FRAME_SWOVFL: return EIO_ESWOVFL; 100 | case -EXANIC_RX_FRAME_HWOVFL: return EIO_EHWOVFL; 101 | } 102 | } 103 | 104 | //This is a very likely path, but we want to optimise the data path 105 | ifunlikely(priv->rx_len == 0){ 106 | return EIO_ETRYAGAIN; 107 | } 108 | 109 | //The user doesn't want this chunk, and therefore the whole frame, skip it 110 | ifunlikely(buffer == NULL || len == NULL){ 111 | iflikely(priv->more_rx_chunks){ 112 | int err = exanic_receive_abort(priv->rx); 113 | ifunlikely( err == -EXANIC_RX_FRAME_SWOVFL){ 114 | return EIO_ESWOVFL; 115 | } 116 | } 117 | return EIO_ENONE; 118 | } 119 | 120 | 121 | iflikely((ssize_t)ts){ 122 | const exanic_cycles32_t ts32 = exanic_receive_chunk_timestamp(priv->rx, priv->chunk_id); 123 | const exanic_cycles_t ts64 = exanic_expand_timestamp(priv->rx_nic,ts32); 124 | // struct exanic_timespecps tsps; 125 | // exanic_cycles_to_timespecps(priv->rx_nic, ts64, &tsps); 126 | //These two timespecps structures look the same but come from different 127 | //names-spaces. Making the copy explicit here just to be safe 128 | //ts->tv_sec = tsps.tv_sec; 129 | *ts = ts64; //tsps.tv_psec; 130 | } 131 | 132 | //All good! Successful "read"! 133 | *buffer = priv->rx_buffer; 134 | *len = priv->rx_len; 135 | 136 | switch(frame_error){ 137 | case -EXANIC_RX_FRAME_CORRUPT: return EIO_EFRAG_CPT; 138 | case -EXANIC_RX_FRAME_ABORTED: return EIO_EFRAG_ABT; 139 | default: 140 | iflikely(priv->more_rx_chunks) return EIO_EFRAG_MOR; 141 | return EIO_ENONE; 142 | } 143 | } 144 | 145 | //Convert an exanic timestamp into cycles 146 | static inline eio_error_t exa_rxcycles_to_timespec(eio_stream_t* this, exanic_cycles_t cycles, struct timespec* ts ) 147 | { 148 | exa_priv_t* priv = IOSTREAM_GET_PRIVATE(this); 149 | 150 | iflikely(ts){ 151 | exanic_cycles_to_timespec(priv->rx_nic, cycles, ts); 152 | } 153 | 154 | return EIO_ENONE; 155 | } 156 | 157 | static inline eio_error_t exa_rxcycles_to_timespecps(eio_stream_t* this, exanic_cycles_t cycles, struct exanic_timespecps* ts ) 158 | { 159 | exa_priv_t* priv = IOSTREAM_GET_PRIVATE(this); 160 | 161 | iflikely(ts){ 162 | exanic_cycles_to_timespecps(priv->rx_nic, cycles, ts); 163 | } 164 | 165 | return EIO_ENONE; 166 | } 167 | 168 | 169 | 170 | 171 | static inline eio_error_t exa_read_release(eio_stream_t* this, int64_t* ts) 172 | { 173 | eio_error_t result = EIO_ENONE; 174 | exa_priv_t* priv = IOSTREAM_GET_PRIVATE(this); 175 | 176 | 177 | ifassert(!priv->rx_buffer){ 178 | return EIO_ERELEASE; 179 | } 180 | 181 | ifunlikely(exanic_receive_chunk_recheck(priv->rx, priv->chunk_id) == 0){ 182 | result = EIO_ESWOVFL; 183 | } 184 | 185 | priv->rx_buffer = NULL; 186 | (void)ts; 187 | //eio_nowns(ts); 188 | 189 | //Nothing to do here; 190 | return result; 191 | } 192 | 193 | 194 | #endif /* EXACTIO_EXA_H_ */ 195 | -------------------------------------------------------------------------------- /src/exactio/exactio_file.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018 All rights reserved. 3 | * See LICENSE.txt for full details. 4 | * 5 | * Created: 19 Jun 2017 6 | * Author: Matthew P. Grosvenor 7 | * Description: 8 | * Definition of the file read/write interface using the exactio abstract I/O 9 | * interface. 10 | */ 11 | 12 | #ifndef EXACTIO_FILE_H_ 13 | #define EXACTIO_FILE_H_ 14 | 15 | #include "exactio_stream.h" 16 | 17 | typedef struct { 18 | char* filename; 19 | uint64_t read_buff_size; 20 | uint64_t write_buff_size; 21 | uint64_t on_mod; 22 | } file_args_t; 23 | 24 | NEW_IOSTREAM_DECLARE(file, file_args_t); 25 | 26 | #endif /* EXACTIO_FILE_H_ */ 27 | -------------------------------------------------------------------------------- /src/exactio/exactio_stream.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018 All rights reserved. 3 | * See LICENSE.txt for full details. 4 | * 5 | * Created: 17 Jul 2017 6 | * Author: Matthew P. Grosvenor 7 | * Description: 8 | * Definition of the exactio abstract straeming I/O interface. This is loosely 9 | * based on code from CAMIO (https://github.com/mgrosvenor/camio2). It provides 10 | * a fast way to interface to a number of underlying transports using a common 11 | * API. A C++ style interface is implemented directly in C, similar to Linux 12 | * kernel abstract interfaces. 13 | */ 14 | 15 | 16 | #ifndef SRC_EXACTIO_EXACTIO_STREAM_H_ 17 | #define SRC_EXACTIO_EXACTIO_STREAM_H_ 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include 25 | #include "../data_structs/timespecps.h" 26 | 27 | //This compile out checks that aren't strictly necessary 28 | #ifdef NOIFASSERT 29 | #define ifassert(x) if(0 && (x)) 30 | #else 31 | #define ifassert(x) ifunlikely((ssize_t)(x)) 32 | #endif 33 | 34 | 35 | 36 | typedef struct exactio_stream_s eio_stream_t; 37 | 38 | typedef enum { 39 | EIO_ENONE = 0, 40 | EIO_EUNSPEC, //Unspecified error 41 | EIO_ETRYAGAIN, //No data available, try again 42 | EIO_ECLOSED, //The underlying data source has closed 43 | EIO_EEOF, //End of file has been reached 44 | EIO_ENOTIMPL, //This function is not implemented 45 | EIO_ETOOBIG, //Returned buffer len is too long 46 | EIO_ERELEASE, //Release the buffer first before trying acquire again 47 | EIO_EACQUIRE, //Acquire the buffer first before trying release again 48 | EIO_ENOMEM, //No memory to back this 49 | 50 | //These error codes only apply to fragment based transports. 51 | EIO_EFRAG_MOR, //More fragments to come 52 | EIO_EFRAG_EOF, //End of fragment sequence 53 | EIO_EFRAG_ABT, //Fragment aborted by sender 54 | EIO_EFRAG_CPT, //Fragment corrupt (e.g. csum failure) 55 | 56 | EIO_EHWOVFL, //Hardware overflow (e.g. insufficient PCIe bandwidth) 57 | EIO_ESWOVFL, //Software overflow (e.g. software not keeping up) 58 | 59 | EIO_EINVALID, //Something was done wrong 60 | 61 | } eio_error_t; 62 | 63 | /** 64 | * Nice simple generic read/write I/O abstraction 65 | */ 66 | typedef struct exactio_stream_interface_s{ 67 | 68 | void (*destroy)(eio_stream_t* this); 69 | 70 | //Read operations 71 | //---------------- 72 | eio_error_t (*read_acquire)(eio_stream_t* this, char** buffer, int64_t* len, int64_t* ts ); 73 | eio_error_t (*read_release)(eio_stream_t* this,int64_t* ts); 74 | 75 | //Write operations 76 | //---------------- 77 | //for write_acquire len can be supplied as a hint. If len is 0, the MTU will be used 78 | //for write_release len is zero, the frame the data is not committed 79 | eio_error_t (*write_acquire)(eio_stream_t* this, char** buffer, int64_t* len, int64_t* ts ); 80 | eio_error_t (*write_release)(eio_stream_t* this, int64_t len, int64_t* ts ); 81 | 82 | 83 | } exactio_stream_interface_t; 84 | 85 | /** 86 | * All streams must implement this. Streams will use the macros provided to overload 87 | * this structure with and include a private data structure. 88 | */ 89 | struct exactio_stream_s { 90 | 91 | /** 92 | * vtable that holds the function pointers, usually this would be a pointer to a constant, but saving 6x8bytes seems a 93 | * little silly when it will cost a pointer dereference on the critical 94 | * path. 95 | */ 96 | exactio_stream_interface_t vtable; 97 | 98 | int fd; 99 | 100 | }; 101 | 102 | 103 | #define IOSTREAM_GET_PRIVATE(THIS) ( (void*)(THIS + 1)) 104 | 105 | #define NEW_IOSTREAM(name, out, args)\ 106 | new_##name##_stream(out, args) 107 | 108 | #define NEW_IOSTREAM_DECLARE(NAME, ARGT)\ 109 | int new_##NAME##_stream(eio_stream_t** out, ARGT* args) 110 | 111 | #define NEW_IOSTREAM_DEFINE(NAME, ARGT, PRIVATE_TYPE) \ 112 | static const exactio_stream_interface_t NAME##_stream_interface = {\ 113 | .read_acquire = NAME##_read_acquire,\ 114 | .read_release = NAME##_read_release,\ 115 | .write_acquire = NAME##_write_acquire,\ 116 | .write_release = NAME##_write_release,\ 117 | .destroy = NAME##_destroy,\ 118 | };\ 119 | \ 120 | NEW_IOSTREAM_DECLARE(NAME, ARGT)\ 121 | {\ 122 | eio_stream_t* result = (eio_stream_t*)calloc(1,sizeof(eio_stream_t) + sizeof(PRIVATE_TYPE));\ 123 | if(!result) return EIO_ENOMEM;\ 124 | result->vtable = NAME##_stream_interface;\ 125 | result->fd = -1;\ 126 | *out = result;\ 127 | int err = NAME##_construct(result, args);\ 128 | if(err){ free(result); *out = NULL;}\ 129 | return err;\ 130 | } 131 | 132 | 133 | #endif /* SRC_EXACTIO_EXACTIO_STREAM_H_ */ 134 | -------------------------------------------------------------------------------- /src/exactio/exactio_timing.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018 All rights reserved. 3 | * See LICENSE.txt for full details. 4 | * 5 | * Created: 17 Jul 2017 6 | * Author: Matthew P. Grosvenor 7 | * Description: 8 | * Implementation of utilities for operating with time (quickly) 9 | */ 10 | 11 | 12 | 13 | 14 | #include 15 | #include 16 | 17 | #include "exactio_timing.h" 18 | #include "exactio.h" 19 | 20 | 21 | //static inline uint64_t rdtscp( uint32_t *aux ) 22 | //{ 23 | // uint64_t rax,rdx; 24 | // __asm__ __volatile__( "rdtscp\n" : "=a" (rax), "=d" (rdx), "=c" (aux) : : ); 25 | // return (rdx << 32) + rax; 26 | //} 27 | 28 | void eio_nowns(int64_t* ts) 29 | { 30 | 31 | 32 | iflikely(!ts){ 33 | return; 34 | } 35 | 36 | struct timespec now; 37 | clock_gettime(CLOCK_REALTIME, &now ); 38 | *ts = now.tv_sec * 1000 * 1000 * 1000 + now.tv_nsec; 39 | 40 | // uint32_t aux; 41 | // ts->tv_sec = 0; 42 | // ts->tv_psec = rdtscp(&aux); // * 1000 / 3.5; 43 | } 44 | 45 | 46 | double eio_tspstonsf(timespecps_t* ts) 47 | { 48 | if(!ts){ 49 | return NAN; 50 | } 51 | 52 | 53 | const double secs = ts->tv_sec; 54 | const double psec = ts->tv_psec; 55 | 56 | return secs * 1000.0 * 1000.0 * 1000.0 + psec / 1000.0; 57 | } 58 | 59 | int64_t eio_tspstonsll(timespecps_t* ts) 60 | { 61 | 62 | if(!ts){ 63 | return ~0; 64 | } 65 | 66 | const int64_t secs = ts->tv_sec; 67 | const int64_t psec = ts->tv_psec; 68 | //return (double)psec/3.5; 69 | 70 | return secs * 1000 * 1000 * 1000 + psec / 1000; 71 | } 72 | 73 | -------------------------------------------------------------------------------- /src/exactio/exactio_timing.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018 All rights reserved. 3 | * See LICENSE.txt for full details. 4 | * 5 | * Created: 17 Jul 2017 6 | * Author: Matthew P. Grosvenor 7 | * Description: 8 | * Definition of utilities for operating with time (quickly) 9 | */ 10 | 11 | #ifndef SRC_EXACTIO_EXACTIO_TIMING_H_ 12 | #define SRC_EXACTIO_EXACTIO_TIMING_H_ 13 | 14 | #include "../data_structs/timespecps.h" 15 | 16 | 17 | //Some handy time related utils 18 | //Get the current time in timespecps format 19 | void eio_nowns(int64_t* ts); 20 | 21 | //Convert a timespec ps to int64 nanoseconds 22 | //int64_t eio_tspstonsll(timespecps_t* ts); 23 | 24 | //Convert a timespec ps to double double 25 | //double eio_tspstonsf(timespecps_t* ts); 26 | 27 | #endif /* SRC_EXACTIO_EXACTIO_TIMING_H_ */ 28 | -------------------------------------------------------------------------------- /src/utils.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 All rights reserved. 3 | * See LICENSE.txt for full details. 4 | * 5 | * Created: 6 Apr. 2018 6 | * Author: Matthew P. Grosvenor 7 | * Description: Application wide useful utility functions 8 | */ 9 | 10 | 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include 18 | #include 19 | 20 | #include "data_structs/expcap.h" 21 | 22 | #include "utils.h" 23 | 24 | 25 | 26 | 27 | int64_t time_now_ns() 28 | { 29 | struct timespec now = {0}; 30 | clock_gettime(CLOCK_REALTIME, &now); 31 | return now.tv_sec * 1000ULL * 1000 * 1000 + now.tv_nsec; 32 | } 33 | 34 | 35 | void init_dummy_data(char* dummy_data, int len) 36 | { 37 | const char* dummy_text = "EXABLAZEEXACTCAPPADDING PACKET "; 38 | const int64_t dummy_text_len = strlen (dummy_text); 39 | memset (dummy_data, 0xFF, 16); 40 | for (int i = 16; i < len; i++) 41 | { 42 | dummy_data[i] = dummy_text[(i - 16) % dummy_text_len]; 43 | } 44 | 45 | } 46 | 47 | 48 | void print_flags(uint8_t flags) 49 | { 50 | if(flags & EXPCAP_FLAG_ABRT) printf("ABRT "); 51 | if(flags & EXPCAP_FLAG_CRPT) printf("CRPT "); 52 | if(flags & EXPCAP_FLAG_HASCRC) printf("HCRC "); 53 | if(flags & EXPCAP_FLAG_HWOVFL) printf("HWOVF "); 54 | if(flags & EXPCAP_FLAG_SWOVFL) printf("SWOVF "); 55 | if(flags & EXPCAP_FLAG_TRNC) printf("TRNC "); 56 | } 57 | 58 | 59 | int parse_device_id(const char *str, int* dev_number) 60 | { 61 | char* p; 62 | 63 | p = strchr(str, ':'); 64 | if (p == NULL){ 65 | return -1; 66 | } 67 | 68 | if ((p - str) >= 16){ 69 | return -1; 70 | } 71 | 72 | *dev_number = strtol(p - 1, NULL, 10); 73 | 74 | return 0; 75 | } 76 | 77 | 78 | 79 | //Parses a string of the format ":" 80 | int parse_device_port(const char *str, char *device, int *port_number) 81 | { 82 | char *p, *q; 83 | 84 | p = strchr(str, ':'); 85 | if (p == NULL){ 86 | return -1; 87 | } 88 | 89 | if ((p - str) >= 16){ 90 | return -1; 91 | } 92 | 93 | strncpy(device, str, p - str); 94 | device[p - str] = '\0'; 95 | *port_number = strtol(p + 1, &q, 10); 96 | if (*(p + 1) == '\0' || *q != '\0'){ 97 | // strtol failed 98 | return -1; 99 | } 100 | 101 | return 0; 102 | } 103 | 104 | 105 | int parse_device (const char* interface, 106 | char device[16], int* dev_number, int *port_number) 107 | { 108 | if(!exanic_find_port_by_interface_name (interface, device, 16, port_number)) 109 | { 110 | parse_device_id(device, dev_number); 111 | return 0; 112 | } 113 | 114 | if(!parse_device_port (interface, device, port_number)) 115 | { 116 | parse_device_id(device, dev_number); 117 | return 0; 118 | 119 | } 120 | 121 | return 1; 122 | 123 | 124 | } 125 | 126 | 127 | 128 | /* This is a dumb way to do this, but who cares? */ 129 | static int get_digitsll (int64_t num) 130 | { 131 | int i = 1; 132 | for (; num > 0; i++) 133 | { 134 | num = num / 10; 135 | } 136 | 137 | return i; 138 | } 139 | 140 | int max_digitsll (int64_t a, int64_t b, int64_t c, int64_t d, int64_t e, 141 | int64_t f, int64_t g, int64_t h, int64_t i, int64_t j) 142 | { 143 | int result = 0; 144 | result = MAX (result, get_digitsll (a)); 145 | result = MAX (result, get_digitsll (b)); 146 | result = MAX (result, get_digitsll (c)); 147 | result = MAX (result, get_digitsll (d)); 148 | result = MAX (result, get_digitsll (e)); 149 | result = MAX (result, get_digitsll (f)); 150 | result = MAX (result, get_digitsll (g)); 151 | result = MAX (result, get_digitsll (h)); 152 | result = MAX (result, get_digitsll (i)); 153 | result = MAX (result, get_digitsll (j)); 154 | 155 | return result; 156 | } 157 | 158 | /* This is a dumb way to do this, but who cares? */ 159 | 160 | static int get_digitsf (double num) 161 | { 162 | int i = 1; 163 | for (; num > 1; i++) 164 | { 165 | num = num / 10; 166 | } 167 | 168 | return i; 169 | } 170 | 171 | int max_digitsf (double a, double b, double c, double d, double e, double f, 172 | double g, double h, double i, double j) 173 | { 174 | int result = 0; 175 | result = MAX (result, get_digitsf (a)); 176 | result = MAX (result, get_digitsf (b)); 177 | result = MAX (result, get_digitsf (c)); 178 | result = MAX (result, get_digitsf (d)); 179 | result = MAX (result, get_digitsf (e)); 180 | result = MAX (result, get_digitsf (f)); 181 | result = MAX (result, get_digitsf (g)); 182 | result = MAX (result, get_digitsf (h)); 183 | result = MAX (result, get_digitsf (i)); 184 | result = MAX (result, get_digitsf (j)); 185 | return result; 186 | } 187 | 188 | -------------------------------------------------------------------------------- /src/utils.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 All rights reserved. 3 | * See LICENSE.txt for full details. 4 | * 5 | * Created: 6 Apr. 2018 6 | * Author: Matthew P. Grosvenor 7 | * Description: Application wide useful utility functions and macros 8 | */ 9 | 10 | 11 | #ifndef SRC_UTILS_H_ 12 | #define SRC_UTILS_H_ 13 | 14 | #include 15 | #include 16 | 17 | #if __BYTE_ORDER == __LITTLE_ENDIAN 18 | # define htonll(x) __bswap_64(x) 19 | # define ntohll(x) __bswap_64(x) 20 | #elif __BYTE_ORDER == __BIG_ENDIAN 21 | # define htonll(x) (x) 22 | # define ntohll(x) (x) 23 | #else 24 | # error Unknown byte order 25 | #endif 26 | 27 | 28 | #define PKT(hdr) ((char*)(hdr+1)) 29 | #define PKT_OFF(hdr, off) (PKT(hdr) + off) 30 | 31 | int64_t time_now_ns(); 32 | 33 | void init_dummy_data(char* dummy_data, int len); 34 | 35 | void print_flags(uint8_t flags); 36 | 37 | int parse_device (const char* interface, 38 | char device[16], int* dev_number, int *port_number); 39 | 40 | 41 | int max_digitsf (double a, double b, double c, double d, double e, double f, 42 | double g, double h, double i, double j); 43 | int max_digitsll (int64_t a, int64_t b, int64_t c, int64_t d, int64_t e, 44 | int64_t f, int64_t g, int64_t h, int64_t i, int64_t j); 45 | 46 | #endif /* SRC_UTILS_H_ */ 47 | -------------------------------------------------------------------------------- /tools/checksum.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "data_structs/pseudo_ip_header.h" 4 | 5 | const uint32_t crc32_tab[] = { 6 | 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 7 | 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 8 | 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2, 9 | 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, 10 | 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 11 | 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, 12 | 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c, 13 | 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, 14 | 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 15 | 0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 16 | 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106, 17 | 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, 18 | 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 19 | 0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 20 | 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, 21 | 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, 22 | 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 23 | 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, 24 | 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, 25 | 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, 26 | 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 27 | 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 28 | 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84, 29 | 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, 30 | 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 31 | 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 32 | 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e, 33 | 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, 34 | 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 35 | 0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 36 | 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, 37 | 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, 38 | 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, 39 | 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 40 | 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, 41 | 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, 42 | 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 43 | 0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 44 | 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, 45 | 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, 46 | 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 47 | 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 48 | 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d 49 | }; 50 | 51 | uint32_t crc32(const void* buf, size_t size) 52 | { 53 | const uint8_t* p = buf; 54 | uint32_t crc; 55 | 56 | crc = ~0U; 57 | while (size--) 58 | crc = crc32_tab[(crc ^ *p++) & 0xFF] ^ (crc >> 8); 59 | return crc ^ ~0U; 60 | } 61 | 62 | uint16_t csum(uint8_t* buf, size_t len, uint16_t initial) 63 | { 64 | uint32_t sum = (uint32_t)(initial); 65 | const uint16_t *u16 = (const uint16_t*)buf; 66 | 67 | while (len >= (sizeof(*u16) * 4)) { 68 | sum += u16[0]; 69 | sum += u16[1]; 70 | sum += u16[2]; 71 | sum += u16[3]; 72 | len -= sizeof(*u16) * 4; 73 | u16 += 4; 74 | } 75 | while (len >= sizeof(*u16)) { 76 | sum += *u16; 77 | len -= sizeof(*u16); 78 | u16 += 1; 79 | } 80 | 81 | /* if length is in odd bytes */ 82 | if (len == 1) 83 | sum += *((const uint8_t*)u16); 84 | 85 | while(sum>>16) 86 | sum = (sum & 0xFFFF) + (sum>>16); 87 | return (uint16_t)~sum; 88 | } 89 | 90 | /* Format a pseudo IP header, used for UDP/TCP checksum calculation */ 91 | static inline void get_pseudo_iphdr(struct iphdr* ip_hdr, uint16_t hdr_len, struct pseudo_iphdr* hdro) 92 | { 93 | hdro->saddr = ip_hdr->saddr; 94 | hdro->daddr = ip_hdr->daddr; 95 | hdro->zero = 0; 96 | hdro->protocol = ip_hdr->protocol; 97 | hdro->len = hdr_len; 98 | } 99 | 100 | uint16_t calc_l4_csum(uint8_t* buf, struct iphdr* ip_hdr, uint16_t len) 101 | { 102 | struct pseudo_iphdr p_hdr; 103 | uint16_t len_be = htobe16(len); 104 | get_pseudo_iphdr(ip_hdr, len_be, &p_hdr); 105 | uint16_t pseudo_csum = csum((uint8_t*)&p_hdr, sizeof(struct pseudo_iphdr), 0); 106 | return csum(buf, len, ~pseudo_csum); 107 | } 108 | -------------------------------------------------------------------------------- /tools/data_structs/buff.h: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | #include 5 | #include "data_structs/pcap-structures.h" 6 | 7 | #define BUFF_SIZE (128 * 1024 * 1024) /* 128MB */ 8 | #define MAX_FILENAME 2048 9 | 10 | #define BUFF_TRY(x) \ 11 | do { \ 12 | buff_error_t err = (x); \ 13 | if (err != BUFF_ENONE) { \ 14 | ch_log_warn("%s failed at %s:%d\n", #x, __FILE__, __LINE__); \ 15 | ch_log_fatal("Reason: %s\n", buff_strerror(err)); \ 16 | } \ 17 | } while(0) \ 18 | 19 | typedef struct { 20 | char* filename; 21 | char* file_header; 22 | size_t header_size; 23 | char* data; 24 | int eof; 25 | int fd; 26 | bool conserve_fds; 27 | uint64_t filesize; 28 | int64_t max_filesize; 29 | uint64_t offset; 30 | uint64_t pkt_idx; 31 | int file_seg; 32 | int file_dup; 33 | uint64_t file_bytes_written; 34 | bool read_only; 35 | bool allow_duplicates; 36 | } buff_t; 37 | 38 | typedef enum { 39 | BUFF_ENONE = 0, 40 | BUFF_EALLOC, // Failed to allocate memory for a buff_t 41 | BUFF_EOPEN, // Failed to open a file for read/write 42 | BUFF_EWRITE, // Failed to write out buff_t data 43 | BUFF_EMMAP, // Failed to map a buff_t from a file 44 | BUFF_ESTAT, // Failed to stat a file 45 | BUFF_ECOPY, // Failed to copy bytes to a buff_t 46 | BUFF_EOVERFLOW, // Buffer offset is greater than the allocated buffer size. 47 | BUFF_EREADONLY, // Attempting to write to a read-only buffer. 48 | BUFF_EBADHEADER, // Failed to open file as buff_t, due to a badly formed file header. 49 | BUFF_ECLOSE // Failed to close buff_t. 50 | } buff_error_t; 51 | 52 | /* Allocate and initialize a buff_t. */ 53 | buff_error_t buff_init(char* filename, int64_t max_filesize, bool conserve_fds, bool allow_duplicates, buff_t** buffo); 54 | 55 | /* Read a file into a buff_t */ 56 | buff_error_t buff_init_from_file(buff_t** buff, char* filename, size_t header_size); 57 | 58 | /* Create a new pcap file header within a buff_t */ 59 | buff_error_t buff_new_file(buff_t* buff); 60 | 61 | /* Copy bytes to buff, flushing to disk if required */ 62 | buff_error_t buff_copy_bytes(buff_t* buff, void* bytes, uint64_t len); 63 | 64 | /* Write out the contents of a buff_t to disk */ 65 | buff_error_t buff_flush_to_disk(buff_t* wr_buff); 66 | 67 | /* Get the amount of bytes left available in the buffer */ 68 | buff_error_t buff_remaining(buff_t* buff, int64_t* remaining); 69 | 70 | /* Get the amount of bytes left in the current segment */ 71 | int64_t buff_seg_remaining(buff_t* buff); 72 | 73 | /* Get full buff filename */ 74 | void buff_get_full_filename(buff_t* buff, char* full_filename, size_t len); 75 | 76 | /* Write file header */ 77 | buff_error_t buff_write_file_header(buff_t *buff); 78 | 79 | /* Close and resources associated with this buff_t */ 80 | buff_error_t buff_close(buff_t* buff); 81 | 82 | /* Translate an error value to a string */ 83 | const char* buff_strerror(buff_error_t err); 84 | -------------------------------------------------------------------------------- /tools/data_structs/fusion_hpt.h: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #define be40toh(x) (be64toh(x) >> 24) 4 | 5 | typedef struct fusion_hpt_trailer { 6 | uint8_t device_id; 7 | uint8_t port; 8 | uint64_t seconds_since_epoch : 32; 9 | uint64_t frac_seconds : 40; 10 | uint8_t __reserved; 11 | uint32_t new_fcs; 12 | } __attribute__((packed)) fusion_hpt_trailer_t; 13 | -------------------------------------------------------------------------------- /tools/data_structs/pcap_buff.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include 8 | #include 9 | #include "pcap_buff.h" 10 | 11 | buff_error_t pcap_buff_init(char* filename, int64_t snaplen, int64_t max_filesize, bool usec, 12 | bool conserve_fds, bool allow_duplicates, pcap_buff_t* pcap_buffo) 13 | { 14 | buff_t* buff = NULL; 15 | BUFF_TRY(buff_init(filename, max_filesize, conserve_fds, allow_duplicates, &buff)); 16 | 17 | pcap_buffo->_buff = buff; 18 | pcap_buffo->snaplen = snaplen; 19 | pcap_buffo->usec = usec; 20 | 21 | /* Generate the output file header */ 22 | pcap_file_header_t* hdr = (pcap_file_header_t*)malloc(sizeof(pcap_file_header_t)); 23 | if(!hdr){ 24 | ch_log_fatal("Failed to allocate memory for file header!\n"); 25 | } 26 | 27 | buff->file_header = (char*)hdr; 28 | buff->header_size = sizeof(pcap_file_header_t); 29 | 30 | hdr->magic = usec ? TCPDUMP_MAGIC: NSEC_TCPDUMP_MAGIC; 31 | hdr->version_major = PCAP_VERSION_MAJOR; 32 | hdr->version_minor = PCAP_VERSION_MINOR; 33 | hdr->thiszone = 0; 34 | hdr->sigfigs = 0; /* 9? libpcap always writes 0 */ 35 | hdr->snaplen = snaplen + (int)sizeof(expcap_pktftr_t); 36 | hdr->linktype = DLT_EN10MB; 37 | 38 | BUFF_TRY(buff_new_file(buff)); 39 | 40 | return BUFF_ENONE; 41 | } 42 | 43 | buff_error_t pcap_buff_from_file(pcap_buff_t* pcap_buff, char* filename, bool is_expcap) 44 | { 45 | pcap_buff->expcap = is_expcap; 46 | BUFF_TRY(buff_init_from_file(&pcap_buff->_buff, filename, sizeof(pcap_file_header_t))); 47 | return BUFF_ENONE; 48 | } 49 | 50 | pkt_info_t pcap_buff_get_info(pcap_buff_t* pcap_buff) 51 | { 52 | if(pcap_buff->hdr == NULL){ 53 | pcap_buff->hdr = (pcap_pkthdr_t*)(pcap_buff->_buff->data + sizeof(pcap_file_header_t)); 54 | pcap_buff->idx = 0; 55 | } 56 | 57 | pcap_buff->pkt = (char*)(pcap_buff->hdr + 1); 58 | 59 | /* Check if we've overflowed */ 60 | buff_t* buff = pcap_buff->_buff; 61 | int64_t offset = (char*)pcap_buff->hdr - buff->data; 62 | buff->eof = offset >= buff->filesize; 63 | if(buff->eof){ 64 | return PKT_EOF; 65 | } 66 | 67 | if(pcap_buff->hdr->len == 0){ 68 | return PKT_PADDING; 69 | } 70 | 71 | if(pcap_buff->hdr->caplen < 64){ 72 | return PKT_RUNT; 73 | } 74 | 75 | pcap_file_header_t* hdr = (pcap_file_header_t*)buff->file_header; 76 | if(pcap_buff->hdr->len > hdr->snaplen){ 77 | return PKT_OVER_SNAPLEN; 78 | } 79 | 80 | if(pcap_buff->hdr->len > pcap_buff->hdr->caplen){ 81 | return PKT_SNAPPED; 82 | } 83 | 84 | if(pcap_buff->expcap){ 85 | pcap_buff->ftr = (expcap_pktftr_t*)(pcap_buff->pkt + pcap_buff->hdr->caplen - sizeof(expcap_pktftr_t)); 86 | 87 | if(pcap_buff->ftr->foot.extra.dropped > 0){ 88 | ch_log_warn("%li packets were droped before this one\n", 89 | pcap_buff->ftr->foot.extra.dropped); 90 | } 91 | 92 | if( (pcap_buff->ftr->flags & EXPCAP_FLAG_ABRT) || 93 | (pcap_buff->ftr->flags & EXPCAP_FLAG_CRPT) || 94 | (pcap_buff->ftr->flags & EXPCAP_FLAG_SWOVFL)){ 95 | 96 | return PKT_ERROR; 97 | } 98 | } else { 99 | pcap_buff->ftr = NULL; 100 | } 101 | 102 | return PKT_OK; 103 | } 104 | 105 | pkt_info_t pcap_buff_next_packet(pcap_buff_t* pcap_buff) 106 | { 107 | pcap_pkthdr_t* curr_hdr = pcap_buff->hdr; 108 | 109 | if(curr_hdr == NULL){ 110 | pcap_buff->hdr = (pcap_pkthdr_t*)(pcap_buff->_buff->data + sizeof(pcap_file_header_t)); 111 | pcap_buff->idx = 0; 112 | } else { 113 | const int64_t curr_cap_len = curr_hdr->caplen; 114 | pcap_pkthdr_t* next_hdr = (pcap_pkthdr_t*)((char*)(curr_hdr+1) + curr_cap_len); 115 | pcap_buff->hdr = next_hdr; 116 | pcap_buff->idx++; 117 | } 118 | 119 | return pcap_buff_get_info(pcap_buff); 120 | } 121 | 122 | bool pcap_buff_eof(pcap_buff_t* pcap_buff) 123 | { 124 | buff_t* buff = pcap_buff->_buff; 125 | return buff->eof; 126 | } 127 | 128 | char* pcap_buff_get_filename(pcap_buff_t* pcap_buff) 129 | { 130 | return pcap_buff->_buff->filename; 131 | } 132 | 133 | buff_error_t pcap_buff_flush_to_disk(pcap_buff_t* pcap_buff){ 134 | return buff_flush_to_disk(pcap_buff->_buff); 135 | } 136 | 137 | buff_error_t pcap_buff_write(pcap_buff_t* pcap_buff, pcap_pkthdr_t* hdr, char* data, size_t data_len, expcap_pktftr_t* ftr) 138 | { 139 | buff_t* buff = pcap_buff->_buff; 140 | const int64_t pcap_record_bytes = sizeof(pcap_pkthdr_t) + data_len + (ftr ? sizeof(expcap_pktftr_t) : 0); 141 | 142 | // there isn't enough space in the buff_t. must flush to disk. 143 | int64_t bytes_remaining; 144 | BUFF_TRY(buff_remaining(buff, &bytes_remaining)); 145 | const bool buff_full = bytes_remaining < pcap_record_bytes; 146 | 147 | if(buff_full){ 148 | BUFF_TRY(buff_flush_to_disk(buff)); 149 | } 150 | 151 | if(buff->max_filesize > 0){ 152 | /* tcpdump allows one packet to be written past the filesize limit. */ 153 | if(buff_seg_remaining(buff) < 0){ 154 | BUFF_TRY(buff_flush_to_disk(buff)); 155 | buff->file_seg++; 156 | buff_new_file(buff); 157 | } 158 | } 159 | 160 | /* Flush the buffer if we need to */ 161 | if(ftr){ 162 | hdr->caplen += sizeof(expcap_pktftr_t); 163 | } 164 | 165 | ch_log_debug1("header bytes=%li\n", sizeof(pcap_pkthdr_t)); 166 | ch_log_debug1("footer bytes=%li\n", sizeof(expcap_pktftr_t)); 167 | ch_log_debug1("max pcap_record_bytes=%li\n", pcap_record_bytes); 168 | 169 | BUFF_TRY(buff_copy_bytes(buff, hdr, sizeof(pcap_pkthdr_t))); 170 | BUFF_TRY(buff_copy_bytes(buff, data, data_len)); 171 | 172 | if(ftr){ 173 | BUFF_TRY(buff_copy_bytes(buff, ftr, sizeof(expcap_pktftr_t))); 174 | } 175 | 176 | return BUFF_ENONE; 177 | } 178 | 179 | buff_error_t pcap_buff_close(pcap_buff_t* pcap_buff){ 180 | return buff_close(pcap_buff->_buff); 181 | } 182 | 183 | static const char* pkt_infolist[] = { 184 | "Packet OK.", // PKT_OK 185 | "Packet is expcap padding.", // PKT_PADDING 186 | "Packet is a runt (len < 64B).", // PKT_RUNT 187 | "Packet contains an error.", // PKT_ERROR 188 | "No more packets left in this buffer. ", // PKT_EOF 189 | "Packet length exceeds the pcap file's snaplen.", // PKT_OVER_SNAPLEN 190 | "Packet data length is less than the length on disk.", // PKT_BAD_LEN 191 | "Packet has been snapped to less than it's length on the wire." // PKT_SNAPPED 192 | }; 193 | 194 | const char* pcap_buff_strinfo(pkt_info_t info){ 195 | return pkt_infolist[info]; 196 | } 197 | -------------------------------------------------------------------------------- /tools/data_structs/pcap_buff.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include "../../src/data_structs/expcap.h" 3 | #include "../../src/data_structs/pcap-structures.h" 4 | #include "buff.h" 5 | 6 | typedef struct { 7 | pcap_pkthdr_t* hdr; // header 8 | char* pkt; // packet data 9 | expcap_pktftr_t* ftr; // footer 10 | uint64_t idx; // index 11 | int64_t snaplen; 12 | int64_t max_filesize; 13 | bool usec; 14 | bool expcap; 15 | buff_t* _buff; // _private buff_t. 16 | } pcap_buff_t; 17 | 18 | typedef enum { 19 | PKT_OK = 0, 20 | PKT_PADDING, 21 | PKT_RUNT, 22 | PKT_ERROR, 23 | PKT_EOF, 24 | PKT_OVER_SNAPLEN, 25 | PKT_SNAPPED 26 | } pkt_info_t; 27 | 28 | /* Initialize a packet buffer for writing. */ 29 | buff_error_t pcap_buff_init(char* filename, int64_t snaplen, int64_t max_filesize, bool usec, 30 | bool conserve_fds, bool allow_duplicates, pcap_buff_t* pcap_buffo); 31 | 32 | /* Read in a pcap from disk. */ 33 | /* The underlying buff is read only */ 34 | buff_error_t pcap_buff_from_file(pcap_buff_t* pcap_buff, char* filename, bool is_expcap); 35 | 36 | /* Return information about the current packet */ 37 | pkt_info_t pcap_buff_get_info(pcap_buff_t* pcap_buff); 38 | 39 | /* Adjust hdr, data, idx, ftr to point to the next packet and return packet information */ 40 | pkt_info_t pcap_buff_next_packet(pcap_buff_t* pcap_buff); 41 | 42 | /* Get the filename used by the buff_t. */ 43 | char* pcap_buff_get_filename(pcap_buff_t* pcap_buff); 44 | 45 | /* Write a full packet (headers + data + footer). Footer can be NULL if not used. */ 46 | /* Header caplen will be adjusted to account for the presence of a footer.. */ 47 | buff_error_t pcap_buff_write(pcap_buff_t* pcap_buff, pcap_pkthdr_t* hdr, char* data, size_t data_len, expcap_pktftr_t* ftr); 48 | 49 | /* Flushes _buff to disk. */ 50 | buff_error_t pcap_buff_flush_to_disk(pcap_buff_t* pcap_buff); 51 | 52 | /* Check if _buff is at eof. */ 53 | bool pcap_buff_eof(pcap_buff_t* pcap_buff); 54 | 55 | /* Close and release resources associated with this pcap_buff */ 56 | buff_error_t pcap_buff_close(pcap_buff_t* pcap_buff); 57 | 58 | /* Translate an info value to a string */ 59 | const char* pcap_buff_strinfo(pkt_info_t info); 60 | -------------------------------------------------------------------------------- /tools/data_structs/pseudo_ip_header.h: -------------------------------------------------------------------------------- 1 | /* Pseudo IP header, used in calculating UDP/TCP checksums */ 2 | struct pseudo_iphdr { 3 | uint32_t saddr; 4 | uint32_t daddr; 5 | uint8_t zero; 6 | uint8_t protocol; 7 | uint16_t len; 8 | } __attribute__((packed)); 9 | -------------------------------------------------------------------------------- /tools/data_structs/timespecps.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017,2018 All rights reserved. 3 | * See LICENSE.txt for full details. 4 | * 5 | * Created: 13 July 2017 6 | * Author: Matthew P. Grosvenor 7 | * Description: 8 | * A local definition of a timespec like structure for containing picosecond 9 | * values 10 | */ 11 | 12 | #ifndef SRC_DATA_STRUCTS_TIMESPECPS_H_ 13 | #define SRC_DATA_STRUCTS_TIMESPECPS_H_ 14 | 15 | #include 16 | 17 | typedef struct timespecps { 18 | int64_t tv_sec; 19 | int64_t tv_psec; 20 | } __attribute__((packed)) timespecps_t; 21 | 22 | 23 | #endif /* SRC_DATA_STRUCTS_TIMESPECPS_H_ */ 24 | -------------------------------------------------------------------------------- /tools/data_structs/vlan_ethhdr.h: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | typedef struct vlan_ethhdr { 4 | uint64_t h_dest : 48; 5 | uint64_t h_source : 48; 6 | uint16_t h_vlan_proto; 7 | uint16_t h_vlan_TCI; 8 | uint16_t h_vlan_encapsulated_proto; 9 | }__attribute__((packed)) vlan_ethhdr_t; 10 | -------------------------------------------------------------------------------- /tools/exact-pcap-parse.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018 All rights reserved. 3 | * See LICENSE.txt for full details. 4 | * 5 | * Created: 28 Jul 2017 6 | * Author: Matthew P. Grosvenor 7 | * Description: A tool for parsing pcaps and expcaps and outputting in ASCII 8 | * for debugging and inspection. 9 | * 10 | */ 11 | 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | #include "data_structs/pcap_buff.h" 27 | #include "data_structs/expcap.h" 28 | 29 | 30 | USE_CH_LOGGER_DEFAULT; 31 | USE_CH_OPTIONS; 32 | 33 | 34 | struct { 35 | char* input; 36 | char* csv; 37 | bool verbose; 38 | char* format; 39 | ch_word offset; 40 | ch_word max; 41 | ch_word num; 42 | bool write_header; 43 | } options; 44 | 45 | 46 | static volatile bool stop = false; 47 | void signal_handler(int signum) 48 | { 49 | ch_log_warn("Caught signal %li, shutting down\n", signum); 50 | if(stop == 1){ 51 | ch_log_fatal("Hard exit\n"); 52 | } 53 | stop = 1; 54 | } 55 | 56 | 57 | void dprint_packet(int fd, bool expcap, pcap_pkthdr_t* pkt_hdr, char* packet, 58 | bool nl, bool content, int total_out, int64_t timedelta_ns) 59 | { 60 | char fmtd[4096] = {0}; 61 | 62 | if(content && options.num != 0){ 63 | int n = 0; 64 | 65 | if(options.num < 0){ 66 | options.num = INT64_MAX; 67 | } 68 | 69 | for(int64_t i = 0; i < MIN((int64_t)pkt_hdr->caplen,options.num); i++){ 70 | n += snprintf(fmtd + n, 4096 -n, "%02x", *((uint8_t*)packet +i)); 71 | } 72 | } 73 | dprintf(fd, "%04i,%lins,%i.%09i,%i,%i,", 74 | total_out, timedelta_ns, 75 | pkt_hdr->ts.ns.ts_sec, pkt_hdr->ts.ns.ts_nsec, 76 | pkt_hdr->caplen, pkt_hdr->len); 77 | 78 | 79 | if(expcap && packet){ 80 | expcap_pktftr_t* pkt_ftr = (expcap_pktftr_t*)((char*)(packet) 81 | + pkt_hdr->caplen - sizeof(expcap_pktftr_t)); 82 | 83 | dprintf(fd, "%i,%i,%li.%012li,", 84 | pkt_ftr->dev_id, 85 | pkt_ftr->port_id, 86 | (int64_t)pkt_ftr->ts_secs, (int64_t)pkt_ftr->ts_psecs); 87 | } 88 | 89 | dprintf(fd, "%s",fmtd); 90 | 91 | if(nl){ 92 | dprintf(fd, "\n"); 93 | } 94 | } 95 | 96 | int main(int argc, char** argv) 97 | { 98 | signal(SIGHUP, signal_handler); 99 | signal(SIGINT, signal_handler); 100 | signal(SIGPIPE, signal_handler); 101 | signal(SIGALRM, signal_handler); 102 | signal(SIGTERM, signal_handler); 103 | 104 | ch_opt_addsu(CH_OPTION_REQUIRED,'i',"input","PCAP file to read", &options.input); 105 | ch_opt_addsi(CH_OPTION_OPTIONAL,'c',"csv","CSV output file to write to ", &options.csv, NULL); 106 | ch_opt_addbi(CH_OPTION_FLAG,'v',"verbose","Printout verbose output", &options.verbose, false); 107 | ch_opt_addsu(CH_OPTION_REQUIRED,'f',"format","Input format [pcap | expcap]", &options.format); 108 | ch_opt_addii(CH_OPTION_OPTIONAL,'o',"offset","Offset into the file to start ", &options.offset, 0); 109 | ch_opt_addii(CH_OPTION_OPTIONAL,'m',"max","Max packets to output (<0 means all)", &options.max, -1); 110 | ch_opt_addii(CH_OPTION_OPTIONAL,'n',"num-chars","Number of characters to output (<=0 means all)", &options.num, 64); 111 | ch_opt_addbi(CH_OPTION_FLAG,'w',"write-header","Write out a header row", &options.write_header, false); 112 | 113 | ch_opt_parse(argc,argv); 114 | 115 | if(!options.verbose && !options.csv){ 116 | ch_log_fatal("Must choose an output type. Use either --verbose or --csv\n"); 117 | } 118 | 119 | bool expcap = false; 120 | if(strncmp(options.format, "pcap", strlen("pcap")) == 0){ 121 | expcap = false; 122 | } 123 | else if(strncmp(options.format, "expcap", strlen("expcap")) == 0){ 124 | expcap = true; 125 | } 126 | else{ 127 | ch_log_fatal("Unknown format type =\"%s\". Must be \"pcap\" or \"expcap\"\n", options.format); 128 | } 129 | 130 | if(options.max < 0){ 131 | options.max = INT64_MAX; 132 | } 133 | 134 | ch_log_info("Starting PCAP parser...\n"); 135 | pcap_buff_t buff = {0}; 136 | if(pcap_buff_from_file(&buff, options.input, expcap) != BUFF_ENONE){ 137 | ch_log_fatal("Failed to create a new pcap buff from file: %s\n", options.input); 138 | } 139 | 140 | int csv_fd = -1; 141 | if(options.csv){ 142 | csv_fd = open(options.csv,O_WRONLY | O_CREAT | O_TRUNC, 0666); 143 | if(csv_fd < 0){ 144 | ch_log_fatal("Could not open in missed file %s (%s)\n", options.csv, strerror(errno)); 145 | } 146 | 147 | if(options.write_header){ 148 | dprintf(csv_fd,"packet_num,delta_ns,seconds.nanos,capture_length,packet_length"); 149 | if(expcap){ 150 | dprintf(csv_fd,",device_id,port_id,seconds.picos"); 151 | } 152 | 153 | dprintf(csv_fd,"\n"); 154 | } 155 | } 156 | 157 | int64_t timenowns = 0; 158 | int64_t timeprevns = 0; 159 | int64_t total_out = 0; 160 | pkt_info_t info; 161 | for(int pkt_num = 0; !stop && pkt_num < options.offset + options.max; pkt_num++, 162 | timeprevns = timenowns ){ 163 | if(pkt_num && pkt_num % (1000 * 1000) == 0){ 164 | ch_log_info("Loaded %li,000,000 packets\n", pkt_num/1000/1000); 165 | } 166 | 167 | info = pcap_buff_next_packet(&buff); 168 | if(info == PKT_EOF){ 169 | break; 170 | } 171 | 172 | pcap_pkthdr_t hdr = *buff.hdr; 173 | timenowns = hdr.ts.ns.ts_sec * 1000ULL * 1000 * 1000 + hdr.ts.ns.ts_nsec; 174 | 175 | if(timeprevns == 0){ 176 | timeprevns = timenowns; 177 | } 178 | 179 | const int64_t time_delta = timenowns - timeprevns; 180 | 181 | if(pkt_num < options.offset){ 182 | continue; 183 | } 184 | 185 | if(options.verbose){ 186 | dprint_packet(STDOUT_FILENO, expcap, &hdr, buff.pkt, true, true, total_out, time_delta); 187 | } 188 | 189 | if(csv_fd > 0){ 190 | dprint_packet(csv_fd, expcap, &hdr, buff.pkt, true, true, total_out, time_delta); 191 | } 192 | 193 | total_out++; 194 | } 195 | 196 | 197 | if(csv_fd){ 198 | close(csv_fd); 199 | } 200 | 201 | ch_log_info("Output %li packets\n", total_out); 202 | ch_log_info("PCAP parser, finished\n"); 203 | return 0; 204 | 205 | } 206 | -------------------------------------------------------------------------------- /tools/utils.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 All rights reserved. 3 | * See LICENSE.txt for full details. 4 | * 5 | * Created: 6 Apr. 2018 6 | * Author: Matthew P. Grosvenor 7 | * Description: Application wide useful utility functions 8 | */ 9 | 10 | 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "utils.h" 18 | 19 | int64_t time_now_ns() 20 | { 21 | struct timespec now = {0}; 22 | clock_gettime(CLOCK_REALTIME, &now); 23 | return now.tv_sec * 1000ULL * 1000 * 1000 + now.tv_nsec; 24 | } 25 | 26 | timespecps_t sub_tsps_tsps(const timespecps_t* lhs, const timespecps_t* rhs) 27 | { 28 | timespecps_t result = {0}; 29 | result.tv_psec = lhs->tv_psec - rhs->tv_psec; 30 | result.tv_sec = lhs->tv_sec - rhs->tv_sec; 31 | 32 | if(result.tv_psec < 0){ 33 | result.tv_psec = PS_IN_SECS + result.tv_psec; 34 | result.tv_sec -= 1; 35 | } 36 | 37 | return result; 38 | } 39 | 40 | timespecps_t add_tsps_tsps(const timespecps_t* lhs, const timespecps_t* rhs) 41 | { 42 | timespecps_t result = {0}; 43 | int64_t carry_secs = 0; 44 | result.tv_psec = lhs->tv_psec + rhs->tv_psec; 45 | 46 | if(result.tv_psec > PS_IN_SECS){ 47 | carry_secs++; 48 | result.tv_psec -= PS_IN_SECS; 49 | } 50 | 51 | result.tv_sec = lhs->tv_sec + rhs->tv_sec + carry_secs; 52 | 53 | return result; 54 | } 55 | 56 | timespecps_t sub_tsps_ps(const timespecps_t* lhs, const int64_t ps) 57 | { 58 | timespecps_t rhs = {.tv_sec = 0, .tv_psec = ps}; 59 | return sub_tsps_tsps(lhs, &rhs); 60 | } 61 | 62 | 63 | timespecps_t add_tsps_ps(const timespecps_t* lhs, const int64_t ps) 64 | { 65 | timespecps_t rhs = {.tv_sec = 0, .tv_psec = ps}; 66 | return add_tsps_tsps(lhs, &rhs); 67 | } 68 | 69 | 70 | double tsps_to_double_ns(const timespecps_t* lhs) 71 | { 72 | return (double)(lhs->tv_psec/1000.0) + (double)lhs->tv_sec*PS_IN_SECS/1000.0; 73 | } 74 | -------------------------------------------------------------------------------- /tools/utils.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 All rights reserved. 3 | * See LICENSE.txt for full details. 4 | * 5 | * Created: 6 Apr. 2018 6 | * Author: Matthew P. Grosvenor 7 | * Description: Application wide useful utility functions and macros 8 | */ 9 | 10 | 11 | #ifndef SRC_UTILS_H_ 12 | #define SRC_UTILS_H_ 13 | 14 | #include 15 | #include 16 | #include "data_structs/timespecps.h" 17 | 18 | int64_t time_now_ns(); 19 | 20 | #define NS_IN_SECS (1000LL*1000*1000) 21 | #define PS_IN_SECS (1000LL*1000*1000*1000) 22 | timespecps_t sub_tsps_tsps(const timespecps_t* lhs, const timespecps_t* rhs); 23 | timespecps_t add_tsps_tsps(const timespecps_t* lhs, const timespecps_t* rhs); 24 | timespecps_t sub_tsps_ps(const timespecps_t* lhs, const int64_t ps); 25 | timespecps_t add_tsps_ps(const timespecps_t* lhs, const int64_t ps); 26 | double tsps_to_double_ns(const timespecps_t* lhs); 27 | 28 | #endif /* SRC_UTILS_H_ */ 29 | --------------------------------------------------------------------------------