├── .gitattributes ├── .gitignore ├── .vscode ├── launch.json ├── settings.json └── tasks.json ├── Makefile ├── Makefile_DPDK ├── Makefile_PCAP ├── Makefile_deprecated ├── README.md ├── doc ├── 00-a-tour-of-rubik.markdown ├── 01-layout-and-header-stage.markdown ├── 02-expression-and-variable.markdown ├── 03-semantic-completeness.markdown ├── 04-peephole-optimizations.markdown └── 04-peephole-optimizations.png ├── native ├── drivers │ ├── dpdk.c │ ├── dpdk_deprecated.c │ └── pcap.c ├── runtime │ ├── Makefile │ ├── malloc.h │ ├── profile.c │ ├── profile.h │ ├── seq.h │ ├── seq_next.h │ ├── seq_test.c │ ├── timer.h │ ├── tommyds │ │ ├── .gitignore │ │ ├── .travis.yml │ │ ├── AUTHORS │ │ ├── HISTORY │ │ ├── INSTALL │ │ ├── LICENSE │ │ ├── Makefile │ │ ├── README │ │ ├── benchmark.cc │ │ ├── benchmark.geany │ │ ├── benchmark.sln │ │ ├── benchmark.vcxproj │ │ ├── benchmark │ │ │ ├── gr_all.bat │ │ │ ├── gr_all.sh │ │ │ ├── gr_common.gnu │ │ │ ├── gr_def_random_change.gnu │ │ │ ├── gr_def_random_hit.gnu │ │ │ ├── gr_forward_change.gnu │ │ │ ├── gr_forward_hit.gnu │ │ │ ├── gr_forward_insert.gnu │ │ │ ├── gr_forward_miss.gnu │ │ │ ├── gr_forward_remove.gnu │ │ │ ├── gr_forward_size.gnu │ │ │ ├── gr_other_ck_problem.gnu │ │ │ ├── gr_other_googlelibchash_problem.gnu │ │ │ ├── gr_other_judy_problem.gnu │ │ │ ├── gr_random_change.gnu │ │ │ ├── gr_random_hit.gnu │ │ │ ├── gr_random_insert.gnu │ │ │ ├── gr_random_miss.gnu │ │ │ ├── gr_random_remove.gnu │ │ │ └── gr_random_size.gnu │ │ ├── check.c │ │ ├── makecov.sh │ │ ├── makescan.sh │ │ ├── tommy-footer.html │ │ ├── tommy-header.html │ │ ├── tommy.css │ │ ├── tommy.doxygen │ │ ├── tommyds │ │ │ ├── tommy.c │ │ │ ├── tommy.h │ │ │ ├── tommyalloc.c │ │ │ ├── tommyalloc.h │ │ │ ├── tommyarray.c │ │ │ ├── tommyarray.h │ │ │ ├── tommyarrayblk.c │ │ │ ├── tommyarrayblk.h │ │ │ ├── tommyarrayblkof.c │ │ │ ├── tommyarrayblkof.h │ │ │ ├── tommyarrayof.c │ │ │ ├── tommyarrayof.h │ │ │ ├── tommychain.h │ │ │ ├── tommyhash.c │ │ │ ├── tommyhash.h │ │ │ ├── tommyhashdyn.c │ │ │ ├── tommyhashdyn.h │ │ │ ├── tommyhashlin.c │ │ │ ├── tommyhashlin.h │ │ │ ├── tommyhashtbl.c │ │ │ ├── tommyhashtbl.h │ │ │ ├── tommylist.c │ │ │ ├── tommylist.h │ │ │ ├── tommytree.c │ │ │ ├── tommytree.h │ │ │ ├── tommytrie.c │ │ │ ├── tommytrie.h │ │ │ ├── tommytrieinp.c │ │ │ ├── tommytrieinp.h │ │ │ └── tommytypes.h │ │ ├── tommyweb-header.html │ │ └── tommyweb.doxygen │ └── types.h └── weaver.h ├── rubik ├── __init__.py ├── __main__.py ├── compile.py ├── compile2.py ├── lang.py ├── prog.py └── util.py └── stock ├── __init__.py ├── gtp.py ├── pptp.py ├── protocols ├── __init__.py ├── eth.py ├── gre.py ├── gtp.py ├── ip.py ├── loopback.py ├── ppp.py ├── pptp.py ├── quic_frame.py ├── quic_header.py ├── quic_udp.py ├── sctp.py ├── tcp.py └── udp.py ├── quic.py ├── sctp.py └── tcp_ip.py /.gitattributes: -------------------------------------------------------------------------------- 1 | native/runtime/tommyds/* linguist-vendored 2 | app/http-parser/* linguist-vendored 3 | app/libac/* linguist-vendored -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | 132 | pcap/ 133 | weaver_whitebox.c 134 | *.cfg 135 | stack.py 136 | *_test 137 | *.snapshot -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "gcc - Build and debug active file", 9 | "type": "cppdbg", 10 | "request": "launch", 11 | "program": "${fileDirname}/${fileBasenameNoExtension}", 12 | "args": [], 13 | "stopAtEntry": false, 14 | "cwd": "${workspaceFolder}", 15 | "environment": [], 16 | "externalConsole": false, 17 | "MIMode": "gdb", 18 | "setupCommands": [ 19 | { 20 | "description": "Enable pretty-printing for gdb", 21 | "text": "-enable-pretty-printing", 22 | "ignoreFailures": true 23 | } 24 | ], 25 | "preLaunchTask": "C/C++: gcc build active file", 26 | "miDebuggerPath": "/usr/bin/gdb" 27 | } 28 | ] 29 | } -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "python.pythonPath": "./env/bin/python3.7", 3 | "python.formatting.provider": "black", 4 | "C_Cpp.clang_format_fallbackStyle": "{ BasedOnStyle: LLVM, ContinuationIndentWidth: 2, AlignAfterOpenBracket: AlwaysBreak, BinPackArguments: false }" 5 | } -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | "tasks": [ 3 | { 4 | "type": "shell", 5 | "label": "C/C++: gcc build active file", 6 | "command": "/usr/bin/gcc", 7 | "args": [ 8 | "-g", 9 | "${file}", 10 | "-o", 11 | "${fileDirname}/${fileBasenameNoExtension}" 12 | ], 13 | "options": { 14 | "cwd": "${workspaceFolder}" 15 | }, 16 | "problemMatcher": [ 17 | "$gcc" 18 | ], 19 | "group": { 20 | "kind": "build", 21 | "isDefault": true 22 | } 23 | } 24 | ], 25 | "version": "2.0.0" 26 | } -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SHELL := /bin/bash 2 | 3 | 4 | A ?= weaver_whitebox.c 5 | T ?= pcap 6 | C ?= stack_conf 7 | bb := weaver_blackbox.c 8 | wb := weaver_whitebox.template.c 9 | sep = Weaver Auto-generated Blackbox Code 10 | 11 | BUILD_DIR = ./build/ 12 | APP = procpkts 13 | TARGET_FLAG = -DWV_TARGET_$(T) 14 | export TARGET_FLAG 15 | 16 | ### GCC ### 17 | GCC = gcc 18 | GCC_OPT = -m64 # -Wall -DNEWEV -Werror 19 | 20 | GCC_OPT += -g -DNETSTAT -DINFO -DDBGERR -DDBGCERR 21 | # GCC_OPT += -O3 -DNDEBUG -DNETSTAT -DINFO -DDBGERR -DDBGCERR 22 | GCC_OPT += $(DBG_OPT) 23 | 24 | ### LIBRARIES AND INCLUDES ### 25 | INC_DIR = ./native 26 | INC= -I$(INC_DIR) -I$(INC_DIR)/runtime -I$(INC_DIR)/runtime/tommyds 27 | 28 | ### SOURCE FILES ### 29 | 30 | LIB_FLAGS += -lstdc++ -lpcre2-8 31 | ifeq ($(T), pcap) 32 | LIB_FLAGS += -lpcap 33 | endif 34 | 35 | ifeq ($(T), dpdk) 36 | ifeq ($(RTE_SDK),) 37 | $(error "Please define RTE_SDK environment variable") 38 | endif 39 | RTE_TARGET ?= x86_64-native-linuxapp-gcc 40 | 41 | NIC ?= XL710 42 | FWD ?= FWD 43 | PERF ?= EVAL_PERF 44 | SRC_DIR = $(PWD) 45 | SRCS = $(bb) $(A) native/drivers/$(T).c $(SRC_DIR)/native/runtime/libwvrt.a 46 | DPDK_INC = -I$(SRC_DIR)/native/ -I$(SRC_DIR)/native/runtime -I$(SRC_DIR)/native/runtime/tommyds 47 | include $(RTE_SDK)/mk/rte.vars.mk 48 | CFLAGS += $(DPDK_INC) -D$(NIC) -D$(PERF) $(GCC_OPT) -D$(FWD) 49 | LDFLAGS += -lpcre2-8 50 | 51 | SRCS-y := $(SRCS) 52 | include $(RTE_SDK)/mk/rte.extapp.mk 53 | 54 | $(SRC_DIR)/native/runtime/libwvrt.a: 55 | cd $(SRC_DIR)/native/runtime && $(MAKE) -C . 56 | 57 | endif 58 | 59 | ifeq ($(T), pcap) 60 | ### GOALS ### 61 | SRCS = $(bb) $(A) native/drivers/$(T).c native/runtime/libwvrt.a 62 | 63 | all: $(APP) 64 | 65 | $(APP): $(SRCS) 66 | $(GCC) $(GCC_OPT) -o $@ $^ $(INC) $(LIBS) $(LIB_FLAGS) $(TARGET_FLAG) 67 | 68 | native/runtime/libwvrt.a: 69 | $(MAKE) -C native/runtime 70 | 71 | clean: 72 | -$(RM) procpkts $(wb) $(bb) 73 | -$(RM) -rf build/ 74 | -$(RM) native/weaver.h.gch 75 | -$(RM) test_seq 76 | $(MAKE) -C native/runtime clean 77 | 78 | .PHONY: all clean weaver_blackbox.c native/runtime/libwvrt.a 79 | endif 80 | 81 | gen: 82 | # https://stackoverflow.com/a/7104422 83 | python3 -m rubik $(C) | tee >(sed -e "/$(sep)/,\$$d" > $(wb)) | sed -n -e "/$(sep)/,\$$w $(bb)" 84 | 85 | test: test_seq 86 | ./test_seq 87 | 88 | test_seq: native/runtime/seq_test.c native/runtime/seq.h 89 | $(GCC) -o test_seq native/runtime/seq_test.c 90 | -------------------------------------------------------------------------------- /Makefile_DPDK: -------------------------------------------------------------------------------- 1 | SHELL := /bin/bash 2 | 3 | 4 | A ?= weaver_whitebox.c 5 | C ?= stack_conf 6 | T ?= dpdk 7 | bb := weaver_blackbox.c 8 | wb := weaver_whitebox.template.c 9 | sep = Weaver Auto-generated Blackbox Code 10 | 11 | BUILD_DIR = ./build/ 12 | APP = procpkts 13 | TARGET_FLAG = -DWV_TARGET_$(T) 14 | export TARGET_FLAG 15 | 16 | ### GCC ### 17 | GCC = gcc 18 | GCC_OPT = -m64 # -Wall -DNEWEV -Werror 19 | 20 | GCC_OPT += -g -DNETSTAT -DINFO -DDBGERR -DDBGCERR 21 | # GCC_OPT += -O3 -DNDEBUG -DNETSTAT -DINFO -DDBGERR -DDBGCERR 22 | GCC_OPT += $(DBG_OPT) 23 | 24 | ### LIBRARIES AND INCLUDES ### 25 | INC_DIR = ./native 26 | INC= -I$(INC_DIR) -I$(INC_DIR)/runtime -I$(INC_DIR)/runtime/tommyds 27 | 28 | ### SOURCE FILES ### 29 | 30 | # Build using pkg-config variables if possible 31 | ifneq ($(shell pkg-config --exists libdpdk && echo 0),0) 32 | $(error "no installation of DPDK found") 33 | endif 34 | 35 | LIB_FLAGS += -lstdc++ -lpcre2-8 36 | 37 | all: shared 38 | .PHONY: shared static 39 | shared: build/$(APP)-shared 40 | ln -sf $(APP)-shared build/$(APP) 41 | static: build/$(APP)-static 42 | ln -sf $(APP)-static build/$(APP) 43 | 44 | PKGCONF ?= pkg-config 45 | 46 | PC_FILE := $(shell $(PKGCONF) --path libdpdk 2>/dev/null) 47 | CFLAGS += -O3 $(shell $(PKGCONF) --cflags libdpdk) 48 | # Add flag to allow experimental API as l2fwd uses rte_ethdev_set_ptype API 49 | CFLAGS += -DALLOW_EXPERIMENTAL_API 50 | LDFLAGS_SHARED = $(shell $(PKGCONF) --libs libdpdk) 51 | LDFLAGS_STATIC = $(shell $(PKGCONF) --static --libs libdpdk) 52 | 53 | ifeq ($(MAKECMDGOALS),static) 54 | # check for broken pkg-config 55 | ifeq ($(shell echo $(LDFLAGS_STATIC) | grep 'whole-archive.*l:lib.*no-whole-archive'),) 56 | $(warning "pkg-config output list does not contain drivers between 'whole-archive'/'no-whole-archive' flags.") 57 | $(error "Cannot generate statically-linked binaries with this version of pkg-config") 58 | endif 59 | endif 60 | 61 | 62 | SRC_DIR = $(PWD) 63 | SRCS = $(bb) $(A) native/drivers/$(T).c $(SRC_DIR)/native/runtime/libwvrt.a 64 | DPDK_INC = -I$(SRC_DIR)/native/ -I$(SRC_DIR)/native/runtime -I$(SRC_DIR)/native/runtime/tommyds 65 | CFLAGS += $(DPDK_INC) $(GCC_OPT) 66 | # LDFLAGS += -lpcre2-8 67 | 68 | SRCS-y := $(SRCS) 69 | 70 | $(SRC_DIR)/native/runtime/libwvrt.a: 71 | cd $(SRC_DIR)/native/runtime && $(MAKE) -C . 72 | 73 | 74 | gen: 75 | # https://stackoverflow.com/a/7104422 76 | python3 -m rubik $(C) | tee >(sed -e "/$(sep)/,\$$d" > $(wb)) | sed -n -e "/$(sep)/,\$$w $(bb)" 77 | 78 | test: test_seq 79 | ./test_seq 80 | 81 | test_seq: native/runtime/seq_test.c native/runtime/seq.h 82 | $(GCC) -o test_seq native/runtime/seq_test.c 83 | 84 | build/$(APP)-shared: $(SRCS-y) Makefile_DPDK $(PC_FILE) | build 85 | $(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_SHARED) $(TARGET_FLAG) 86 | 87 | build/$(APP)-static: $(SRCS-y) Makefile_DPDK $(PC_FILE) | build 88 | $(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_STATIC) $(TARGET_FLAG) 89 | 90 | build: 91 | @mkdir -p $@ 92 | 93 | .PHONY: clean 94 | clean: 95 | rm -f build/$(APP) build/$(APP)-static build/$(APP)-shared 96 | test -d build && rmdir -p build || true 97 | -$(RM) procpkts $(wb) $(bb) 98 | -$(RM) -rf build/ 99 | -$(RM) native/weaver.h.gch 100 | -$(RM) test_seq 101 | $(MAKE) -C native/runtime clean -------------------------------------------------------------------------------- /Makefile_PCAP: -------------------------------------------------------------------------------- 1 | SHELL := /bin/bash 2 | 3 | A ?= weaver_whitebox.c 4 | T ?= pcap 5 | C ?= stack_conf 6 | bb := weaver_blackbox.c 7 | wb := weaver_whitebox.template.c 8 | sep = Weaver Auto-generated Blackbox Code 9 | 10 | BUILD_DIR = ./build/ 11 | APP = procpkts 12 | TARGET_FLAG = -DWV_TARGET_$(T) 13 | export TARGET_FLAG 14 | 15 | ### GCC ### 16 | GCC = gcc 17 | GCC_OPT = -m64 # -Wall -DNEWEV -Werror 18 | 19 | GCC_OPT += -g -DNETSTAT -DINFO -DDBGERR -DDBGCERR 20 | # GCC_OPT += -O3 -DNDEBUG -DNETSTAT -DINFO -DDBGERR -DDBGCERR 21 | GCC_OPT += $(DBG_OPT) 22 | 23 | ### LIBRARIES AND INCLUDES ### 24 | INC_DIR = ./native 25 | INC= -I$(INC_DIR) -I$(INC_DIR)/runtime -I$(INC_DIR)/runtime/tommyds 26 | 27 | ### SOURCE FILES ### 28 | 29 | LIB_FLAGS += -lstdc++ -lpcre2-8 30 | ifeq ($(T), pcap) 31 | LIB_FLAGS += -lpcap 32 | endif 33 | 34 | 35 | ifeq ($(T), pcap) 36 | ### GOALS ### 37 | SRCS = $(bb) $(A) native/drivers/$(T).c native/runtime/libwvrt.a 38 | 39 | all: $(APP) 40 | 41 | $(APP): $(SRCS) 42 | $(GCC) $(GCC_OPT) -o $@ $^ $(INC) $(LIBS) $(LIB_FLAGS) $(TARGET_FLAG) 43 | 44 | native/runtime/libwvrt.a: 45 | $(MAKE) -C native/runtime 46 | 47 | clean: 48 | -$(RM) procpkts $(wb) $(bb) 49 | -$(RM) -rf build/ 50 | -$(RM) native/weaver.h.gch 51 | -$(RM) test_seq 52 | $(MAKE) -C native/runtime clean 53 | 54 | .PHONY: all clean weaver_blackbox.c native/runtime/libwvrt.a 55 | endif 56 | 57 | gen: 58 | # https://stackoverflow.com/a/7104422 59 | python3 -m rubik $(C) | tee >(sed -e "/$(sep)/,\$$d" > $(wb)) | sed -n -e "/$(sep)/,\$$w $(bb)" 60 | 61 | test: test_seq 62 | ./test_seq 63 | 64 | test_seq: native/runtime/seq_test.c native/runtime/seq.h 65 | $(GCC) -o test_seq native/runtime/seq_test.c 66 | -------------------------------------------------------------------------------- /Makefile_deprecated: -------------------------------------------------------------------------------- 1 | SHELL := /bin/bash 2 | 3 | 4 | A ?= weaver_whitebox.c 5 | T ?= pcap 6 | C ?= stack_conf 7 | bb := weaver_blackbox.c 8 | wb := weaver_whitebox.template.c 9 | sep = Weaver Auto-generated Blackbox Code 10 | 11 | BUILD_DIR = ./build/ 12 | APP = procpkts 13 | TARGET_FLAG = -DWV_TARGET_$(T) 14 | export TARGET_FLAG 15 | 16 | ### GCC ### 17 | GCC = gcc 18 | GCC_OPT = -m64 # -Wall -DNEWEV -Werror 19 | 20 | GCC_OPT += -g -DNETSTAT -DINFO -DDBGERR -DDBGCERR 21 | # GCC_OPT += -O3 -DNDEBUG -DNETSTAT -DINFO -DDBGERR -DDBGCERR 22 | GCC_OPT += $(DBG_OPT) 23 | 24 | ### LIBRARIES AND INCLUDES ### 25 | INC_DIR = ./native 26 | INC= -I$(INC_DIR) -I$(INC_DIR)/runtime -I$(INC_DIR)/runtime/tommyds 27 | 28 | ### SOURCE FILES ### 29 | 30 | LIB_FLAGS += -lstdc++ -lpcre2-8 31 | ifeq ($(T), pcap) 32 | LIB_FLAGS += -lpcap 33 | endif 34 | 35 | ifeq ($(T), dpdk) 36 | ifeq ($(RTE_SDK),) 37 | $(error "Please define RTE_SDK environment variable") 38 | endif 39 | RTE_TARGET ?= x86_64-native-linuxapp-gcc 40 | 41 | SRC_DIR = $(PWD) 42 | SRCS = $(bb) $(A) native/drivers/$(T).c $(SRC_DIR)/native/runtime/libwvrt.a 43 | DPDK_INC = -I$(SRC_DIR)/native/ -I$(SRC_DIR)/native/runtime -I$(SRC_DIR)/native/runtime/tommyds 44 | include $(RTE_SDK)/mk/rte.vars.mk 45 | CFLAGS += $(DPDK_INC) -D$(NIC) -D$(PERF) $(GCC_OPT) -D$(FWD) 46 | LDFLAGS += -lpcre2-8 47 | 48 | SRCS-y := $(SRCS) 49 | include $(RTE_SDK)/mk/rte.extapp.mk 50 | 51 | $(SRC_DIR)/native/runtime/libwvrt.a: 52 | cd $(SRC_DIR)/native/runtime && $(MAKE) -C . 53 | 54 | endif 55 | 56 | ifeq ($(T), pcap) 57 | ### GOALS ### 58 | SRCS = $(bb) $(A) native/drivers/$(T).c native/runtime/libwvrt.a 59 | 60 | all: $(APP) 61 | 62 | $(APP): $(SRCS) 63 | $(GCC) $(GCC_OPT) -o $@ $^ $(INC) $(LIBS) $(LIB_FLAGS) $(TARGET_FLAG) 64 | 65 | native/runtime/libwvrt.a: 66 | $(MAKE) -C native/runtime 67 | 68 | clean: 69 | -$(RM) procpkts $(wb) $(bb) 70 | -$(RM) -rf build/ 71 | -$(RM) native/weaver.h.gch 72 | -$(RM) test_seq 73 | $(MAKE) -C native/runtime clean 74 | 75 | .PHONY: all clean weaver_blackbox.c native/runtime/libwvrt.a 76 | endif 77 | 78 | gen: 79 | # https://stackoverflow.com/a/7104422 80 | python3 -m rubik $(C) | tee >(sed -e "/$(sep)/,\$$d" > $(wb)) | sed -n -e "/$(sep)/,\$$w $(bb)" 81 | 82 | test: test_seq 83 | ./test_seq 84 | 85 | test_seq: native/runtime/seq_test.c native/runtime/seq.h 86 | $(GCC) -o test_seq native/runtime/seq_test.c 87 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Proof-of-concept and evaluation prototype for NSDI'21: *Programming Network Stack for Middleboxes with Rubik* 2 | 3 | Step 0, make sure `python3` installed with version >= 3.7; C toolchain (e.g. `build-essential` on Ubuntu) installed; `libpcap-dev` is required for `pcap` target and DPDK SDK is required for `dpdk` target. And mako package is installed: 4 | 5 | ``` 6 | pip3 install mako 7 | ``` 8 | 9 | Step 1, generate `weaver_blackbox.c` according to configure 10 | 11 | ``` 12 | # to build stocking protocol stacks 13 | make gen C=stock.tcp_ip 14 | make gen C=stock.gtp 15 | ``` 16 | 17 | Step 2, compile the blackbox along with custom code 18 | 19 | ``` 20 | # auto-generated blank template 21 | make A=weaver_whitebox.template.c 22 | # generally it's recommend to edit the copy of the template 23 | cp weaver_whitebox.template.c weaver_whitebox.c 24 | # and build it with custom code 25 | make 26 | # DPDK target 27 | make T=dpdk 28 | ``` 29 | 30 | Finally, run built executable `procpkts`. 31 | 32 | ---- 33 | 34 | Rubik is a perfect tool for: 35 | * building software middlebox for network stacks, e.g. TCP/IP, GTP, QUIC 36 | * validating the functionality of newly-designed protocols 37 | * modeling network protocols/stacks with a comprehensive abstraction 38 | 39 | To learn more, please head on to [a tour of Rubik](./doc/00-a-tour-of-rubik.markdown) and enjoy hacking! 40 | -------------------------------------------------------------------------------- /doc/01-layout-and-header-stage.markdown: -------------------------------------------------------------------------------- 1 | > Author: Sun Guangda \ 2 | > Date: 2020.9.29 3 | 4 | This is the catch-up document for layout declaration and header stage. You have learned the basic usage of them in [the tour](doc/00-a-tour-of-rubik.markdown). 5 | 6 | #### Variable-length field 7 | 8 | Variable-length fields are fields that cannot determine length until runtime. There must exist an expression that only depends on the values of fields **occur before this field** and whose value is the **bit length** of this field. A common case of variable-length field is an IP option, whose value field always follows a single-byte length field to record the byte length of it. It could be declared as a layout like: 9 | 10 | ```python 11 | class ip_option(layout): 12 | length = Bit(8) 13 | value = Bit(length << 3) 14 | ``` 15 | 16 | Variable-length fields always have byte slice type. 17 | 18 | #### Declare layout for permanent variables 19 | 20 | Every permanent variable must have an initial value, which will be used if the instance is just created. The initial value can be any expression that only depends on header fields, and it is indicated with `init` property: 21 | 22 | ```python 23 | class tcp_perm_vars(layout): 24 | window_left = Bit(32, init=0) 25 | window_right = Bit(32, init=(1 << 32) - 1) 26 | ``` 27 | 28 | #### Sequence parsing 29 | 30 | You can use `+` to connect layouts to parse them sequentially: 31 | 32 | ```python 33 | myproto.header = mylayout1 + mylayout2 34 | ``` 35 | 36 | #### Conditional parsing 37 | 38 | You can use `If` statement to parse a layout base on some conditions. The condition could be an expression that depends on header fields that already parsed when encountering the condition. 39 | 40 | ```python 41 | myproto.header = mylayout1 + If(mylayout1.has_layout2 == 1) >> mylayout2 # myproto.header.has_layout2 is also available 42 | ``` 43 | 44 | Unlike preparation stage, `Else` syntax is not impemented here. There's also a bug currently occuring if you assign `If` statement to a Python name and then reference to that name in header stage assignment, so just prevent to do it for now. 45 | 46 | #### Options-parsing loop 47 | 48 | It is an idiomatic way to design protocol header with variable-number, variable-length optional fields as type-length-variable (TLV). Rubik provides a handy way to define it. 49 | 50 | First, declare each of all types of value a layout. The type field of them must appear at the beginning of the layout and have the same length: 51 | 52 | ```python 53 | class optional_timestamp(layout): 54 | type = Bit(8, const=6) # use `const` property to declare type number 55 | timestamp = UInt(32) 56 | 57 | class optional_flags(layout): 58 | type = Bit(8, const=13) 59 | flag_a = Bit(1) 60 | flag_b = Bit(4) 61 | _reserved = Bit(3) 62 | 63 | class optional_string(layout): 64 | type = Bit(8, const=25) 65 | length = Bit(8) 66 | string = Bit(length << 3) 67 | 68 | class optional_others(layout): 69 | type = Bit(8) # fallback case 70 | length = Bit(8) 71 | remain = Bit(length << 3) 72 | 73 | class end_of_options(layout): 74 | type = Bit(8, const=0) 75 | ``` 76 | 77 | A fallback case is useful if any of option types we don't care have the same layout. Note that the content of layout (corresponding struct in C) of the fallback case will be override multiple times, so its content is garbage, so you have to define a `optional_string` layout even it has the exact same fields as the fallback layout, since you care about its content rather than just skip it. 78 | 79 | An `AnyUntil` statement could be used in header stage with all the option layouts: 80 | 81 | ```python 82 | myproto.header = AnyUntil([ 83 | optional_timestamp, 84 | optional_flags, 85 | optional_string, 86 | optional_others,' 87 | end_of_options, 88 | ], myproto.header.type == 0) 89 | ``` 90 | 91 | This will generate a `while` loop with a `switch` statement inside, which fill all the layouts of the appeared options. Every option type could only present once in the packet, or the packet will be treated as malformed and the processing will be terminated. 92 | -------------------------------------------------------------------------------- /doc/02-expression-and-variable.markdown: -------------------------------------------------------------------------------- 1 | > Author: Sun Guangda \ 2 | > Date: 2020.9.30 3 | 4 | This is the catch-up document for expressions and variables in Rubik. Expressions could appear in: 5 | * length expression of variable-length field 6 | * condition in header stage 7 | * preparation statement 8 | * sequence property 9 | * prediction and action statement in PSM stage 10 | * condition and action statement in event stage 11 | * layer entry condition 12 | 13 | The "statement" above only includes two types: `If`-`Else` block and `Assign` statement, and the latter one make an assignment of the value of an expression to a variable. Notice that there's one place which suprisingly do not allowed expression: instance selector definition. It must be a list or a tuple of two lists which only contains plain header fields. 14 | 15 | #### Expression and variable type 16 | 17 | There are two variable types in Rubik: number type and byte slice type. Number type includes fixed fields of all full-byte length and also bit fields. Byte slice type is implemented as a tuple composed by a pointer to the head byte of buffer and a length field, which is the same as Rust's built-in `&[u8]` slice type. Byte slice is technically a "byte slice view", which does not own any data by itself. All byte slices created in a layer configure are either a part of packet or empty slice. As a result, all slices share the same underlaying buffer and there's no modifying opertor defined for them. 18 | 19 | Each expression also has a type. Constant number has number type and constant empty slice has slice type. Expressions that simply read a variable have the same type as read variable. The type of expression decides which set of operators are defined on it. For a number expression, you can: 20 | * operate `+`, `-`, `<<`, `>>` with another number-typed expression as the right-hand operand, and the result expression has number type 21 | * operate `==`, `!=` with another number-typed expression as the right-hand operand, or use `NotOp(expr)` which is an unary operator, and the result expression has number type (`0` for false and `1` for true) 22 | * operate `&`, `|` with anumber number-typed expression as the right-head operand, which has two cases: 23 | * if the left-hand expression is a variable, these are bitwise operators 24 | * if the left-hand expression is compound expression, these are logical operators 25 | 26 | Both cases result an number-typed expression. 27 | 28 | For a slice expression, you can: 29 | * operate `[a:b]` where `a` and `b` are number-typed expressions to shorten the slice from two sides, and the result is a slice 30 | * operate `[i]` where `i` is number-typed expression to extract a byte from slice, and the result is a number 31 | * access read-only property `.length` to get the length of the slice, and the result is a number 32 | 33 | Adding custom operators to number and slice types are quite easy. The corresponding document will be released once the related interface is stable. 34 | 35 | #### Built-in variables 36 | 37 | There are a few variables provided by Rubik runtime which is useful on centain stages and cases. 38 | * `payload` is the *unparsed* part of the packet, which may keep changing in header stage 39 | * `cursor` is the length of parsed part by current layer of the packet 40 | * `sdu` equals to `payload` if `Assemble` statement is not executed, and stores the result of `Assemble` statement 41 | * `current_state` is the index of current state in PSM, the state corresponds to current packet in event stage and corresoponds to previous packet (or start state if no previous packet) before event stage 42 | * `to_active` and `to_passive` indicate the direction of current packet, currently they are implemented as expression 43 | 44 | You should never assign to these variables. 45 | 46 | #### Virtual expression 47 | 48 | The concept of virtual expression is shown in the tour. Internally Rubik create a permanent variable for each virtual expression to record that if the expression "has been" true for any past packet (including current one), and virtual expression will be true if both and permanent variable is set and sequence has no unsorted fragment (there no "hole" in recived buffer). Thus you must define instance and sequence stage if you want to use virtual expression. 49 | 50 | A simple virtual expression is easy to understand, for example, `sp.v.header.x == 1` means "a packet with x field set to 1 has been seen and sequence is sorted". However, things get complicated when virtual expressions are compounded with boolean operators. With negating operator, `NotOp(sp.v.header.x == 1)` means "the expected packet has not been seen or the sequence is still unsorted", which has a different meaning with `sp.v.header.x != 1`. With logical and expression, `sp.v.header.x == 1 & sp.v.header.y == 1` means "both flags has been seen to be set **in the same packet**, and the sequence is sorted now". The internal permanent variable is actually tracking expression `sp.header.x == 1 & sp.header.y == 1`, which means virtual expressions *extend* across logical and operators. So the expression is equivalent to `sp.v.header.x == 1 & sp.header.y == 1`, but it's less obvious to write in this way. In one word, virtual expressions extend across logical and, but not extend across logical or and logical not operators. If you really need the semantic different from virtual expression's behaviour, try to desugar the code and use permanent variable directly. -------------------------------------------------------------------------------- /doc/03-semantic-completeness.markdown: -------------------------------------------------------------------------------- 1 | | NF | Description | Can Rubik help development? | How can Rubik support it/Why does Rubik not support it | 2 | | ------------------------------------------------------------ | --------------------------------------------------------- | --------------------------- | ------------------------------------------------------------ | 3 | | Stateless Firewall | Reject traffic from certain hosts/ports | Yes | **Rubik**: pose the 5-tuple of each packet to the high-level function
**High-level function**: match the 5-tuple and reject the traffic | 4 | | Stateful Firewall | Reject the adversaries with half-connection attacks | Yes | **Rubik**: raise an event of the first and the third handshake of each TCP connection, and pose the connection information to the high-level function.
**High-level function**: Upon the first handshake, maintaining a list that records every newly-recevied SYN and expires those that times out. SYN from the expired host will be dropped for some time; Upon the third handshake, removing the host from list | 5 | | IDS | Detect malicious content in traffic | Yes | **Rubik**: pose the reassembled payload to high-level function
**High-level function**: match the payload against a set of rules | 6 | | NAT | Rewrite packet header between LAN and WAN | Yes | **Rubik**: pose the source and destination IP address and their pointers of each IP packet to high-level function
**High-level function**: maintain a hash table storing the active mapping from LAN IP address to WAN IP address, rewrite the header of each packet according to the mappings | 7 | | IPS | Prevent attack on detecting malicious content | No | **Rubik**: pose the reassembled payload to high-level function
**High-level function**: the packets have been sent out when the high-level function can inspect. Therefore, the NF cannot modify or drop the packet | 8 | | HTTP proxy | Proxy/filter HTTP traffic | No | **Rubik**: pose the reassembled payload to high-level function
**High-level function**: The NF can modify L2-L4 header to proxy the traffic, but cannot filter L7 content, e.g. remove ads | 9 | | Inline stream edit [MiddleClick](https://www.diva-portal.org/smash/record.jsf?dswid=1144&pid=diva2%3A1582880&c=1&searchType=SIMPLE&language=en&query=Combined+stateful+classification+and+session+splicing+for+high-speed+NFV+service+chaining&af=%5B%5D&aq=%5B%5B%5D%5D&aq2=%5B%5B%5D%5D&aqe=%5B%5D&noOfRows=50&sortOrder=author_sort_asc&sortOrder2=title_sort_asc&onlyFullText=false&sf=all) | Prevent forwarding packets before stream data is modified | No | **Rubik**: To make the sender keep sending packets, the NF must proactively send ACK to collect enough data. However, Rubik currently does not support sending packets | 10 | 11 | -------------------------------------------------------------------------------- /doc/04-peephole-optimizations.markdown: -------------------------------------------------------------------------------- 1 | ![Peephole Optimizations](./04-peephole-optimizations.png) -------------------------------------------------------------------------------- /doc/04-peephole-optimizations.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ants-xjtu/rubik/107cc58254f56c02902f55c2b87054f5ebfcd502/doc/04-peephole-optimizations.png -------------------------------------------------------------------------------- /native/drivers/pcap.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include 7 | #include 8 | 9 | WV_U8 ctrl_c = 0; 10 | 11 | void ctrl_c_handler(int sig) { 12 | if (ctrl_c) { 13 | printf("shut down badly\n"); 14 | exit(1); 15 | } 16 | printf("\nwill shut down (ctrl-c again to kill)\n"); 17 | ctrl_c = 1; 18 | } 19 | 20 | typedef struct { 21 | WV_Runtime *runtime; 22 | pcap_t *pcap; 23 | } PcapUser; 24 | 25 | void proc(WV_Byte *user, const struct pcap_pkthdr *pcap_header, const WV_Byte *pcap_data) { 26 | WV_Runtime *runtime = ((PcapUser *)user)->runtime; 27 | WV_ByteSlice packet = { .cursor = pcap_data, .length = pcap_header->len }; 28 | WV_U8 status = WV_ProcessPacket(packet, runtime); 29 | WV_ProfileRecord(WV_GetProfile(runtime), pcap_header->len, status); 30 | if (ctrl_c) { 31 | pcap_breakloop(((PcapUser *)user)->pcap); 32 | } 33 | } 34 | 35 | int main(int argc, char *argv[]) { 36 | char *pcap_filename = NULL; 37 | WV_U8 no_loop = 0; 38 | for (int i = 1; i < argc; i += 1) { 39 | if (strcmp(argv[i], "--noloop") == 0) { 40 | no_loop = 1; 41 | } else { 42 | pcap_filename = argv[i]; 43 | } 44 | } 45 | if (pcap_filename == NULL) { 46 | printf("no pcap file\n"); 47 | return 0; 48 | } 49 | 50 | WV_Runtime *runtime; 51 | if (!(runtime = WV_AllocRuntime())) { 52 | fprintf(stderr, "runtime initialization fail\n"); 53 | return 1; 54 | } 55 | 56 | char errbuf[PCAP_ERRBUF_SIZE]; 57 | pcap_t *pcap_packets = pcap_open_offline(pcap_filename, errbuf); 58 | if (!pcap_packets) { 59 | fprintf(stderr, "pcap_open_offline: %s\n", errbuf); 60 | return 1; 61 | } 62 | 63 | PcapUser user = { .runtime = runtime, .pcap = pcap_packets }; 64 | 65 | signal(SIGINT, ctrl_c_handler); 66 | WV_Setup(); 67 | WV_ProfileStart(WV_GetProfile(runtime)); 68 | for (;;) { 69 | pcap_loop(pcap_packets, -1, proc, (void *)&user); 70 | pcap_close(pcap_packets); 71 | if (ctrl_c || no_loop) { 72 | break; 73 | } 74 | pcap_packets = pcap_open_offline(pcap_filename, errbuf); 75 | if (!pcap_packets) { 76 | fprintf(stderr, "pcap_open_offline: %s\n", errbuf); 77 | return 1; 78 | } 79 | } 80 | 81 | WV_ProfileRecordPrint(WV_GetProfile(runtime)); 82 | if (WV_FreeRuntime(runtime)) { 83 | fprintf(stderr, "runtime cleanup fail\n"); 84 | return 1; 85 | } 86 | 87 | printf("shut down correctly\n"); 88 | 89 | return 0; 90 | } -------------------------------------------------------------------------------- /native/runtime/Makefile: -------------------------------------------------------------------------------- 1 | all: libwvrt.a 2 | 3 | libwvrt.a: profile.o tommyds/tommy.o 4 | $(AR) rc $@ $^ 5 | 6 | tommyds/tommy.o: 7 | $(MAKE) -C tommyds 8 | 9 | %.o: %.c %.h types.h 10 | $(CC) -c -g -O3 $< -I./tommyds $(TARGET_FLAG) 11 | 12 | clean: 13 | -$(RM) libwvrt.a 14 | -$(RM) *.o *.gch 15 | -$(MAKE) -C tommyds clean 16 | 17 | .PHONY: all clean 18 | -------------------------------------------------------------------------------- /native/runtime/malloc.h: -------------------------------------------------------------------------------- 1 | #ifndef WEAVER_RUNTIME_MALLOC_H 2 | #define WEAVER_RUNTIME_MALLOC_H 3 | 4 | #ifdef WV_TARGET_dpdk 5 | #include 6 | #include 7 | #define WV_Malloc(n) rte_malloc(NULL, n, 0) 8 | #define WV_Free rte_free 9 | #define WV_Memcpy rte_memcpy 10 | #else 11 | #include 12 | #include 13 | #define WV_Malloc malloc 14 | #define WV_Free free 15 | #define WV_Memcpy memcpy 16 | #endif 17 | 18 | #endif -------------------------------------------------------------------------------- /native/runtime/profile.c: -------------------------------------------------------------------------------- 1 | #include "profile.h" 2 | #include 3 | #include 4 | #include 5 | 6 | WV_U8 WV_ProfileStart(WV_Profile* profile) 7 | { 8 | memset(profile, 0, sizeof(WV_Profile)); 9 | WV_F current = clock() / CLOCKS_PER_SEC; 10 | profile->last_record_sec = current; 11 | profile->next_checkpoint_sec = (WV_U32)current + 1; 12 | } 13 | 14 | WV_U8 WV_ProfileRecord(WV_Profile* profile, WV_U32 byte_length, WV_U8 status) 15 | { 16 | // TODO: use status 17 | profile->interval_byte_count += byte_length; 18 | profile->interval_packet_count += 1; 19 | if (profile->interval_packet_count % 1000000 != 0) { 20 | return 0; 21 | } 22 | WV_F current = (WV_F)clock() / CLOCKS_PER_SEC; 23 | if (current < profile->next_checkpoint_sec) { 24 | return 0; 25 | } 26 | 27 | WV_ProfileRecordPrint(profile); 28 | 29 | profile->interval_byte_count = 0; 30 | profile->interval_packet_count = 0; 31 | profile->next_checkpoint_sec = (WV_U32)current + 2; 32 | profile->last_record_sec = current; 33 | return 0; 34 | } 35 | 36 | WV_U8 WV_ProfileRecordPrint(WV_Profile* profile) 37 | { 38 | WV_F current = (WV_F)clock() / CLOCKS_PER_SEC; 39 | WV_F interval = current - profile->last_record_sec; 40 | WV_F throughput = profile->interval_byte_count / interval / 1e9 * 8; 41 | 42 | profile->last_10_throughput[profile->record_count % 10] = throughput; 43 | profile->record_count += 1; 44 | WV_U8 count = 10; 45 | if (count > profile->record_count) { 46 | count = profile->record_count; 47 | } 48 | WV_F throughput_avg = 0; 49 | for (WV_U8 i = 0; i < count && i < 10; i += 1) { 50 | throughput_avg += profile->last_10_throughput[i]; 51 | } 52 | throughput_avg /= count; 53 | 54 | printf("checkpoint: %f ms, throughput: %f(%f) Gbps (last %d avg.)\n", current * 1000, throughput, throughput_avg, count); 55 | return 0; 56 | } -------------------------------------------------------------------------------- /native/runtime/profile.h: -------------------------------------------------------------------------------- 1 | #ifndef WEAVER_RUNTIME_PROFILE_H 2 | #define WEAVER_RUNTIME_PROFILE_H 3 | 4 | #include "types.h" 5 | 6 | typedef struct _WV_Profile { 7 | WV_U64 interval_byte_count; 8 | WV_U32 interval_packet_count; 9 | WV_U32 next_checkpoint_sec; 10 | WV_F last_record_sec; 11 | WV_F last_10_throughput[10]; 12 | WV_U8 record_count; 13 | } WV_Profile; 14 | 15 | WV_U8 WV_ProfileStart(WV_Profile *); 16 | 17 | WV_U8 WV_ProfileRecord(WV_Profile *, WV_U32, WV_U8); 18 | 19 | WV_U8 WV_ProfileRecordPrint(WV_Profile *); 20 | 21 | #endif -------------------------------------------------------------------------------- /native/runtime/seq_next.h: -------------------------------------------------------------------------------- 1 | #ifndef WEAVER_RUNTIME_SEQ_NEXT_H 2 | #define WEAVER_RUNTIME_SEQ_NEXT_H 3 | 4 | #include "malloc.h" 5 | #include "types.h" 6 | 7 | #define WV_CONFIG_SeqNodeCount 32 8 | #define WV_CONFIG_SeqBufferSize (8 * (1 << 10)) 9 | 10 | typedef struct { 11 | WV_U32 left, right; 12 | } _Part; 13 | 14 | typedef struct { 15 | WV_U32 offset; 16 | _Part parts[WV_CONFIG_SeqNodeCount]; 17 | WV_U8 part_count; 18 | } ZeroBasedSeq; 19 | 20 | WV_U8 InitSeqZD(ZeroBasedSeq *seq) { 21 | seq->offset = 0; 22 | seq->part_count = 0; 23 | } 24 | 25 | #endif -------------------------------------------------------------------------------- /native/runtime/seq_test.c: -------------------------------------------------------------------------------- 1 | #include "seq.h" 2 | #include 3 | #include 4 | #include 5 | 6 | void test_create() { 7 | WV_Seq seq; 8 | WV_InitSeq(&seq, 1, 0); 9 | WV_CleanSeq(&seq, 1); 10 | } 11 | 12 | void test_insert_in_order() { 13 | WV_Seq seq; 14 | WV_InitSeq(&seq, 1, 0); 15 | WV_Byte buf[100]; 16 | for (int i = 0; i < 100; i += 1) { 17 | memset(buf, i, sizeof(buf)); 18 | WV_ByteSlice payload = {.cursor = buf, .length = sizeof(buf)}; 19 | WV_Insert(&seq, i * sizeof(buf), payload, sizeof(buf), 1, 0, 65536); 20 | WV_Byte *free_ptr; 21 | WV_ByteSlice assembled = WV_SeqAssemble(&seq, &free_ptr, 1); 22 | assert(assembled.length == payload.length); 23 | assert( 24 | memcmp(assembled.cursor, payload.cursor, sizeof(assembled.length)) == 0); 25 | if (free_ptr != NULL) { 26 | free(free_ptr); 27 | } 28 | } 29 | WV_CleanSeq(&seq, 1); 30 | } 31 | 32 | void test_insert_out_of_order() { 33 | WV_Seq seq; 34 | WV_InitSeq(&seq, 1, 1); 35 | WV_Byte buf[100]; 36 | memset(buf, 0xCC, sizeof(buf)); 37 | WV_ByteSlice payload = {.cursor = buf, .length = sizeof(buf)}; 38 | WV_Insert(&seq, sizeof(buf), payload, sizeof(buf), 1, 0, 65536); 39 | WV_Byte *free_ptr; 40 | assert(WV_SeqAssemble(&seq, &free_ptr, 1).length == 0); 41 | memset(buf, 0xCD, sizeof(buf)); 42 | WV_Insert(&seq, 0, payload, sizeof(buf), 1, 0, 65536); 43 | WV_ByteSlice assembled = WV_SeqAssemble(&seq, &free_ptr, 1); 44 | assert(assembled.length == 2 * sizeof(buf)); 45 | for (int i = 0; i < 2 * sizeof(buf); i += 1) { 46 | assert(assembled.cursor[i] == (i < sizeof(buf) ? 0xCD : 0xCC)); 47 | } 48 | if (free_ptr != NULL) { 49 | free(free_ptr); 50 | } 51 | WV_CleanSeq(&seq, 1); 52 | } 53 | 54 | void (*TESTCASES[])() = { 55 | test_create, test_insert_in_order, test_insert_out_of_order, NULL}; 56 | 57 | int main() { 58 | for (int i = 0; TESTCASES[i] != NULL; i += 1) { 59 | TESTCASES[i](); 60 | } 61 | return 0; 62 | } -------------------------------------------------------------------------------- /native/runtime/timer.h: -------------------------------------------------------------------------------- 1 | #ifndef WEAVER_NATIVE_RUNTIME_TIMER_H 2 | #define WEAVER_NATIVE_RUNTIME_TIMER_H 3 | 4 | #include "types.h" 5 | #include 6 | 7 | #define TIMEOUT 30 8 | 9 | #define TIMER_FIELDS(inst_type) \ 10 | inst_type *inst_type##_timer_head, *inst_type##_timer_last; 11 | 12 | #define TIMER_INJECT_FIELDS(inst_type) \ 13 | struct inst_type *prev, *next; \ 14 | WV_U64 last_update; 15 | 16 | #define TIMER_INIT(rt, inst_type) \ 17 | rt->inst_type##_timer_head = rt->inst_type##_timer_last = NULL 18 | 19 | #define TIMER_INSERT(rt, inst_type, inst) \ 20 | if (rt->inst_type##_timer_last == NULL) { \ 21 | rt->inst_type##_timer_head = rt->inst_type##_timer_last = inst; \ 22 | inst->prev = inst->next = NULL; \ 23 | } else { \ 24 | inst->prev = NULL; \ 25 | inst->next = rt->inst_type##_timer_head; \ 26 | rt->inst_type##_timer_head->prev = inst; \ 27 | rt->inst_type##_timer_head = inst; \ 28 | } \ 29 | struct timeval tv; \ 30 | gettimeofday(&tv, NULL); \ 31 | inst->last_update = tv.tv_sec 32 | 33 | #define TIMER_FETCH(rt, inst_type, inst) \ 34 | if (inst->prev != NULL) { \ 35 | inst->prev->next = inst->next; \ 36 | } \ 37 | if (inst->next != NULL) { \ 38 | inst->next->prev = inst->prev; \ 39 | } else { \ 40 | rt->inst_type##_timer_last = inst->prev; \ 41 | } \ 42 | if (inst->prev != NULL) { \ 43 | inst->prev = NULL; \ 44 | inst->next = rt->inst_type##_timer_head; \ 45 | rt->inst_type##_timer_head->prev = inst; \ 46 | rt->inst_type##_timer_head = inst; \ 47 | } \ 48 | struct timeval tv; \ 49 | gettimeofday(&tv, NULL); \ 50 | inst->last_update = tv.tv_sec 51 | 52 | #endif -------------------------------------------------------------------------------- /native/runtime/tommyds/.gitignore: -------------------------------------------------------------------------------- 1 | # archives 2 | *.zip 3 | *.tar.gz 4 | 5 | # backups 6 | *~ 7 | 8 | # autotools 9 | # Makefile 10 | Makefile.in 11 | aclocal.m4 12 | autom4te.cache/ 13 | config.guess 14 | config.h 15 | config.h.in 16 | config.log 17 | config.status 18 | config.sub 19 | configure 20 | install-sh 21 | missing 22 | stamp-h1 23 | 24 | # objects 25 | *.o 26 | *.s 27 | *.S 28 | 29 | # coverage 30 | *.gcda 31 | *.gcno 32 | *.info 33 | cov/ 34 | 35 | # project 36 | *.dst 37 | *.epr 38 | makebench.sh 39 | 40 | # specific 41 | tommycheck 42 | tommybench 43 | contrib/ 44 | archive/ 45 | *.ttf 46 | doc/ 47 | benchmark/data/ 48 | benchmark/exec/ 49 | www/ 50 | 51 | 52 | -------------------------------------------------------------------------------- /native/runtime/tommyds/.travis.yml: -------------------------------------------------------------------------------- 1 | # Travis CI configuration file 2 | 3 | sudo: false 4 | 5 | language: c 6 | 7 | script: make check 8 | 9 | compiler: 10 | - clang 11 | - gcc 12 | 13 | os: 14 | - linux 15 | - osx 16 | 17 | -------------------------------------------------------------------------------- /native/runtime/tommyds/AUTHORS: -------------------------------------------------------------------------------- 1 | TommyDS AUTHORS 2 | =============== 3 | 4 | The author of TommyDS is Andrea Mazzoleni. 5 | 6 | You can contact me sending an email at: 7 | 8 | amadvance@gmail.com 9 | 10 | -------------------------------------------------------------------------------- /native/runtime/tommyds/HISTORY: -------------------------------------------------------------------------------- 1 | TommyDS HISTORY 2 | =============== 3 | 4 | 3.0 2018/04 5 | =========== 6 | * Use 64 bits integers when building on a 64 bits platforms. You can store 7 | more than 2^32 objects in the containers. This doesn't applies to tries 8 | that are able to store only 32 bit integers, but you can change this with 9 | the TOMMY_TRIE_BIT and TOMMY_TRIE_INPLACE_BIT definitions. 10 | * Removed the tommy_count_t type that is now always replaced by tommy_size_t. 11 | * Renamed the "key" field of the tommy_node_t to "index" to be more generic 12 | on the possible use. 13 | 14 | 2.2 2018/02 15 | =========== 16 | * Removed tommy_list_remove_head_not_empty() as not used and wrongly 17 | implemented [Daniel Roethlisberger]. 18 | 19 | 2.1 2016/11 20 | =========== 21 | * Added a new hash function for strings: tommy_strhash_u32(). 22 | * Added a new tree implementation. It's not intended to be fast but useful if 23 | you need elements in order. 24 | * Fixed some references to TOMMY_ARRAYBLKOF_SIZE, where instead 25 | TOMMY_ARRAYBLK_SIZE was incorrectly used [Rocco]. 26 | 27 | 2.0 2014/12 28 | =========== 29 | * Fixed a Segmentation Fault bug in the trie_inplace container when inserting 30 | duplicate elements. 31 | * Faster array and hashlin implementation when accessing elements. 32 | * Added new hashtable functions to iterate over all the elements. 33 | * Added a new tommy_calloc() function used for allocating initialized memory. 34 | If you redefined tommy_malloc(), likely you have to redefine also tommy_calloc(). 35 | * Reached 100% code coverage in the regression test. 36 | * Different source code organization. 37 | * Added benchmark comparison with Binary Search Tesseract by Gregorius van 38 | den Hoven. 39 | 40 | 1.8 2013/12 41 | =========== 42 | * Fixed build of tommy_arrayblk in C++. 43 | * Changed the default node size of tommy_trie to fit a cache line of 64 bytes. 44 | * Added benchmark comparison with STX BTree. 45 | 46 | 1.7 2013/12 47 | =========== 48 | * Extends tommy_hashlin_done() to work also if the hashtable is not empty. 49 | * Removes the empty tommy_trie_done() because the real deallocation is done 50 | by the allocator. 51 | 52 | 1.6 2013/11 53 | =========== 54 | * Added a new tommy_arrayblk and tommy_arrayblkof types to store elements 55 | in an array minimizing memory occupation. 56 | 57 | 1.5 2013/06 58 | =========== 59 | * Fixed inline declaration to allow building with clang. 60 | * Added a new tommy_arrayof type to store in an array elements of arbitrary 61 | size. 62 | 63 | 1.4 2013/03 64 | =========== 65 | * Added benchmark comparison with Google BTree, and C++ map and unordered_map. 66 | * Benchmark for Linux is now compiled with "-O3 -march=pentium4 -mtune=generic", 67 | and the Windows one with "/Ox /GL /GS- /arch:SSE2". 68 | 69 | 1.3 2013/02 70 | =========== 71 | * Fixed a Segmentation Fault bug in the hashlin container if exact power 72 | of 2 sizes were used. 73 | * Removed some warnings with newer gcc. 74 | * Minor documentation changes. 75 | * Added benchmark comparison with the judy array implementation by Karl Malbrain. 76 | 77 | 1.2 2012/05 78 | =========== 79 | * Minor documentation changes. 80 | * In the check application, added a speed comparison with the C qsort() 81 | implementation. 82 | 83 | 1.1 2012/05 84 | =========== 85 | * Fixed the tommy_hashdyn_remove() function. Now it shrinks the hashtable if required. 86 | * Minor documentation changes. 87 | 88 | 1.0 2011/03 89 | =========== 90 | * First official version of TommyDS. 91 | * Added tommy_list_foreach functions. 92 | 93 | 0.2 2011/03 94 | =========== 95 | * Added tommy_array. A dynamic vector. 96 | 97 | 0.1 2011/01 98 | =========== 99 | * First release of Tommy. 100 | 101 | -------------------------------------------------------------------------------- /native/runtime/tommyds/INSTALL: -------------------------------------------------------------------------------- 1 | TommyDS INSTALL 2 | =============== 3 | 4 | TommyDS doesn't need any installation. 5 | 6 | You have only to import the required .c and .h files into your program 7 | and use the them. 8 | 9 | -------------------------------------------------------------------------------- /native/runtime/tommyds/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2010, Andrea Mazzoleni. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions 5 | are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 18 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 19 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 20 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 21 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 22 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 23 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 24 | POSSIBILITY OF SUCH DAMAGE. 25 | -------------------------------------------------------------------------------- /native/runtime/tommyds/Makefile: -------------------------------------------------------------------------------- 1 | ############################################################################# 2 | # Tommy Makefile 3 | 4 | # Version of TommyDS 5 | VERSION = 3.0 6 | 7 | # Build options for the check program 8 | ifdef COVERAGE 9 | CFLAGS = -O0 -g -fprofile-arcs -ftest-coverage 10 | else 11 | CFLAGS = -O3 -march=native -Wall -Wextra -Wshadow -Wuninitialized -Wcast-align -Wcast-qual -g 12 | endif 13 | 14 | # Build options for the benchmark 15 | # -std=gnu++0x required by Google btree 16 | BENCHCXXFLAGS = -m32 -O3 -march=native -flto -fpermissive -std=gnu++0x -Wall -g 17 | 18 | # Programs 19 | CC ?= gcc 20 | CXX ?= g++ 21 | OBJDUMP ?= objdump 22 | UNAME = $(shell uname) 23 | 24 | # Linux 25 | ifeq ($(UNAME),Linux) 26 | LIB=-lrt 27 | BENCHLIB=benchmark/lib/judy/libJudyL.a benchmark/lib/judy/libJudyMalloc.a 28 | EXE= 29 | OX=.o 30 | endif 31 | 32 | # Darwin 33 | ifeq ($(UNAME),Darwin) 34 | LIB= 35 | EXE= 36 | OX=.o 37 | endif 38 | 39 | # Windows 40 | ifeq ($(UNAME),) 41 | BENCHLIB=benchmark/lib/judy/src/judy.lib 42 | EXE=.exe 43 | OX=.obj 44 | endif 45 | 46 | #CHECK = ./tommybench -n 1000000 -d tommy-hashlin 47 | CHECK = ./tommycheck 48 | 49 | DEP = \ 50 | tommyds/tommyalloc.c \ 51 | tommyds/tommyalloc.h \ 52 | tommyds/tommyarray.c \ 53 | tommyds/tommyarray.h \ 54 | tommyds/tommyarrayof.c \ 55 | tommyds/tommyarrayof.h \ 56 | tommyds/tommyarrayblk.c \ 57 | tommyds/tommyarrayblk.h \ 58 | tommyds/tommyarrayblkof.c \ 59 | tommyds/tommyarrayblkof.h \ 60 | tommyds/tommy.c \ 61 | tommyds/tommy.h \ 62 | tommyds/tommyhash.c \ 63 | tommyds/tommyhashdyn.c \ 64 | tommyds/tommyhashdyn.h \ 65 | tommyds/tommyhash.h \ 66 | tommyds/tommyhashlin.c \ 67 | tommyds/tommyhashlin.h \ 68 | tommyds/tommyhashtbl.c \ 69 | tommyds/tommyhashtbl.h \ 70 | tommyds/tommylist.c \ 71 | tommyds/tommylist.h \ 72 | tommyds/tommytrie.c \ 73 | tommyds/tommytrie.h \ 74 | tommyds/tommytrieinp.c \ 75 | tommyds/tommytrieinp.h \ 76 | tommyds/tommytypes.h \ 77 | tommyds/tommychain.h 78 | 79 | DEPTEST = \ 80 | check.c \ 81 | benchmark.cc 82 | 83 | all: tommycheck$(EXE) 84 | 85 | bench: tommybench$(EXE) 86 | 87 | tommy$(OX): $(DEP) 88 | $(CC) $(CFLAGS) -c tommyds/tommy.c -o tommy$(OX) 89 | -$(OBJDUMP) -S tommy$(OX) > tommy.s 90 | 91 | tommycheck$(EXE): check.c tommy$(OX) 92 | $(CC) $(CFLAGS) check.c tommy$(OX) -o tommycheck$(EXE) $(LIB) 93 | 94 | tommybench$(EXE): benchmark.cc $(DEP) 95 | $(CXX) $(BENCHCXXFLAGS) benchmark.cc -o tommybench$(EXE) $(LIB) $(BENCHLIB) 96 | 97 | check: tommycheck$(EXE) 98 | ./tommycheck$(EXE) 99 | echo Check completed with success! 100 | 101 | lcov_reset: 102 | lcov -d . -z 103 | rm -f ./lcov.info 104 | 105 | lcov_capture: 106 | lcov -d . --capture -o lcov.info 107 | 108 | lcov_html: 109 | rm -rf ./cov 110 | mkdir cov 111 | genhtml -o ./cov lcov.info 112 | 113 | coverage: 114 | $(MAKE) COVERAGE=1 tommycheck$(EXE) 115 | $(MAKE) lcov_reset 116 | ./tommycheck$(EXE) 117 | $(MAKE) lcov_capture 118 | $(MAKE) lcov_html 119 | 120 | valgrind: 121 | valgrind \ 122 | --tool=memcheck \ 123 | --track-origins=yes \ 124 | --read-var-info=yes \ 125 | -v $(CHECK) \ 126 | 2> valgrind.log 127 | tail valgrind.log 128 | 129 | callgrind: 130 | valgrind \ 131 | --tool=callgrind \ 132 | --dump-instr=yes \ 133 | --trace-jump=yes \ 134 | -v $(CHECK) \ 135 | 2> callgrind.log 136 | tail callgrind.log 137 | 138 | cachegrind: 139 | valgrind \ 140 | --tool=cachegrind \ 141 | -v $(CHECK) \ 142 | 2> cachegrind.log 143 | tail cachegrind.log 144 | 145 | phony: 146 | 147 | graph: phony 148 | cd benchmark && sh gr_all.sh 149 | 150 | doc: phony tommy.doxygen tommy.css $(DEP) 151 | rm -rf doc 152 | mkdir doc 153 | cp -a benchmark/data/def doc/def 154 | cp -a benchmark/data/other doc/other 155 | cp -a benchmark/data/core_i5_650_3G2_linux doc/core_i5_650_3G2_linux 156 | rm -f doc/*/*.lst 157 | rm -f doc/*/*.gnu 158 | doxygen tommy.doxygen 159 | rm -f doc/doxygen.png 160 | rm -f doc/tab_*.png 161 | 162 | web: phony tommyweb.doxygen tommy.css $(DEP) 163 | rm -rf web 164 | mkdir web 165 | cp -a benchmark/data/def web/def 166 | cp -a benchmark/data/other web/other 167 | cp -a benchmark/data/core_i5_650_3G2_linux web/core_i5_650_3G2_linux 168 | rm -f web/*/*.lst 169 | rm -f web/*/*.gnu 170 | doxygen tommyweb.doxygen 171 | rm -f web/doxygen.png 172 | rm -f web/tab_*.png 173 | 174 | clean: 175 | rm -f *.log *.s *.lst *.o 176 | rm -f *.ncb *.suo *.obj 177 | rm -f *.gcno *.gcda lcov.info 178 | rm -rf Debug Release x64 179 | rm -f callgrind.out.* 180 | rm -f cachegrind.out.* 181 | 182 | distclean: clean 183 | rm -f tommybench$(EXE) tommycheck$(EXE) 184 | 185 | maintainerclean: distclean 186 | rm -rf doc web 187 | 188 | DIST=tommyds-$(VERSION) 189 | 190 | DISTFILES=\ 191 | Makefile \ 192 | README LICENSE AUTHORS INSTALL HISTORY \ 193 | tommy.doxygen tommy.css tommy-header.html tommy-footer.html \ 194 | benchmark.vcxproj benchmark.sln \ 195 | benchmark.geany \ 196 | benchmark.cc \ 197 | check.c 198 | 199 | dist: 200 | mkdir $(DIST) 201 | mkdir $(DIST)/tommyds 202 | cp $(DISTFILES) $(DIST) 203 | cp $(DEP) $(DIST)/tommyds 204 | cp $(DEPTEST) $(DIST) 205 | cp -R doc $(DIST) 206 | cp -R benchmark $(DIST)/benchmark 207 | rm -f $(DIST)/benchmark/data/*/*.png 208 | rm -rf $(DIST)/benchmark/data/test 209 | rm -f $(DIST)/benchmark/arial.ttf 210 | rm -f $(DIST).tar.gz 211 | tar cfzo $(DIST).tar.gz $(DIST) 212 | rm -f $(DIST).zip 213 | zip -r $(DIST).zip $(DIST) 214 | rm -r $(DIST) 215 | 216 | distcheck: dist 217 | tar zxvf $(DIST).tar.gz 218 | cd $(DIST) && make check 219 | rm -rf $(DIST) 220 | -------------------------------------------------------------------------------- /native/runtime/tommyds/README: -------------------------------------------------------------------------------- 1 | TommyDS 2 | ======= 3 | 4 | TommyDS is a C library of array, hashtables and tries data structures, 5 | designed for high performance and providing an easy to use interface. 6 | 7 | It's faster than all the similar libraries like rbtree, judy, goodledensehash, 8 | khash, uthash, nedtries and others. 9 | 10 | The data structures provided are: 11 | 12 | tommy_list - A double linked list. 13 | tommy_array - A linear array. It doesn't fragment 14 | the heap. 15 | tommy_arrayblk - A blocked linear array. It doesn't fragment 16 | the heap and it minimizes the space occupation. 17 | tommy_hashtable - A fixed size chained hashtable. 18 | tommy_hashdyn - A dynamic chained hashtable. 19 | tommy_hashlin - A linear chained hashtable. It doesn't have the 20 | problem of the delay when resizing and it doesn't 21 | fragment the heap. 22 | tommy_trie - A trie optimized for cache utilization. 23 | tommy_trie_inplace - A trie completely inplace. 24 | 25 | The documentation is available in HTML format in the doc/index.html file, 26 | and directly in the .h files. 27 | 28 | The official site of TommyDS is: 29 | 30 | http://www.tommyds.it 31 | 32 | -------------------------------------------------------------------------------- /native/runtime/tommyds/benchmark.geany: -------------------------------------------------------------------------------- 1 | 2 | [indentation] 3 | indent_width=4 4 | indent_type=1 5 | indent_hard_tab_width=8 6 | detect_indent=false 7 | indent_mode=2 8 | 9 | [project] 10 | name=benchmark 11 | base_path=/home/am/Desktop/tommy/ 12 | make_in_base_path=true 13 | description= 14 | run_cmd=./tommy 15 | 16 | [files] 17 | current_page=1 18 | FILE_NAME_0=940;Make;0;16;1;1;0;/home/am/Desktop/tommy/Makefile;0 19 | FILE_NAME_1=4126;C;0;16;1;1;0;/home/am/Desktop/tommy//tommyhash.c;0 20 | -------------------------------------------------------------------------------- /native/runtime/tommyds/benchmark.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 11.00 3 | # Visual Studio 2010 4 | Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "benchmark", "benchmark.vcxproj", "{E77E7CFB-E02E-4F25-ADAA-1CF587DB0A04}" 5 | EndProject 6 | Global 7 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 8 | Debug|Win32 = Debug|Win32 9 | Debug|x64 = Debug|x64 10 | Release|Win32 = Release|Win32 11 | Release|x64 = Release|x64 12 | EndGlobalSection 13 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 14 | {E77E7CFB-E02E-4F25-ADAA-1CF587DB0A04}.Debug|Win32.ActiveCfg = Debug|Win32 15 | {E77E7CFB-E02E-4F25-ADAA-1CF587DB0A04}.Debug|Win32.Build.0 = Debug|Win32 16 | {E77E7CFB-E02E-4F25-ADAA-1CF587DB0A04}.Debug|x64.ActiveCfg = Debug|x64 17 | {E77E7CFB-E02E-4F25-ADAA-1CF587DB0A04}.Debug|x64.Build.0 = Debug|x64 18 | {E77E7CFB-E02E-4F25-ADAA-1CF587DB0A04}.Release|Win32.ActiveCfg = Release|Win32 19 | {E77E7CFB-E02E-4F25-ADAA-1CF587DB0A04}.Release|Win32.Build.0 = Release|Win32 20 | {E77E7CFB-E02E-4F25-ADAA-1CF587DB0A04}.Release|x64.ActiveCfg = Release|x64 21 | {E77E7CFB-E02E-4F25-ADAA-1CF587DB0A04}.Release|x64.Build.0 = Release|x64 22 | EndGlobalSection 23 | GlobalSection(SolutionProperties) = preSolution 24 | HideSolutionNode = FALSE 25 | EndGlobalSection 26 | EndGlobal 27 | -------------------------------------------------------------------------------- /native/runtime/tommyds/benchmark/gr_all.bat: -------------------------------------------------------------------------------- 1 | rem Set it to a GNUPLOT 4.4 binary 2 | rem In case download it from http://sourceforge.net/projects/gnuplot/files/gnuplot/4.4.2/gp442win32.zip/download 3 | set GNUPLOT=..\contrib\gnuplot\binary\gnuplot.exe 4 | 5 | %GNUPLOT% gr_def_random_hit.gnu 6 | %GNUPLOT% gr_def_random_change.gnu 7 | %GNUPLOT% gr_other_judy_problem.gnu 8 | %GNUPLOT% gr_other_slow_problem.gnu 9 | 10 | DIR=data\core_i5_650_3G2_linux 11 | %GNUPLOT% %DIR%\gr_def.gnu gr_forward_insert.gnu 12 | %GNUPLOT% %DIR%\gr_def.gnu gr_forward_hit.gnu 13 | %GNUPLOT% %DIR%\gr_def.gnu gr_forward_insert.gnu 14 | %GNUPLOT% %DIR%\gr_def.gnu gr_forward_miss.gnu 15 | %GNUPLOT% %DIR%\gr_def.gnu gr_forward_change.gnu 16 | %GNUPLOT% %DIR%\gr_def.gnu gr_forward_remove.gnu 17 | %GNUPLOT% %DIR%\gr_def.gnu gr_forward_size.gnu 18 | %GNUPLOT% %DIR%\gr_def.gnu gr_random_hit.gnu 19 | %GNUPLOT% %DIR%\gr_def.gnu gr_random_insert.gnu 20 | %GNUPLOT% %DIR%\gr_def.gnu gr_random_miss.gnu 21 | %GNUPLOT% %DIR%\gr_def.gnu gr_random_change.gnu 22 | %GNUPLOT% %DIR%\gr_def.gnu gr_random_remove.gnu 23 | %GNUPLOT% %DIR%\gr_def.gnu gr_random_size.gnu 24 | 25 | DIR=data\core_i7_3740_2G7_win 26 | %GNUPLOT% %DIR%\gr_def.gnu gr_forward_insert.gnu 27 | %GNUPLOT% %DIR%\gr_def.gnu gr_forward_hit.gnu 28 | %GNUPLOT% %DIR%\gr_def.gnu gr_forward_insert.gnu 29 | %GNUPLOT% %DIR%\gr_def.gnu gr_forward_miss.gnu 30 | %GNUPLOT% %DIR%\gr_def.gnu gr_forward_change.gnu 31 | %GNUPLOT% %DIR%\gr_def.gnu gr_forward_remove.gnu 32 | %GNUPLOT% %DIR%\gr_def.gnu gr_forward_size.gnu 33 | %GNUPLOT% %DIR%\gr_def.gnu gr_random_hit.gnu 34 | %GNUPLOT% %DIR%\gr_def.gnu gr_random_insert.gnu 35 | %GNUPLOT% %DIR%\gr_def.gnu gr_random_miss.gnu 36 | %GNUPLOT% %DIR%\gr_def.gnu gr_random_change.gnu 37 | %GNUPLOT% %DIR%\gr_def.gnu gr_random_remove.gnu 38 | %GNUPLOT% %DIR%\gr_def.gnu gr_random_size.gnu 39 | 40 | DIR=data\test 41 | %GNUPLOT% %DIR%\gr_def.gnu gr_forward_insert.gnu 42 | %GNUPLOT% %DIR%\gr_def.gnu gr_forward_hit.gnu 43 | %GNUPLOT% %DIR%\gr_def.gnu gr_forward_insert.gnu 44 | %GNUPLOT% %DIR%\gr_def.gnu gr_forward_miss.gnu 45 | %GNUPLOT% %DIR%\gr_def.gnu gr_forward_change.gnu 46 | %GNUPLOT% %DIR%\gr_def.gnu gr_forward_remove.gnu 47 | %GNUPLOT% %DIR%\gr_def.gnu gr_forward_size.gnu 48 | %GNUPLOT% %DIR%\gr_def.gnu gr_random_hit.gnu 49 | %GNUPLOT% %DIR%\gr_def.gnu gr_random_insert.gnu 50 | %GNUPLOT% %DIR%\gr_def.gnu gr_random_miss.gnu 51 | %GNUPLOT% %DIR%\gr_def.gnu gr_random_change.gnu 52 | %GNUPLOT% %DIR%\gr_def.gnu gr_random_remove.gnu 53 | %GNUPLOT% %DIR%\gr_def.gnu gr_random_size.gnu 54 | 55 | -------------------------------------------------------------------------------- /native/runtime/tommyds/benchmark/gr_all.sh: -------------------------------------------------------------------------------- 1 | echo Ensure to use GNUPLOT 4.4 2 | 3 | export GDFONTPATH=. 4 | export GNUPLOT_DEFAULT_GDFONT=arial 5 | 6 | gnuplot gr_def_random_hit.gnu 7 | gnuplot gr_def_random_change.gnu 8 | gnuplot gr_other_judy_problem.gnu 9 | gnuplot gr_other_googlelibchash_problem.gnu 10 | gnuplot gr_other_ck_problem.gnu 11 | 12 | DIR=data/core_i5_650_3G2_linux 13 | gnuplot $DIR/gr_def.gnu gr_forward_insert.gnu 14 | gnuplot $DIR/gr_def.gnu gr_forward_hit.gnu 15 | gnuplot $DIR/gr_def.gnu gr_forward_insert.gnu 16 | gnuplot $DIR/gr_def.gnu gr_forward_miss.gnu 17 | gnuplot $DIR/gr_def.gnu gr_forward_change.gnu 18 | gnuplot $DIR/gr_def.gnu gr_forward_remove.gnu 19 | gnuplot $DIR/gr_def.gnu gr_forward_size.gnu 20 | gnuplot $DIR/gr_def.gnu gr_random_hit.gnu 21 | gnuplot $DIR/gr_def.gnu gr_random_insert.gnu 22 | gnuplot $DIR/gr_def.gnu gr_random_miss.gnu 23 | gnuplot $DIR/gr_def.gnu gr_random_change.gnu 24 | gnuplot $DIR/gr_def.gnu gr_random_remove.gnu 25 | gnuplot $DIR/gr_def.gnu gr_random_size.gnu 26 | 27 | DIR=data/core_i7_3740_2G7_linux 28 | gnuplot $DIR/gr_def.gnu gr_forward_insert.gnu 29 | gnuplot $DIR/gr_def.gnu gr_forward_hit.gnu 30 | gnuplot $DIR/gr_def.gnu gr_forward_insert.gnu 31 | gnuplot $DIR/gr_def.gnu gr_forward_miss.gnu 32 | gnuplot $DIR/gr_def.gnu gr_forward_change.gnu 33 | gnuplot $DIR/gr_def.gnu gr_forward_remove.gnu 34 | gnuplot $DIR/gr_def.gnu gr_forward_size.gnu 35 | gnuplot $DIR/gr_def.gnu gr_random_hit.gnu 36 | gnuplot $DIR/gr_def.gnu gr_random_insert.gnu 37 | gnuplot $DIR/gr_def.gnu gr_random_miss.gnu 38 | gnuplot $DIR/gr_def.gnu gr_random_change.gnu 39 | gnuplot $DIR/gr_def.gnu gr_random_remove.gnu 40 | gnuplot $DIR/gr_def.gnu gr_random_size.gnu 41 | 42 | DIR=data/core_i7_3740_2G7_win 43 | gnuplot $DIR/gr_def.gnu gr_forward_insert.gnu 44 | gnuplot $DIR/gr_def.gnu gr_forward_hit.gnu 45 | gnuplot $DIR/gr_def.gnu gr_forward_insert.gnu 46 | gnuplot $DIR/gr_def.gnu gr_forward_miss.gnu 47 | gnuplot $DIR/gr_def.gnu gr_forward_change.gnu 48 | gnuplot $DIR/gr_def.gnu gr_forward_remove.gnu 49 | gnuplot $DIR/gr_def.gnu gr_forward_size.gnu 50 | gnuplot $DIR/gr_def.gnu gr_random_hit.gnu 51 | gnuplot $DIR/gr_def.gnu gr_random_insert.gnu 52 | gnuplot $DIR/gr_def.gnu gr_random_miss.gnu 53 | gnuplot $DIR/gr_def.gnu gr_random_change.gnu 54 | gnuplot $DIR/gr_def.gnu gr_random_remove.gnu 55 | gnuplot $DIR/gr_def.gnu gr_random_size.gnu 56 | 57 | DIR=data/test 58 | gnuplot $DIR/gr_def.gnu gr_forward_insert.gnu 59 | gnuplot $DIR/gr_def.gnu gr_forward_hit.gnu 60 | gnuplot $DIR/gr_def.gnu gr_forward_insert.gnu 61 | gnuplot $DIR/gr_def.gnu gr_forward_miss.gnu 62 | gnuplot $DIR/gr_def.gnu gr_forward_change.gnu 63 | gnuplot $DIR/gr_def.gnu gr_forward_remove.gnu 64 | gnuplot $DIR/gr_def.gnu gr_forward_size.gnu 65 | gnuplot $DIR/gr_def.gnu gr_random_hit.gnu 66 | gnuplot $DIR/gr_def.gnu gr_random_insert.gnu 67 | gnuplot $DIR/gr_def.gnu gr_random_miss.gnu 68 | gnuplot $DIR/gr_def.gnu gr_random_change.gnu 69 | gnuplot $DIR/gr_def.gnu gr_random_remove.gnu 70 | gnuplot $DIR/gr_def.gnu gr_random_size.gnu 71 | 72 | -------------------------------------------------------------------------------- /native/runtime/tommyds/benchmark/gr_common.gnu: -------------------------------------------------------------------------------- 1 | ;set term svg enhanced font 'arial,10' size 800,800 2 | ;bext = ".svg" 3 | set terminal png nocrop enhanced font arial 10 size 800,800 4 | bext = ".png" 5 | set xtics nomirror rotate by -45 font "arial,8" 6 | set key below 7 | set style data linespoints 8 | set datafile missing "0" 9 | set xlabel "Number of elements in logarithmic scale" 10 | set ylabel "Time for element in nanosecond in logarithmic scale\nLower is better" 11 | set xrange [1000:10000000] 12 | set yrange [6:1000] 13 | set logscale y 14 | set logscale x 15 | set format y "%.0fns" 16 | set format x "%.0s%c" 17 | bdir = "data/" 18 | 19 | # for some colors see: http://www.uni-hamburg.de/Wiss/FB/15/Sustainability/schneider/gnuplot/colors.htm 20 | set style line 1 lc 1 lt 1 # hashtable 21 | set style line 2 lc 2 lt 2 # hashdyn 22 | set style line 3 lc 3 lt 3 # hashlin 23 | set style line 4 lc 4 lt 4 # trie 24 | set style line 5 lc 5 lt 5 # trie-inplace 25 | set style line 6 lc 6 lt 6 # rbtre 26 | set style line 7 lc 7 lt 7 # nedtrie 27 | set style line 8 lc 8 lt 8 # khash 28 | set style line 9 lc 9 lt 8 # uthash 29 | set style line 10 lc 10 lt 10 # judy 30 | set style line 11 lc 11 lt 11 # judyarray 31 | set style line 12 lc 12 lt 12 # googledensehash 32 | set style line 13 lc rgb "#FF69B4" lt 13 # googlebtree 33 | set style line 14 lc 14 lt 14 # stxbtree 34 | set style line 15 lc 15 lt 15 # c++unorderedmap 35 | set style line 16 lc 16 lt 16 # c++map 36 | set style line 17 lc 17 lt 17 # tesseract 37 | set style line 18 lc 4 lt 18 pt 1 # libdynamic 38 | set style line 19 lc rgb "#FF69B4" lt 19 # googlelibchash 39 | set style line 20 lc rgb "#1E90FF" lt 20 # concurrencykit 40 | 41 | -------------------------------------------------------------------------------- /native/runtime/tommyds/benchmark/gr_def_random_change.gnu: -------------------------------------------------------------------------------- 1 | load "gr_common.gnu" 2 | 3 | tdir = "def/" 4 | tsub = "\nCore i5 650 3.20 GHz, 4 MB L3 cache, 2400 Uncore Speed\nLinux, gcc 6.2.0, 32 bit" 5 | 6 | set output bdir.tdir."img_random_change".bext 7 | set title "Random Change (Remove + Insert)".tsub 8 | data = bdir.tdir.'dat_random_change.lst' 9 | 10 | plot data using 1:2 title columnheader(2), \ 11 | for [i=3:21] '' using 1:i title columnheader(i) ls i-1 12 | 13 | -------------------------------------------------------------------------------- /native/runtime/tommyds/benchmark/gr_def_random_hit.gnu: -------------------------------------------------------------------------------- 1 | load "gr_common.gnu" 2 | 3 | tdir = "def/" 4 | tsub = "\nCore i5 650 3.20 GHz, 4 MB L3 cache, 2400 Uncore Speed\nLinux, gcc 6.2.0, 32 bit" 5 | 6 | set output bdir.tdir."img_random_hit".bext 7 | set title "Random Hit".tsub 8 | data = bdir.tdir.'dat_random_hit.lst' 9 | 10 | plot data using 1:2 title columnheader(2), \ 11 | for [i=3:21] '' using 1:i title columnheader(i) ls i-1 12 | 13 | -------------------------------------------------------------------------------- /native/runtime/tommyds/benchmark/gr_forward_change.gnu: -------------------------------------------------------------------------------- 1 | load "gr_common.gnu" 2 | 3 | set output bdir.tdir."img_forward_change".bext 4 | set title "Forward Change (Remove + Insert)".tsub 5 | data = bdir.tdir.'dat_forward_change.lst' 6 | 7 | plot data using 1:2 title columnheader(2), \ 8 | for [i=3:21] '' using 1:i title columnheader(i) ls i-1 9 | 10 | -------------------------------------------------------------------------------- /native/runtime/tommyds/benchmark/gr_forward_hit.gnu: -------------------------------------------------------------------------------- 1 | load "gr_common.gnu" 2 | 3 | set output bdir.tdir."img_forward_hit".bext 4 | set title "Forward Hit".tsub 5 | data = bdir.tdir.'dat_forward_hit.lst' 6 | 7 | plot data using 1:2 title columnheader(2), \ 8 | for [i=3:21] '' using 1:i title columnheader(i) ls i-1 9 | 10 | -------------------------------------------------------------------------------- /native/runtime/tommyds/benchmark/gr_forward_insert.gnu: -------------------------------------------------------------------------------- 1 | load "gr_common.gnu" 2 | 3 | set output bdir.tdir."img_forward_insert".bext 4 | set title "Forward Insert".tsub 5 | data = bdir.tdir.'dat_forward_insert.lst' 6 | 7 | plot data using 1:2 title columnheader(2), \ 8 | for [i=3:21] '' using 1:i title columnheader(i) ls i-1 9 | 10 | -------------------------------------------------------------------------------- /native/runtime/tommyds/benchmark/gr_forward_miss.gnu: -------------------------------------------------------------------------------- 1 | load "gr_common.gnu" 2 | 3 | set output bdir.tdir."img_forward_miss".bext 4 | set title "Forward Miss".tsub 5 | data = bdir.tdir.'dat_forward_miss.lst' 6 | 7 | plot data using 1:2 title columnheader(2), \ 8 | for [i=3:21] '' using 1:i title columnheader(i) ls i-1 9 | 10 | -------------------------------------------------------------------------------- /native/runtime/tommyds/benchmark/gr_forward_remove.gnu: -------------------------------------------------------------------------------- 1 | load "gr_common.gnu" 2 | 3 | set output bdir.tdir."img_forward_remove".bext 4 | set title "Forward Remove".tsub 5 | data = bdir.tdir.'dat_forward_remove.lst' 6 | 7 | plot data using 1:2 title columnheader(2), \ 8 | for [i=3:21] '' using 1:i title columnheader(i) ls i-1 9 | 10 | -------------------------------------------------------------------------------- /native/runtime/tommyds/benchmark/gr_forward_size.gnu: -------------------------------------------------------------------------------- 1 | load "gr_common.gnu" 2 | 3 | set output bdir.tdir."img_forward_size".bext 4 | set title "Size".tsub 5 | set format y "%.0f" 6 | set ylabel "Size for element in byte\nLower is better" 7 | unset logscale y 8 | set yrange [0:80] 9 | data = bdir.tdir.'dat_forward_size.lst' 10 | 11 | plot data using 1:2 title columnheader(2), \ 12 | for [i=3:21] '' using 1:i title columnheader(i) ls i-1 13 | 14 | -------------------------------------------------------------------------------- /native/runtime/tommyds/benchmark/gr_other_ck_problem.gnu: -------------------------------------------------------------------------------- 1 | load "gr_common.gnu" 2 | 3 | set yrange [10:10000] 4 | 5 | tdir = "other/" 6 | tsub = "\nCore i5 650 3.20 GHz, 4 MB L3 cache\nLinux, gcc 4.7.1, 32 bit" 7 | 8 | set output bdir.tdir."ck_problem".bext 9 | set title "Random Change (Remove + Insert)".tsub 10 | data = bdir.tdir.'ck_problem.lst' 11 | 12 | plot data using 1:2 title columnheader(2), \ 13 | for [i=3:21] '' using 1:i title columnheader(i) ls i-1 14 | 15 | -------------------------------------------------------------------------------- /native/runtime/tommyds/benchmark/gr_other_googlelibchash_problem.gnu: -------------------------------------------------------------------------------- 1 | load "gr_common.gnu" 2 | 3 | set yrange [10:10000] 4 | 5 | tdir = "other/" 6 | tsub = "\nCore i5 650 3.20 GHz, 4 MB L3 cache\nLinux, gcc 4.7.1, 32 bit" 7 | 8 | set output bdir.tdir."googlelibchash_problem".bext 9 | set title "Random Change (Remove + Insert)".tsub 10 | data = bdir.tdir.'googlelibchash_problem.lst' 11 | 12 | plot data using 1:2 title columnheader(2), \ 13 | for [i=3:21] '' using 1:i title columnheader(i) ls i-1 14 | 15 | -------------------------------------------------------------------------------- /native/runtime/tommyds/benchmark/gr_other_judy_problem.gnu: -------------------------------------------------------------------------------- 1 | load "gr_common.gnu" 2 | 3 | set yrange [10:10000] 4 | 5 | tdir = "other/" 6 | tsub = "\nXeon E5430 2.66 GHz, 2x6 MB L2 cache, 1333 MT/s FSB\nWindows, Visual C 2008, 32 bit" 7 | 8 | set output bdir.tdir."judy_problem".bext 9 | set title "Forward Change (Remove + Insert)".tsub 10 | data = bdir.tdir.'judy_problem.lst' 11 | 12 | plot data using 1:2 title columnheader(2), \ 13 | for [i=3:21] '' using 1:i title columnheader(i) ls i-1 14 | 15 | -------------------------------------------------------------------------------- /native/runtime/tommyds/benchmark/gr_random_change.gnu: -------------------------------------------------------------------------------- 1 | load "gr_common.gnu" 2 | 3 | set output bdir.tdir."img_random_change".bext 4 | set title "Random Change (Remove + Insert)".tsub 5 | data = bdir.tdir.'dat_random_change.lst' 6 | 7 | plot data using 1:2 title columnheader(2), \ 8 | for [i=3:21] '' using 1:i title columnheader(i) ls i-1 9 | 10 | -------------------------------------------------------------------------------- /native/runtime/tommyds/benchmark/gr_random_hit.gnu: -------------------------------------------------------------------------------- 1 | load "gr_common.gnu" 2 | 3 | set output bdir.tdir."img_random_hit".bext 4 | set title "Random Hit".tsub 5 | data = bdir.tdir.'dat_random_hit.lst' 6 | 7 | plot data using 1:2 title columnheader(2), \ 8 | for [i=3:21] '' using 1:i title columnheader(i) ls i-1 9 | 10 | -------------------------------------------------------------------------------- /native/runtime/tommyds/benchmark/gr_random_insert.gnu: -------------------------------------------------------------------------------- 1 | load "gr_common.gnu" 2 | 3 | set output bdir.tdir."img_random_insert".bext 4 | set title "Random Insert".tsub 5 | data = bdir.tdir.'dat_random_insert.lst' 6 | 7 | plot data using 1:2 title columnheader(2), \ 8 | for [i=3:21] '' using 1:i title columnheader(i) ls i-1 9 | 10 | -------------------------------------------------------------------------------- /native/runtime/tommyds/benchmark/gr_random_miss.gnu: -------------------------------------------------------------------------------- 1 | load "gr_common.gnu" 2 | 3 | set output bdir.tdir."img_random_miss".bext 4 | set title "Random Miss".tsub 5 | data = bdir.tdir.'dat_random_miss.lst' 6 | 7 | plot data using 1:2 title columnheader(2), \ 8 | for [i=3:21] '' using 1:i title columnheader(i) ls i-1 9 | 10 | -------------------------------------------------------------------------------- /native/runtime/tommyds/benchmark/gr_random_remove.gnu: -------------------------------------------------------------------------------- 1 | load "gr_common.gnu" 2 | 3 | set output bdir.tdir."img_random_remove".bext 4 | set title "Random Remove".tsub 5 | data = bdir.tdir.'dat_random_remove.lst' 6 | 7 | plot data using 1:2 title columnheader(2), \ 8 | for [i=3:21] '' using 1:i title columnheader(i) ls i-1 9 | 10 | -------------------------------------------------------------------------------- /native/runtime/tommyds/benchmark/gr_random_size.gnu: -------------------------------------------------------------------------------- 1 | load "gr_common.gnu" 2 | 3 | set output bdir.tdir."img_random_size".bext 4 | set title "Size".tsub 5 | set format y "%.0f" 6 | set ylabel "Size for element in byte\nLower is better" 7 | unset logscale y 8 | set yrange [0:80] 9 | data = bdir.tdir.'dat_random_size.lst' 10 | 11 | plot data using 1:2 title columnheader(2), \ 12 | for [i=3:21] '' using 1:i title columnheader(i) ls i-1 13 | 14 | -------------------------------------------------------------------------------- /native/runtime/tommyds/makecov.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Run the Coverage test 4 | # 5 | 6 | make distclean 7 | 8 | if ! make CC=gcc COVERAGE=1 lcov_reset check lcov_capture lcov_html; then 9 | exit 1 10 | fi 11 | 12 | 13 | -------------------------------------------------------------------------------- /native/runtime/tommyds/makescan.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | 4 | rm -r cov-int 5 | rm tommyds.tgz 6 | 7 | make distclean 8 | 9 | export PATH=$PATH:../snapraid/contrib/cov-analysis-linux64-2017.07/bin 10 | 11 | if ! cov-build --dir cov-int make; then 12 | exit 1 13 | fi 14 | 15 | tar czf tommyds.tgz cov-int 16 | 17 | rm -r cov-int 18 | 19 | echo tommyds.tgz ready to upload to https://scan.coverity.com/projects/3780/builds/new 20 | 21 | -------------------------------------------------------------------------------- /native/runtime/tommyds/tommy-footer.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /native/runtime/tommyds/tommy-header.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | TommyDS 7 | 8 | 9 | 10 | 11 |
12 | 13 | -------------------------------------------------------------------------------- /native/runtime/tommyds/tommyds/tommy.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010, Andrea Mazzoleni. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 | * POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | 28 | #include "tommyhash.c" 29 | #include "tommyalloc.c" 30 | #include "tommyarray.c" 31 | #include "tommyarrayof.c" 32 | #include "tommyarrayblk.c" 33 | #include "tommyarrayblkof.c" 34 | #include "tommylist.c" 35 | #include "tommytree.c" 36 | #include "tommytrie.c" 37 | #include "tommytrieinp.c" 38 | #include "tommyhashtbl.c" 39 | #include "tommyhashdyn.c" 40 | #include "tommyhashlin.c" 41 | 42 | -------------------------------------------------------------------------------- /native/runtime/tommyds/tommyds/tommyalloc.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010, Andrea Mazzoleni. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 | * POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | 28 | #include "tommyalloc.h" 29 | 30 | /******************************************************************************/ 31 | /* allocator */ 32 | 33 | /** 34 | * Basic allocation segment. 35 | * Smaller of a memory page, to allow also a little heap overread. 36 | * The heap manager may put it in a single memory page. 37 | */ 38 | #define TOMMY_ALLOCATOR_BLOCK_SIZE (4096 - 64) 39 | 40 | void tommy_allocator_init(tommy_allocator* alloc, tommy_size_t block_size, tommy_size_t align_size) 41 | { 42 | /* setup the minimal alignment */ 43 | if (align_size < sizeof(void*)) 44 | align_size = sizeof(void*); 45 | 46 | /* ensure that the block_size keeps the alignment */ 47 | if (block_size % align_size != 0) 48 | block_size += align_size - block_size % align_size; 49 | 50 | alloc->block_size = block_size; 51 | alloc->align_size = align_size; 52 | 53 | alloc->count = 0; 54 | alloc->free_block = 0; 55 | alloc->used_segment = 0; 56 | } 57 | 58 | /** 59 | * Reset the allocator and free all. 60 | */ 61 | static void allocator_reset(tommy_allocator* alloc) 62 | { 63 | tommy_allocator_entry* block = alloc->used_segment; 64 | 65 | while (block) { 66 | tommy_allocator_entry* block_next = block->next; 67 | tommy_free(block); 68 | block = block_next; 69 | } 70 | 71 | alloc->count = 0; 72 | alloc->free_block = 0; 73 | alloc->used_segment = 0; 74 | } 75 | 76 | void tommy_allocator_done(tommy_allocator* alloc) 77 | { 78 | allocator_reset(alloc); 79 | } 80 | 81 | void* tommy_allocator_alloc(tommy_allocator* alloc) 82 | { 83 | void* ptr; 84 | 85 | /* if no free block available */ 86 | if (!alloc->free_block) { 87 | tommy_uintptr_t off, mis; 88 | tommy_size_t size; 89 | char* data; 90 | tommy_allocator_entry* segment; 91 | 92 | /* default allocation size */ 93 | size = TOMMY_ALLOCATOR_BLOCK_SIZE; 94 | 95 | /* ensure that we can allocate at least one block */ 96 | if (size < sizeof(tommy_allocator_entry) + alloc->align_size + alloc->block_size) 97 | size = sizeof(tommy_allocator_entry) + alloc->align_size + alloc->block_size; 98 | 99 | data = tommy_cast(char*, tommy_malloc(size)); 100 | segment = (tommy_allocator_entry*)data; 101 | 102 | /* put in the segment list */ 103 | segment->next = alloc->used_segment; 104 | alloc->used_segment = segment; 105 | data += sizeof(tommy_allocator_entry); 106 | 107 | /* align if not aligned */ 108 | off = (tommy_uintptr_t)data; 109 | mis = off % alloc->align_size; 110 | if (mis != 0) { 111 | data += alloc->align_size - mis; 112 | size -= alloc->align_size - mis; 113 | } 114 | 115 | /* insert in free list */ 116 | do { 117 | tommy_allocator_entry* free_block = (tommy_allocator_entry*)data; 118 | free_block->next = alloc->free_block; 119 | alloc->free_block = free_block; 120 | 121 | data += alloc->block_size; 122 | size -= alloc->block_size; 123 | } while (size >= alloc->block_size); 124 | } 125 | 126 | /* remove one from the free list */ 127 | ptr = alloc->free_block; 128 | alloc->free_block = alloc->free_block->next; 129 | 130 | ++alloc->count; 131 | 132 | return ptr; 133 | } 134 | 135 | void tommy_allocator_free(tommy_allocator* alloc, void* ptr) 136 | { 137 | tommy_allocator_entry* free_block = tommy_cast(tommy_allocator_entry*, ptr); 138 | 139 | /* put it in the free list */ 140 | free_block->next = alloc->free_block; 141 | alloc->free_block = free_block; 142 | 143 | --alloc->count; 144 | } 145 | 146 | tommy_size_t tommy_allocator_memory_usage(tommy_allocator* alloc) 147 | { 148 | return alloc->count * (tommy_size_t)alloc->block_size; 149 | } 150 | 151 | -------------------------------------------------------------------------------- /native/runtime/tommyds/tommyds/tommyalloc.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010, Andrea Mazzoleni. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 | * POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | 28 | /** \file 29 | * Allocator of fixed size blocks. 30 | */ 31 | 32 | #ifndef __TOMMYALLOC_H 33 | #define __TOMMYALLOC_H 34 | 35 | #include "tommytypes.h" 36 | 37 | /******************************************************************************/ 38 | /* allocator */ 39 | 40 | /** \internal 41 | * Allocator entry. 42 | */ 43 | struct tommy_allocator_entry_struct { 44 | struct tommy_allocator_entry_struct* next; /**< Pointer to the next entry. 0 for last. */ 45 | }; 46 | typedef struct tommy_allocator_entry_struct tommy_allocator_entry; 47 | 48 | /** 49 | * Allocator of fixed size blocks. 50 | */ 51 | typedef struct tommy_allocator_struct { 52 | struct tommy_allocator_entry_struct* free_block; /**< List of free blocks. */ 53 | struct tommy_allocator_entry_struct* used_segment; /**< List of allocated segments. */ 54 | tommy_size_t block_size; /**< Block size. */ 55 | tommy_size_t align_size; /**< Alignment size. */ 56 | tommy_size_t count; /**< Number of allocated elements. */ 57 | } tommy_allocator; 58 | 59 | /** 60 | * Initializes the allocator. 61 | * \param alloc Allocator to initialize. 62 | * \param block_size Size of the block to allocate. 63 | * \param align_size Minimum alignment requirement. No less than sizeof(void*). 64 | */ 65 | void tommy_allocator_init(tommy_allocator* alloc, tommy_size_t block_size, tommy_size_t align_size); 66 | 67 | /** 68 | * Deinitialize the allocator. 69 | * It also releases all the allocated memory to the heap. 70 | * \param alloc Allocator to deinitialize. 71 | */ 72 | void tommy_allocator_done(tommy_allocator* alloc); 73 | 74 | /** 75 | * Allocates a block. 76 | * \param alloc Allocator to use. 77 | */ 78 | void* tommy_allocator_alloc(tommy_allocator* alloc); 79 | 80 | /** 81 | * Deallocates a block. 82 | * You must use the same allocator used in the tommy_allocator_alloc() call. 83 | * \param alloc Allocator to use. 84 | * \param ptr Block to free. 85 | */ 86 | void tommy_allocator_free(tommy_allocator* alloc, void* ptr); 87 | 88 | /** 89 | * Gets the size of allocated memory. 90 | * \param alloc Allocator to use. 91 | */ 92 | tommy_size_t tommy_allocator_memory_usage(tommy_allocator* alloc); 93 | 94 | #endif 95 | 96 | -------------------------------------------------------------------------------- /native/runtime/tommyds/tommyds/tommyarray.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2011, Andrea Mazzoleni. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 | * POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | 28 | #include "tommyarray.h" 29 | 30 | /******************************************************************************/ 31 | /* array */ 32 | 33 | void tommy_array_init(tommy_array* array) 34 | { 35 | tommy_uint_t i; 36 | 37 | /* fixed initial size */ 38 | array->bucket_bit = TOMMY_ARRAY_BIT; 39 | array->bucket_max = (tommy_size_t)1 << array->bucket_bit; 40 | array->bucket[0] = tommy_cast(void**, tommy_calloc(array->bucket_max, sizeof(void*))); 41 | for (i = 1; i < TOMMY_ARRAY_BIT; ++i) 42 | array->bucket[i] = array->bucket[0]; 43 | 44 | array->count = 0; 45 | } 46 | 47 | void tommy_array_done(tommy_array* array) 48 | { 49 | tommy_uint_t i; 50 | 51 | tommy_free(array->bucket[0]); 52 | for (i = TOMMY_ARRAY_BIT; i < array->bucket_bit; ++i) { 53 | void** segment = array->bucket[i]; 54 | tommy_free(&segment[(tommy_ptrdiff_t)1 << i]); 55 | } 56 | } 57 | 58 | void tommy_array_grow(tommy_array* array, tommy_size_t count) 59 | { 60 | if (array->count >= count) 61 | return; 62 | array->count = count; 63 | 64 | while (count > array->bucket_max) { 65 | void** segment; 66 | 67 | /* allocate one more segment */ 68 | segment = tommy_cast(void**, tommy_calloc(array->bucket_max, sizeof(void*))); 69 | 70 | /* store it adjusting the offset */ 71 | /* cast to ptrdiff_t to ensure to get a negative value */ 72 | array->bucket[array->bucket_bit] = &segment[-(tommy_ptrdiff_t)array->bucket_max]; 73 | 74 | ++array->bucket_bit; 75 | array->bucket_max = (tommy_size_t)1 << array->bucket_bit; 76 | } 77 | } 78 | 79 | tommy_size_t tommy_array_memory_usage(tommy_array* array) 80 | { 81 | return array->bucket_max * (tommy_size_t)sizeof(void*); 82 | } 83 | 84 | -------------------------------------------------------------------------------- /native/runtime/tommyds/tommyds/tommyarray.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2011, Andrea Mazzoleni. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 | * POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | 28 | /** \file 29 | * Dynamic array based on segments of exponential growing size. 30 | * 31 | * This array is able to grow dynamically upon request, without any reallocation. 32 | * 33 | * The grow operation involves an allocation of a new array segment, without reallocating 34 | * the already used memory, and then not increasing the heap fragmentation. 35 | * This also implies that the address of the stored elements never change. 36 | * 37 | * Allocated segments grow in size exponentially. 38 | */ 39 | 40 | #ifndef __TOMMYARRAY_H 41 | #define __TOMMYARRAY_H 42 | 43 | #include "tommytypes.h" 44 | 45 | #include /* for assert */ 46 | 47 | /******************************************************************************/ 48 | /* array */ 49 | 50 | /** 51 | * Initial and minimal size of the array expressed as a power of 2. 52 | * The initial size is 2^TOMMY_ARRAY_BIT. 53 | */ 54 | #define TOMMY_ARRAY_BIT 6 55 | 56 | /** 57 | * Array container type. 58 | * \note Don't use internal fields directly, but access the container only using functions. 59 | */ 60 | typedef struct tommy_array_struct { 61 | void** bucket[TOMMY_SIZE_BIT]; /**< Dynamic array of buckets. */ 62 | tommy_size_t bucket_max; /**< Number of buckets. */ 63 | tommy_size_t count; /**< Number of initialized elements in the array. */ 64 | tommy_uint_t bucket_bit; /**< Bits used in the bit mask. */ 65 | } tommy_array; 66 | 67 | /** 68 | * Initializes the array. 69 | */ 70 | void tommy_array_init(tommy_array* array); 71 | 72 | /** 73 | * Deinitializes the array. 74 | */ 75 | void tommy_array_done(tommy_array* array); 76 | 77 | /** 78 | * Grows the size up to the specified value. 79 | * All the new elements in the array are initialized with the 0 value. 80 | */ 81 | void tommy_array_grow(tommy_array* array, tommy_size_t size); 82 | 83 | /** 84 | * Gets a reference of the element at the specified position. 85 | * You must be sure that space for this position is already 86 | * allocated calling tommy_array_grow(). 87 | */ 88 | tommy_inline void** tommy_array_ref(tommy_array* array, tommy_size_t pos) 89 | { 90 | tommy_uint_t bsr; 91 | 92 | assert(pos < array->count); 93 | 94 | /* get the highest bit set, in case of all 0, return 0 */ 95 | bsr = tommy_ilog2(pos | 1); 96 | 97 | return &array->bucket[bsr][pos]; 98 | } 99 | 100 | /** 101 | * Sets the element at the specified position. 102 | * You must be sure that space for this position is already 103 | * allocated calling tommy_array_grow(). 104 | */ 105 | tommy_inline void tommy_array_set(tommy_array* array, tommy_size_t pos, void* element) 106 | { 107 | *tommy_array_ref(array, pos) = element; 108 | } 109 | 110 | /** 111 | * Gets the element at the specified position. 112 | * You must be sure that space for this position is already 113 | * allocated calling tommy_array_grow(). 114 | */ 115 | tommy_inline void* tommy_array_get(tommy_array* array, tommy_size_t pos) 116 | { 117 | return *tommy_array_ref(array, pos); 118 | } 119 | 120 | /** 121 | * Grows and inserts a new element at the end of the array. 122 | */ 123 | tommy_inline void tommy_array_insert(tommy_array* array, void* element) 124 | { 125 | tommy_size_t pos = array->count; 126 | 127 | tommy_array_grow(array, pos + 1); 128 | 129 | tommy_array_set(array, pos, element); 130 | } 131 | 132 | /** 133 | * Gets the initialized size of the array. 134 | */ 135 | tommy_inline tommy_size_t tommy_array_size(tommy_array* array) 136 | { 137 | return array->count; 138 | } 139 | 140 | /** 141 | * Gets the size of allocated memory. 142 | */ 143 | tommy_size_t tommy_array_memory_usage(tommy_array* array); 144 | 145 | #endif 146 | 147 | -------------------------------------------------------------------------------- /native/runtime/tommyds/tommyds/tommyarrayblk.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, Andrea Mazzoleni. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 | * POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | 28 | #include "tommyarrayblk.h" 29 | 30 | /******************************************************************************/ 31 | /* array */ 32 | 33 | void tommy_arrayblk_init(tommy_arrayblk* array) 34 | { 35 | tommy_array_init(&array->block); 36 | 37 | array->count = 0; 38 | } 39 | 40 | void tommy_arrayblk_done(tommy_arrayblk* array) 41 | { 42 | tommy_size_t i; 43 | 44 | for (i = 0; i < tommy_array_size(&array->block); ++i) 45 | tommy_free(tommy_array_get(&array->block, i)); 46 | 47 | tommy_array_done(&array->block); 48 | } 49 | 50 | void tommy_arrayblk_grow(tommy_arrayblk* array, tommy_size_t count) 51 | { 52 | tommy_size_t block_max; 53 | tommy_size_t block_mac; 54 | 55 | if (array->count >= count) 56 | return; 57 | array->count = count; 58 | 59 | block_max = (count + TOMMY_ARRAYBLK_SIZE - 1) / TOMMY_ARRAYBLK_SIZE; 60 | block_mac = tommy_array_size(&array->block); 61 | 62 | if (block_mac < block_max) { 63 | /* grow the block array */ 64 | tommy_array_grow(&array->block, block_max); 65 | 66 | /* allocate new blocks */ 67 | while (block_mac < block_max) { 68 | void** ptr = tommy_cast(void**, tommy_calloc(TOMMY_ARRAYBLK_SIZE, sizeof(void*))); 69 | 70 | /* set the new block */ 71 | tommy_array_set(&array->block, block_mac, ptr); 72 | 73 | ++block_mac; 74 | } 75 | } 76 | } 77 | 78 | tommy_size_t tommy_arrayblk_memory_usage(tommy_arrayblk* array) 79 | { 80 | return tommy_array_memory_usage(&array->block) + tommy_array_size(&array->block) * TOMMY_ARRAYBLK_SIZE * sizeof(void*); 81 | } 82 | 83 | -------------------------------------------------------------------------------- /native/runtime/tommyds/tommyds/tommyarrayblk.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, Andrea Mazzoleni. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 | * POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | 28 | /** \file 29 | * Dynamic array based on blocks of fixed size. 30 | * 31 | * This array is able to grow dynamically upon request, without any reallocation. 32 | * 33 | * The grow operation involves an allocation of a new array block, without reallocating 34 | * the already used memory, and then not increasing the heap fragmentation, 35 | * and minimize the space occupation. 36 | * This also implies that the address of the stored elements never change. 37 | * 38 | * Allocated blocks are always of the same fixed size of 4 Ki pointers. 39 | */ 40 | 41 | #ifndef __TOMMYARRAYBLK_H 42 | #define __TOMMYARRAYBLK_H 43 | 44 | #include "tommytypes.h" 45 | #include "tommyarray.h" 46 | 47 | #include /* for assert */ 48 | 49 | /******************************************************************************/ 50 | /* array */ 51 | 52 | /** 53 | * Elements for each block. 54 | */ 55 | #define TOMMY_ARRAYBLK_SIZE (4 * 1024) 56 | 57 | /** 58 | * Array container type. 59 | * \note Don't use internal fields directly, but access the container only using functions. 60 | */ 61 | typedef struct tommy_arrayblk_struct { 62 | tommy_array block; /**< Array of blocks. */ 63 | tommy_size_t count; /**< Number of initialized elements in the array. */ 64 | } tommy_arrayblk; 65 | 66 | /** 67 | * Initializes the array. 68 | */ 69 | void tommy_arrayblk_init(tommy_arrayblk* array); 70 | 71 | /** 72 | * Deinitializes the array. 73 | */ 74 | void tommy_arrayblk_done(tommy_arrayblk* array); 75 | 76 | /** 77 | * Grows the size up to the specified value. 78 | * All the new elements in the array are initialized with the 0 value. 79 | */ 80 | void tommy_arrayblk_grow(tommy_arrayblk* array, tommy_size_t size); 81 | 82 | /** 83 | * Gets a reference of the element at the specified position. 84 | * You must be sure that space for this position is already 85 | * allocated calling tommy_arrayblk_grow(). 86 | */ 87 | tommy_inline void** tommy_arrayblk_ref(tommy_arrayblk* array, tommy_size_t pos) 88 | { 89 | void** ptr; 90 | 91 | assert(pos < array->count); 92 | 93 | ptr = tommy_cast(void**, tommy_array_get(&array->block, pos / TOMMY_ARRAYBLK_SIZE)); 94 | 95 | return &ptr[pos % TOMMY_ARRAYBLK_SIZE]; 96 | } 97 | 98 | /** 99 | * Sets the element at the specified position. 100 | * You must be sure that space for this position is already 101 | * allocated calling tommy_arrayblk_grow(). 102 | */ 103 | tommy_inline void tommy_arrayblk_set(tommy_arrayblk* array, tommy_size_t pos, void* element) 104 | { 105 | *tommy_arrayblk_ref(array, pos) = element; 106 | } 107 | 108 | /** 109 | * Gets the element at the specified position. 110 | * You must be sure that space for this position is already 111 | * allocated calling tommy_arrayblk_grow(). 112 | */ 113 | tommy_inline void* tommy_arrayblk_get(tommy_arrayblk* array, tommy_size_t pos) 114 | { 115 | return *tommy_arrayblk_ref(array, pos); 116 | } 117 | 118 | /** 119 | * Grows and inserts a new element at the end of the array. 120 | */ 121 | tommy_inline void tommy_arrayblk_insert(tommy_arrayblk* array, void* element) 122 | { 123 | tommy_size_t pos = array->count; 124 | 125 | tommy_arrayblk_grow(array, pos + 1); 126 | 127 | tommy_arrayblk_set(array, pos, element); 128 | } 129 | 130 | /** 131 | * Gets the initialized size of the array. 132 | */ 133 | tommy_inline tommy_size_t tommy_arrayblk_size(tommy_arrayblk* array) 134 | { 135 | return array->count; 136 | } 137 | 138 | /** 139 | * Gets the size of allocated memory. 140 | */ 141 | tommy_size_t tommy_arrayblk_memory_usage(tommy_arrayblk* array); 142 | 143 | #endif 144 | 145 | -------------------------------------------------------------------------------- /native/runtime/tommyds/tommyds/tommyarrayblkof.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, Andrea Mazzoleni. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 | * POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | 28 | #include "tommyarrayblkof.h" 29 | 30 | /******************************************************************************/ 31 | /* array */ 32 | 33 | void tommy_arrayblkof_init(tommy_arrayblkof* array, tommy_size_t element_size) 34 | { 35 | tommy_array_init(&array->block); 36 | 37 | array->element_size = element_size; 38 | array->count = 0; 39 | } 40 | 41 | void tommy_arrayblkof_done(tommy_arrayblkof* array) 42 | { 43 | tommy_size_t i; 44 | 45 | for (i = 0; i < tommy_array_size(&array->block); ++i) 46 | tommy_free(tommy_array_get(&array->block, i)); 47 | 48 | tommy_array_done(&array->block); 49 | } 50 | 51 | void tommy_arrayblkof_grow(tommy_arrayblkof* array, tommy_size_t count) 52 | { 53 | tommy_size_t block_max; 54 | tommy_size_t block_mac; 55 | 56 | if (array->count >= count) 57 | return; 58 | array->count = count; 59 | 60 | block_max = (count + TOMMY_ARRAYBLKOF_SIZE - 1) / TOMMY_ARRAYBLKOF_SIZE; 61 | block_mac = tommy_array_size(&array->block); 62 | 63 | if (block_mac < block_max) { 64 | /* grow the block array */ 65 | tommy_array_grow(&array->block, block_max); 66 | 67 | /* allocate new blocks */ 68 | while (block_mac < block_max) { 69 | void** ptr = tommy_cast(void**, tommy_calloc(TOMMY_ARRAYBLKOF_SIZE, array->element_size)); 70 | 71 | /* set the new block */ 72 | tommy_array_set(&array->block, block_mac, ptr); 73 | 74 | ++block_mac; 75 | } 76 | } 77 | } 78 | 79 | tommy_size_t tommy_arrayblkof_memory_usage(tommy_arrayblkof* array) 80 | { 81 | return tommy_array_memory_usage(&array->block) + tommy_array_size(&array->block) * TOMMY_ARRAYBLKOF_SIZE * array->element_size; 82 | } 83 | 84 | -------------------------------------------------------------------------------- /native/runtime/tommyds/tommyds/tommyarrayblkof.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, Andrea Mazzoleni. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 | * POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | 28 | /** \file 29 | * Dynamic array based on blocks of fixed size. 30 | * 31 | * This array is able to grow dynamically upon request, without any reallocation. 32 | * 33 | * This is very similar at ::tommy_arrayblk, but it allows to store elements of any 34 | * size and not just pointers. 35 | * 36 | * Note that in this case tommy_arrayblkof_ref() returns a pointer to the element, 37 | * that should be used for getting and setting elements in the array, 38 | * as generic getter and setter are not available. 39 | */ 40 | 41 | #ifndef __TOMMYARRAYBLKOF_H 42 | #define __TOMMYARRAYBLKOF_H 43 | 44 | #include "tommytypes.h" 45 | #include "tommyarray.h" 46 | 47 | #include /* for assert */ 48 | 49 | /******************************************************************************/ 50 | /* array */ 51 | 52 | /** 53 | * Elements for each block. 54 | */ 55 | #define TOMMY_ARRAYBLKOF_SIZE (4 * 1024) 56 | 57 | /** 58 | * Array container type. 59 | * \note Don't use internal fields directly, but access the container only using functions. 60 | */ 61 | typedef struct tommy_arrayblkof_struct { 62 | tommy_array block; /**< Array of blocks. */ 63 | tommy_size_t element_size; /**< Size of the stored element in bytes. */ 64 | tommy_size_t count; /**< Number of initialized elements in the array. */ 65 | } tommy_arrayblkof; 66 | 67 | /** 68 | * Initializes the array. 69 | * \param element_size Size in byte of the element to store in the array. 70 | */ 71 | void tommy_arrayblkof_init(tommy_arrayblkof* array, tommy_size_t element_size); 72 | 73 | /** 74 | * Deinitializes the array. 75 | */ 76 | void tommy_arrayblkof_done(tommy_arrayblkof* array); 77 | 78 | /** 79 | * Grows the size up to the specified value. 80 | * All the new elements in the array are initialized with the 0 value. 81 | */ 82 | void tommy_arrayblkof_grow(tommy_arrayblkof* array, tommy_size_t size); 83 | 84 | /** 85 | * Gets a reference of the element at the specified position. 86 | * You must be sure that space for this position is already 87 | * allocated calling tommy_arrayblkof_grow(). 88 | */ 89 | tommy_inline void* tommy_arrayblkof_ref(tommy_arrayblkof* array, tommy_size_t pos) 90 | { 91 | unsigned char* base; 92 | 93 | assert(pos < array->count); 94 | 95 | base = tommy_cast(unsigned char*, tommy_array_get(&array->block, pos / TOMMY_ARRAYBLKOF_SIZE)); 96 | 97 | return base + (pos % TOMMY_ARRAYBLKOF_SIZE) * array->element_size; 98 | } 99 | 100 | /** 101 | * Gets the initialized size of the array. 102 | */ 103 | tommy_inline tommy_size_t tommy_arrayblkof_size(tommy_arrayblkof* array) 104 | { 105 | return array->count; 106 | } 107 | 108 | /** 109 | * Gets the size of allocated memory. 110 | */ 111 | tommy_size_t tommy_arrayblkof_memory_usage(tommy_arrayblkof* array); 112 | 113 | #endif 114 | 115 | -------------------------------------------------------------------------------- /native/runtime/tommyds/tommyds/tommyarrayof.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, Andrea Mazzoleni. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 | * POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | 28 | #include "tommyarrayof.h" 29 | 30 | /******************************************************************************/ 31 | /* array */ 32 | 33 | void tommy_arrayof_init(tommy_arrayof* array, tommy_size_t element_size) 34 | { 35 | tommy_uint_t i; 36 | 37 | /* fixed initial size */ 38 | array->element_size = element_size; 39 | array->bucket_bit = TOMMY_ARRAYOF_BIT; 40 | array->bucket_max = (tommy_size_t)1 << array->bucket_bit; 41 | array->bucket[0] = tommy_calloc(array->bucket_max, array->element_size); 42 | for (i = 1; i < TOMMY_ARRAYOF_BIT; ++i) 43 | array->bucket[i] = array->bucket[0]; 44 | 45 | array->count = 0; 46 | } 47 | 48 | void tommy_arrayof_done(tommy_arrayof* array) 49 | { 50 | tommy_uint_t i; 51 | 52 | tommy_free(array->bucket[0]); 53 | for (i = TOMMY_ARRAYOF_BIT; i < array->bucket_bit; ++i) { 54 | unsigned char* segment = tommy_cast(unsigned char*, array->bucket[i]); 55 | tommy_free(segment + ((tommy_ptrdiff_t)1 << i) * array->element_size); 56 | } 57 | } 58 | 59 | void tommy_arrayof_grow(tommy_arrayof* array, tommy_size_t count) 60 | { 61 | if (array->count >= count) 62 | return; 63 | array->count = count; 64 | 65 | while (count > array->bucket_max) { 66 | unsigned char* segment; 67 | 68 | /* allocate one more segment */ 69 | segment = tommy_cast(unsigned char*, tommy_calloc(array->bucket_max, array->element_size)); 70 | 71 | /* store it adjusting the offset */ 72 | /* cast to ptrdiff_t to ensure to get a negative value */ 73 | array->bucket[array->bucket_bit] = segment - (tommy_ptrdiff_t)array->bucket_max * array->element_size; 74 | 75 | ++array->bucket_bit; 76 | array->bucket_max = (tommy_size_t)1 << array->bucket_bit; 77 | } 78 | } 79 | 80 | tommy_size_t tommy_arrayof_memory_usage(tommy_arrayof* array) 81 | { 82 | return array->bucket_max * (tommy_size_t)array->element_size; 83 | } 84 | 85 | -------------------------------------------------------------------------------- /native/runtime/tommyds/tommyds/tommyarrayof.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, Andrea Mazzoleni. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 | * POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | 28 | /** \file 29 | * Dynamic array based on segments of exponential growing size. 30 | * 31 | * This array is able to grow dynamically upon request, without any reallocation. 32 | * 33 | * This is very similar at ::tommy_array, but it allows to store elements of any 34 | * size and not just pointers. 35 | * 36 | * Note that in this case tommy_arrayof_ref() returns a pointer to the element, 37 | * that should be used for getting and setting elements in the array, 38 | * as generic getter and setter are not available. 39 | */ 40 | 41 | #ifndef __TOMMYARRAYOF_H 42 | #define __TOMMYARRAYOF_H 43 | 44 | #include "tommytypes.h" 45 | 46 | #include /* for assert */ 47 | 48 | /******************************************************************************/ 49 | /* array */ 50 | 51 | /** 52 | * Initial and minimal size of the array expressed as a power of 2. 53 | * The initial size is 2^TOMMY_ARRAYOF_BIT. 54 | */ 55 | #define TOMMY_ARRAYOF_BIT 6 56 | 57 | /** 58 | * Array container type. 59 | * \note Don't use internal fields directly, but access the container only using functions. 60 | */ 61 | typedef struct tommy_arrayof_struct { 62 | void* bucket[TOMMY_SIZE_BIT]; /**< Dynamic array of buckets. */ 63 | tommy_size_t element_size; /**< Size of the stored element in bytes. */ 64 | tommy_size_t bucket_max; /**< Number of buckets. */ 65 | tommy_size_t count; /**< Number of initialized elements in the array. */ 66 | tommy_uint_t bucket_bit; /**< Bits used in the bit mask. */ 67 | } tommy_arrayof; 68 | 69 | /** 70 | * Initializes the array. 71 | * \param element_size Size in byte of the element to store in the array. 72 | */ 73 | void tommy_arrayof_init(tommy_arrayof* array, tommy_size_t element_size); 74 | 75 | /** 76 | * Deinitializes the array. 77 | */ 78 | void tommy_arrayof_done(tommy_arrayof* array); 79 | 80 | /** 81 | * Grows the size up to the specified value. 82 | * All the new elements in the array are initialized with the 0 value. 83 | */ 84 | void tommy_arrayof_grow(tommy_arrayof* array, tommy_size_t size); 85 | 86 | /** 87 | * Gets a reference of the element at the specified position. 88 | * You must be sure that space for this position is already 89 | * allocated calling tommy_arrayof_grow(). 90 | */ 91 | tommy_inline void* tommy_arrayof_ref(tommy_arrayof* array, tommy_size_t pos) 92 | { 93 | unsigned char* ptr; 94 | tommy_uint_t bsr; 95 | 96 | assert(pos < array->count); 97 | 98 | /* get the highest bit set, in case of all 0, return 0 */ 99 | bsr = tommy_ilog2(pos | 1); 100 | 101 | ptr = tommy_cast(unsigned char*, array->bucket[bsr]); 102 | 103 | return ptr + pos * array->element_size; 104 | } 105 | 106 | /** 107 | * Gets the initialized size of the array. 108 | */ 109 | tommy_inline tommy_size_t tommy_arrayof_size(tommy_arrayof* array) 110 | { 111 | return array->count; 112 | } 113 | 114 | /** 115 | * Gets the size of allocated memory. 116 | */ 117 | tommy_size_t tommy_arrayof_memory_usage(tommy_arrayof* array); 118 | 119 | #endif 120 | 121 | -------------------------------------------------------------------------------- /native/runtime/tommyds/tommyds/tommychain.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010, Andrea Mazzoleni. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 | * POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | 28 | /** \file 29 | * Chain of nodes. 30 | * A chain of nodes is an abstraction used to implements complex list operations 31 | * like sorting. 32 | * 33 | * Do not use this directly. Use lists instead. 34 | */ 35 | 36 | #ifndef __TOMMYCHAIN_H 37 | #define __TOMMYCHAIN_H 38 | 39 | #include "tommytypes.h" 40 | 41 | /******************************************************************************/ 42 | /* chain */ 43 | 44 | /** 45 | * Chain of nodes. 46 | * A chain of nodes is a sequence of nodes with the following properties: 47 | * - It contains at least one node. A chains of zero nodes cannot exist. 48 | * - The next field of the tail is of *undefined* value. 49 | * - The prev field of the head is of *undefined* value. 50 | * - All the other inner prev and next fields are correctly set. 51 | */ 52 | typedef struct tommy_chain_struct { 53 | tommy_node* head; /**< Pointer to the head of the chain. */ 54 | tommy_node* tail; /**< Pointer to the tail of the chain. */ 55 | } tommy_chain; 56 | 57 | /** 58 | * Splices a chain in the middle of another chain. 59 | */ 60 | tommy_inline void tommy_chain_splice(tommy_node* first_before, tommy_node* first_after, tommy_node* second_head, tommy_node* second_tail) 61 | { 62 | /* set the prev list */ 63 | first_after->prev = second_tail; 64 | second_head->prev = first_before; 65 | 66 | /* set the next list */ 67 | first_before->next = second_head; 68 | second_tail->next = first_after; 69 | } 70 | 71 | /** 72 | * Concats two chains. 73 | */ 74 | tommy_inline void tommy_chain_concat(tommy_node* first_tail, tommy_node* second_head) 75 | { 76 | /* set the prev list */ 77 | second_head->prev = first_tail; 78 | 79 | /* set the next list */ 80 | first_tail->next = second_head; 81 | } 82 | 83 | /** 84 | * Merges two chains. 85 | */ 86 | tommy_inline void tommy_chain_merge(tommy_chain* first, tommy_chain* second, tommy_compare_func* cmp) 87 | { 88 | tommy_node* first_i = first->head; 89 | tommy_node* second_i = second->head; 90 | 91 | /* merge */ 92 | while (1) { 93 | if (cmp(first_i->data, second_i->data) > 0) { 94 | tommy_node* next = second_i->next; 95 | if (first_i == first->head) { 96 | tommy_chain_concat(second_i, first_i); 97 | first->head = second_i; 98 | } else { 99 | tommy_chain_splice(first_i->prev, first_i, second_i, second_i); 100 | } 101 | if (second_i == second->tail) 102 | break; 103 | second_i = next; 104 | } else { 105 | if (first_i == first->tail) { 106 | tommy_chain_concat(first_i, second_i); 107 | first->tail = second->tail; 108 | break; 109 | } 110 | first_i = first_i->next; 111 | } 112 | } 113 | } 114 | 115 | /** 116 | * Merges two chains managing special degenerated cases. 117 | * It's funtionally equivalent at tommy_chain_merge() but faster with already ordered chains. 118 | */ 119 | tommy_inline void tommy_chain_merge_degenerated(tommy_chain* first, tommy_chain* second, tommy_compare_func* cmp) 120 | { 121 | /* identify the condition first <= second */ 122 | if (cmp(first->tail->data, second->head->data) <= 0) { 123 | tommy_chain_concat(first->tail, second->head); 124 | first->tail = second->tail; 125 | return; 126 | } 127 | 128 | /* identify the condition second < first */ 129 | /* here we must be strict on comparison to keep the sort stable */ 130 | if (cmp(second->tail->data, first->head->data) < 0) { 131 | tommy_chain_concat(second->tail, first->head); 132 | first->head = second->head; 133 | return; 134 | } 135 | 136 | tommy_chain_merge(first, second, cmp); 137 | } 138 | 139 | /** 140 | * Sorts a chain. 141 | * It's a stable merge sort using power of 2 buckets, with O(N*log(N)) complexity, 142 | * similar at the one used in the SGI STL libraries and in the Linux Kernel, 143 | * but faster on degenerated cases like already ordered lists. 144 | * 145 | * SGI STL stl_list.h 146 | * http://www.sgi.com/tech/stl/stl_list.h 147 | * 148 | * Linux Kernel lib/list_sort.c 149 | * http://lxr.linux.no/#linux+v2.6.36/lib/list_sort.c 150 | */ 151 | tommy_inline void tommy_chain_mergesort(tommy_chain* chain, tommy_compare_func* cmp) 152 | { 153 | /* 154 | * Bit buckets of chains. 155 | * Each bucket contains 2^i nodes or it's empty. 156 | * The chain at address TOMMY_BIT_MAX is an independet variable operating as "carry". 157 | * We keep it in the same "bit" vector to avoid reports from the valgrind tool sgcheck. 158 | */ 159 | tommy_chain bit[TOMMY_SIZE_BIT + 1]; 160 | 161 | /** 162 | * Value stored inside the bit bucket. 163 | * It's used to know which bucket is empty of full. 164 | */ 165 | tommy_size_t counter; 166 | tommy_node* node = chain->head; 167 | tommy_node* tail = chain->tail; 168 | tommy_size_t mask; 169 | tommy_size_t i; 170 | 171 | counter = 0; 172 | while (1) { 173 | tommy_node* next; 174 | tommy_chain* last; 175 | 176 | /* carry bit to add */ 177 | last = &bit[TOMMY_SIZE_BIT]; 178 | bit[TOMMY_SIZE_BIT].head = node; 179 | bit[TOMMY_SIZE_BIT].tail = node; 180 | next = node->next; 181 | 182 | /* add the bit, propagating the carry */ 183 | i = 0; 184 | mask = counter; 185 | while ((mask & 1) != 0) { 186 | tommy_chain_merge_degenerated(&bit[i], last, cmp); 187 | mask >>= 1; 188 | last = &bit[i]; 189 | ++i; 190 | } 191 | 192 | /* copy the carry in the first empty bit */ 193 | bit[i] = *last; 194 | 195 | /* add the carry in the counter */ 196 | ++counter; 197 | 198 | if (node == tail) 199 | break; 200 | node = next; 201 | } 202 | 203 | /* merge the buckets */ 204 | i = tommy_ctz(counter); 205 | mask = counter >> i; 206 | while (mask != 1) { 207 | mask >>= 1; 208 | if (mask & 1) 209 | tommy_chain_merge_degenerated(&bit[i + 1], &bit[i], cmp); 210 | else 211 | bit[i + 1] = bit[i]; 212 | ++i; 213 | } 214 | 215 | *chain = bit[i]; 216 | } 217 | 218 | #endif 219 | 220 | -------------------------------------------------------------------------------- /native/runtime/tommyds/tommyds/tommyhash.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010, Andrea Mazzoleni. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 | * POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | 28 | #include "tommyhash.h" 29 | 30 | /******************************************************************************/ 31 | /* hash */ 32 | 33 | tommy_inline tommy_uint32_t tommy_le_uint32_read(const void* ptr) 34 | { 35 | /* allow unaligned read on Intel x86 and x86_64 platforms */ 36 | #if defined(__i386__) || defined(_M_IX86) || defined(_X86_) || defined(__x86_64__) || defined(_M_X64) 37 | /* defines from http://predef.sourceforge.net/ */ 38 | return *(const tommy_uint32_t*)ptr; 39 | #else 40 | const unsigned char* ptr8 = tommy_cast(const unsigned char*, ptr); 41 | return ptr8[0] + ((tommy_uint32_t)ptr8[1] << 8) + ((tommy_uint32_t)ptr8[2] << 16) + ((tommy_uint32_t)ptr8[3] << 24); 42 | #endif 43 | } 44 | 45 | #define tommy_rot(x, k) \ 46 | (((x) << (k)) | ((x) >> (32 - (k)))) 47 | 48 | #define tommy_mix(a, b, c) \ 49 | do { \ 50 | a -= c; a ^= tommy_rot(c, 4); c += b; \ 51 | b -= a; b ^= tommy_rot(a, 6); a += c; \ 52 | c -= b; c ^= tommy_rot(b, 8); b += a; \ 53 | a -= c; a ^= tommy_rot(c, 16); c += b; \ 54 | b -= a; b ^= tommy_rot(a, 19); a += c; \ 55 | c -= b; c ^= tommy_rot(b, 4); b += a; \ 56 | } while (0) 57 | 58 | #define tommy_final(a, b, c) \ 59 | do { \ 60 | c ^= b; c -= tommy_rot(b, 14); \ 61 | a ^= c; a -= tommy_rot(c, 11); \ 62 | b ^= a; b -= tommy_rot(a, 25); \ 63 | c ^= b; c -= tommy_rot(b, 16); \ 64 | a ^= c; a -= tommy_rot(c, 4); \ 65 | b ^= a; b -= tommy_rot(a, 14); \ 66 | c ^= b; c -= tommy_rot(b, 24); \ 67 | } while (0) 68 | 69 | tommy_uint32_t tommy_hash_u32(tommy_uint32_t init_val, const void* void_key, tommy_size_t key_len) 70 | { 71 | const unsigned char* key = tommy_cast(const unsigned char*, void_key); 72 | tommy_uint32_t a, b, c; 73 | 74 | a = b = c = 0xdeadbeef + ((tommy_uint32_t)key_len) + init_val; 75 | 76 | while (key_len > 12) { 77 | a += tommy_le_uint32_read(key + 0); 78 | b += tommy_le_uint32_read(key + 4); 79 | c += tommy_le_uint32_read(key + 8); 80 | 81 | tommy_mix(a, b, c); 82 | 83 | key_len -= 12; 84 | key += 12; 85 | } 86 | 87 | switch (key_len) { 88 | case 0 : 89 | return c; /* used only when called with a zero length */ 90 | case 12 : 91 | c += tommy_le_uint32_read(key + 8); 92 | b += tommy_le_uint32_read(key + 4); 93 | a += tommy_le_uint32_read(key + 0); 94 | break; 95 | case 11 : c += ((tommy_uint32_t)key[10]) << 16; /* fallthrough */ 96 | case 10 : c += ((tommy_uint32_t)key[9]) << 8; /* fallthrough */ 97 | case 9 : c += key[8]; /* fallthrough */ 98 | case 8 : 99 | b += tommy_le_uint32_read(key + 4); 100 | a += tommy_le_uint32_read(key + 0); 101 | break; 102 | case 7 : b += ((tommy_uint32_t)key[6]) << 16; /* fallthrough */ 103 | case 6 : b += ((tommy_uint32_t)key[5]) << 8; /* fallthrough */ 104 | case 5 : b += key[4]; /* fallthrough */ 105 | case 4 : 106 | a += tommy_le_uint32_read(key + 0); 107 | break; 108 | case 3 : a += ((tommy_uint32_t)key[2]) << 16; /* fallthrough */ 109 | case 2 : a += ((tommy_uint32_t)key[1]) << 8; /* fallthrough */ 110 | case 1 : a += key[0]; /* fallthrough */ 111 | } 112 | 113 | tommy_final(a, b, c); 114 | 115 | return c; 116 | } 117 | 118 | tommy_uint64_t tommy_hash_u64(tommy_uint64_t init_val, const void* void_key, tommy_size_t key_len) 119 | { 120 | const unsigned char* key = tommy_cast(const unsigned char*, void_key); 121 | tommy_uint32_t a, b, c; 122 | 123 | a = b = c = 0xdeadbeef + ((tommy_uint32_t)key_len) + (init_val & 0xffffffff); 124 | c += init_val >> 32; 125 | 126 | while (key_len > 12) { 127 | a += tommy_le_uint32_read(key + 0); 128 | b += tommy_le_uint32_read(key + 4); 129 | c += tommy_le_uint32_read(key + 8); 130 | 131 | tommy_mix(a, b, c); 132 | 133 | key_len -= 12; 134 | key += 12; 135 | } 136 | 137 | switch (key_len) { 138 | case 0 : 139 | return c + ((tommy_uint64_t)b << 32); /* used only when called with a zero length */ 140 | case 12 : 141 | c += tommy_le_uint32_read(key + 8); 142 | b += tommy_le_uint32_read(key + 4); 143 | a += tommy_le_uint32_read(key + 0); 144 | break; 145 | case 11 : c += ((tommy_uint32_t)key[10]) << 16; /* fallthrough */ 146 | case 10 : c += ((tommy_uint32_t)key[9]) << 8; /* fallthrough */ 147 | case 9 : c += key[8]; /* fallthrough */ 148 | case 8 : 149 | b += tommy_le_uint32_read(key + 4); 150 | a += tommy_le_uint32_read(key + 0); 151 | break; 152 | case 7 : b += ((tommy_uint32_t)key[6]) << 16; /* fallthrough */ 153 | case 6 : b += ((tommy_uint32_t)key[5]) << 8; /* fallthrough */ 154 | case 5 : b += key[4]; /* fallthrough */ 155 | case 4 : 156 | a += tommy_le_uint32_read(key + 0); 157 | break; 158 | case 3 : a += ((tommy_uint32_t)key[2]) << 16; /* fallthrough */ 159 | case 2 : a += ((tommy_uint32_t)key[1]) << 8; /* fallthrough */ 160 | case 1 : a += key[0]; /* fallthrough */ 161 | } 162 | 163 | tommy_final(a, b, c); 164 | 165 | return c + ((tommy_uint64_t)b << 32); 166 | } 167 | 168 | tommy_uint32_t tommy_strhash_u32(tommy_uint64_t init_val, const void* void_key) 169 | { 170 | const unsigned char* key = tommy_cast(const unsigned char*, void_key); 171 | tommy_uint32_t a, b, c; 172 | tommy_uint32_t m[3] = { 0xff, 0xff00, 0xff0000 }; 173 | 174 | a = b = c = 0xdeadbeef + init_val; 175 | /* this is different than original lookup3 and the result won't match */ 176 | 177 | while (1) { 178 | tommy_uint32_t v = tommy_le_uint32_read(key); 179 | 180 | if (tommy_haszero_u32(v)) { 181 | if (v & m[0]) { 182 | a += v & m[0]; 183 | if (v & m[1]) { 184 | a += v & m[1]; 185 | if (v & m[2]) 186 | a += v & m[2]; 187 | } 188 | } 189 | 190 | break; 191 | } 192 | 193 | a += v; 194 | 195 | v = tommy_le_uint32_read(key + 4); 196 | 197 | if (tommy_haszero_u32(v)) { 198 | if (v & m[0]) { 199 | b += v & m[0]; 200 | if (v & m[1]) { 201 | b += v & m[1]; 202 | if (v & m[2]) 203 | b += v & m[2]; 204 | } 205 | } 206 | 207 | break; 208 | } 209 | 210 | b += v; 211 | 212 | v = tommy_le_uint32_read(key + 8); 213 | 214 | if (tommy_haszero_u32(v)) { 215 | if (v & m[0]) { 216 | c += v & m[0]; 217 | if (v & m[1]) { 218 | c += v & m[1]; 219 | if (v & m[2]) 220 | c += v & m[2]; 221 | } 222 | } 223 | 224 | break; 225 | } 226 | 227 | c += v; 228 | 229 | tommy_mix(a, b, c); 230 | 231 | key += 12; 232 | } 233 | 234 | /* for lengths that are multiplers of 12 we already have called mix */ 235 | /* this is different than the original lookup3 and the result won't match */ 236 | 237 | tommy_final(a, b, c); 238 | 239 | return c; 240 | } 241 | 242 | -------------------------------------------------------------------------------- /native/runtime/tommyds/tommyds/tommyhash.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010, Andrea Mazzoleni. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 | * POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | 28 | /** \file 29 | * Hash functions for the use with ::tommy_hashtable, ::tommy_hashdyn and ::tommy_hashlin. 30 | */ 31 | 32 | #ifndef __TOMMYHASH_H 33 | #define __TOMMYHASH_H 34 | 35 | #include "tommytypes.h" 36 | 37 | /******************************************************************************/ 38 | /* hash */ 39 | 40 | /** 41 | * Hash function with a 32 bits result. 42 | * Implementation of the Robert Jenkins "lookup3" hash 32 bits version, 43 | * from http://www.burtleburtle.net/bob/hash/doobs.html, function hashlittle(). 44 | * 45 | * This hash is designed to provide a good overall performance in all platforms, 46 | * including 32 bits. If you target only 64 bits, you can use faster hashes, 47 | * like SpookyHash or FarmHash. 48 | * 49 | * \param init_val Initialization value. 50 | * Using a different initialization value, you can generate a completely different set of hash values. 51 | * Use 0 if not relevant. 52 | * \param void_key Pointer to the data to hash. 53 | * \param key_len Size of the data to hash. 54 | * \note 55 | * This function is endianess independent. 56 | * \return The hash value of 32 bits. 57 | */ 58 | tommy_uint32_t tommy_hash_u32(tommy_uint32_t init_val, const void* void_key, tommy_size_t key_len); 59 | 60 | /** 61 | * Hash function with a 64 bits result. 62 | * Implementation of the Robert Jenkins "lookup3" hash 64 bits versions, 63 | * from http://www.burtleburtle.net/bob/hash/doobs.html, function hashlittle2(). 64 | * 65 | * This hash is designed to provide a good overall performance in all platforms, 66 | * including 32 bits. If you target only 64 bits, you can use faster hashes, 67 | * like SpookyHash or FarmHash. 68 | * 69 | * \param init_val Initialization value. 70 | * Using a different initialization value, you can generate a completely different set of hash values. 71 | * Use 0 if not relevant. 72 | * \param void_key Pointer to the data to hash. 73 | * \param key_len Size of the data to hash. 74 | * \note 75 | * This function is endianess independent. 76 | * \return The hash value of 64 bits. 77 | */ 78 | tommy_uint64_t tommy_hash_u64(tommy_uint64_t init_val, const void* void_key, tommy_size_t key_len); 79 | 80 | /** 81 | * String hash function with a 32 bits result. 82 | * Implementation is based on the the Robert Jenkins "lookup3" hash 32 bits version, 83 | * from http://www.burtleburtle.net/bob/hash/doobs.html, function hashlittle(). 84 | * 85 | * This hash is designed to handle strings with an unknown length. If you 86 | * know the string length, the other hash functions are surely faster. 87 | * 88 | * \param init_val Initialization value. 89 | * Using a different initialization value, you can generate a completely different set of hash values. 90 | * Use 0 if not relevant. 91 | * \param void_key Pointer to the string to hash. It has to be 0 terminated. 92 | * \note 93 | * This function is endianess independent. 94 | * \return The hash value of 32 bits. 95 | */ 96 | tommy_uint32_t tommy_strhash_u32(tommy_uint64_t init_val, const void* void_key); 97 | 98 | /** 99 | * Integer reversible hash function for 32 bits. 100 | * Implementation of the Robert Jenkins "4-byte Integer Hashing", 101 | * from http://burtleburtle.net/bob/hash/integer.html 102 | */ 103 | tommy_inline tommy_uint32_t tommy_inthash_u32(tommy_uint32_t key) 104 | { 105 | key -= key << 6; 106 | key ^= key >> 17; 107 | key -= key << 9; 108 | key ^= key << 4; 109 | key -= key << 3; 110 | key ^= key << 10; 111 | key ^= key >> 15; 112 | 113 | return key; 114 | } 115 | 116 | /** 117 | * Integer reversible hash function for 64 bits. 118 | * Implementation of the Thomas Wang "Integer Hash Function", 119 | * from http://web.archive.org/web/20071223173210/http://www.concentric.net/~Ttwang/tech/inthash.htm 120 | */ 121 | tommy_inline tommy_uint64_t tommy_inthash_u64(tommy_uint64_t key) 122 | { 123 | key = ~key + (key << 21); 124 | key = key ^ (key >> 24); 125 | key = key + (key << 3) + (key << 8); 126 | key = key ^ (key >> 14); 127 | key = key + (key << 2) + (key << 4); 128 | key = key ^ (key >> 28); 129 | key = key + (key << 31); 130 | 131 | return key; 132 | } 133 | 134 | #endif 135 | 136 | -------------------------------------------------------------------------------- /native/runtime/tommyds/tommyds/tommyhashdyn.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010, Andrea Mazzoleni. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 | * POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | 28 | #include "tommyhashdyn.h" 29 | #include "tommylist.h" 30 | 31 | /******************************************************************************/ 32 | /* hashdyn */ 33 | 34 | void tommy_hashdyn_init(tommy_hashdyn* hashdyn) 35 | { 36 | /* fixed initial size */ 37 | hashdyn->bucket_bit = TOMMY_HASHDYN_BIT; 38 | hashdyn->bucket_max = (tommy_size_t)1 << hashdyn->bucket_bit; 39 | hashdyn->bucket_mask = hashdyn->bucket_max - 1; 40 | hashdyn->bucket = tommy_cast(tommy_hashdyn_node**, tommy_calloc(hashdyn->bucket_max, sizeof(tommy_hashdyn_node*))); 41 | 42 | hashdyn->count = 0; 43 | } 44 | 45 | void tommy_hashdyn_done(tommy_hashdyn* hashdyn) 46 | { 47 | tommy_free(hashdyn->bucket); 48 | } 49 | 50 | /** 51 | * Resize the bucket vector. 52 | */ 53 | static void tommy_hashdyn_resize(tommy_hashdyn* hashdyn, tommy_size_t new_bucket_bit) 54 | { 55 | tommy_size_t bucket_bit; 56 | tommy_size_t bucket_max; 57 | tommy_size_t new_bucket_max; 58 | tommy_size_t new_bucket_mask; 59 | tommy_hashdyn_node** new_bucket; 60 | 61 | bucket_bit = hashdyn->bucket_bit; 62 | bucket_max = hashdyn->bucket_max; 63 | 64 | new_bucket_max = (tommy_size_t)1 << new_bucket_bit; 65 | new_bucket_mask = new_bucket_max - 1; 66 | 67 | /* allocate the new vector using malloc() and not calloc() */ 68 | /* because data is fully initialized in the update process */ 69 | new_bucket = tommy_cast(tommy_hashdyn_node**, tommy_malloc(new_bucket_max * sizeof(tommy_hashdyn_node*))); 70 | 71 | /* reinsert all the elements */ 72 | if (new_bucket_bit > bucket_bit) { 73 | tommy_size_t i; 74 | 75 | /* grow */ 76 | for (i = 0; i < bucket_max; ++i) { 77 | tommy_hashdyn_node* j; 78 | 79 | /* setup the new two buckets */ 80 | new_bucket[i] = 0; 81 | new_bucket[i + bucket_max] = 0; 82 | 83 | /* reinsert the bucket */ 84 | j = hashdyn->bucket[i]; 85 | while (j) { 86 | tommy_hashdyn_node* j_next = j->next; 87 | tommy_size_t pos = j->index & new_bucket_mask; 88 | if (new_bucket[pos]) 89 | tommy_list_insert_tail_not_empty(new_bucket[pos], j); 90 | else 91 | tommy_list_insert_first(&new_bucket[pos], j); 92 | j = j_next; 93 | } 94 | } 95 | } else { 96 | tommy_size_t i; 97 | 98 | /* shrink */ 99 | for (i = 0; i < new_bucket_max; ++i) { 100 | /* setup the new bucket with the lower bucket*/ 101 | new_bucket[i] = hashdyn->bucket[i]; 102 | 103 | /* concat the upper bucket */ 104 | tommy_list_concat(&new_bucket[i], &hashdyn->bucket[i + new_bucket_max]); 105 | } 106 | } 107 | 108 | tommy_free(hashdyn->bucket); 109 | 110 | /* setup */ 111 | hashdyn->bucket_bit = new_bucket_bit; 112 | hashdyn->bucket_max = new_bucket_max; 113 | hashdyn->bucket_mask = new_bucket_mask; 114 | hashdyn->bucket = new_bucket; 115 | } 116 | 117 | /** 118 | * Grow. 119 | */ 120 | tommy_inline void hashdyn_grow_step(tommy_hashdyn* hashdyn) 121 | { 122 | /* grow if more than 50% full */ 123 | if (hashdyn->count >= hashdyn->bucket_max / 2) 124 | tommy_hashdyn_resize(hashdyn, hashdyn->bucket_bit + 1); 125 | } 126 | 127 | /** 128 | * Shrink. 129 | */ 130 | tommy_inline void hashdyn_shrink_step(tommy_hashdyn* hashdyn) 131 | { 132 | /* shrink if less than 12.5% full */ 133 | if (hashdyn->count <= hashdyn->bucket_max / 8 && hashdyn->bucket_bit > TOMMY_HASHDYN_BIT) 134 | tommy_hashdyn_resize(hashdyn, hashdyn->bucket_bit - 1); 135 | } 136 | 137 | void tommy_hashdyn_insert(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node, void* data, tommy_hash_t hash) 138 | { 139 | tommy_size_t pos = hash & hashdyn->bucket_mask; 140 | 141 | tommy_list_insert_tail(&hashdyn->bucket[pos], node, data); 142 | 143 | node->index = hash; 144 | 145 | ++hashdyn->count; 146 | 147 | hashdyn_grow_step(hashdyn); 148 | } 149 | 150 | void* tommy_hashdyn_remove_existing(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node) 151 | { 152 | tommy_size_t pos = node->index & hashdyn->bucket_mask; 153 | 154 | tommy_list_remove_existing(&hashdyn->bucket[pos], node); 155 | 156 | --hashdyn->count; 157 | 158 | hashdyn_shrink_step(hashdyn); 159 | 160 | return node->data; 161 | } 162 | 163 | void* tommy_hashdyn_remove(tommy_hashdyn* hashdyn, tommy_search_func* cmp, const void* cmp_arg, tommy_hash_t hash) 164 | { 165 | tommy_size_t pos = hash & hashdyn->bucket_mask; 166 | tommy_hashdyn_node* node = hashdyn->bucket[pos]; 167 | 168 | while (node) { 169 | /* we first check if the hash matches, as in the same bucket we may have multiples hash values */ 170 | if (node->index == hash && cmp(cmp_arg, node->data) == 0) { 171 | tommy_list_remove_existing(&hashdyn->bucket[pos], node); 172 | 173 | --hashdyn->count; 174 | 175 | hashdyn_shrink_step(hashdyn); 176 | 177 | return node->data; 178 | } 179 | node = node->next; 180 | } 181 | 182 | return 0; 183 | } 184 | 185 | void tommy_hashdyn_foreach(tommy_hashdyn* hashdyn, tommy_foreach_func* func) 186 | { 187 | tommy_size_t bucket_max = hashdyn->bucket_max; 188 | tommy_hashdyn_node** bucket = hashdyn->bucket; 189 | tommy_size_t pos; 190 | 191 | for (pos = 0; pos < bucket_max; ++pos) { 192 | tommy_hashdyn_node* node = bucket[pos]; 193 | 194 | while (node) { 195 | void* data = node->data; 196 | node = node->next; 197 | func(data); 198 | } 199 | } 200 | } 201 | 202 | void tommy_hashdyn_foreach_arg(tommy_hashdyn* hashdyn, tommy_foreach_arg_func* func, void* arg) 203 | { 204 | tommy_size_t bucket_max = hashdyn->bucket_max; 205 | tommy_hashdyn_node** bucket = hashdyn->bucket; 206 | tommy_size_t pos; 207 | 208 | for (pos = 0; pos < bucket_max; ++pos) { 209 | tommy_hashdyn_node* node = bucket[pos]; 210 | 211 | while (node) { 212 | void* data = node->data; 213 | node = node->next; 214 | func(arg, data); 215 | } 216 | } 217 | } 218 | 219 | tommy_size_t tommy_hashdyn_memory_usage(tommy_hashdyn* hashdyn) 220 | { 221 | return hashdyn->bucket_max * (tommy_size_t)sizeof(hashdyn->bucket[0]) 222 | + tommy_hashdyn_count(hashdyn) * (tommy_size_t)sizeof(tommy_hashdyn_node); 223 | } 224 | 225 | -------------------------------------------------------------------------------- /native/runtime/tommyds/tommyds/tommyhashtbl.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010, Andrea Mazzoleni. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 | * POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | 28 | #include "tommyhashtbl.h" 29 | #include "tommylist.h" 30 | 31 | #include /* for memset */ 32 | 33 | /******************************************************************************/ 34 | /* hashtable */ 35 | 36 | void tommy_hashtable_init(tommy_hashtable* hashtable, tommy_size_t bucket_max) 37 | { 38 | if (bucket_max < 16) 39 | bucket_max = 16; 40 | else 41 | bucket_max = tommy_roundup_pow2(bucket_max); 42 | 43 | hashtable->bucket_max = bucket_max; 44 | hashtable->bucket_mask = hashtable->bucket_max - 1; 45 | 46 | /* initialize the vector using malloc()+memset() instead of calloc() */ 47 | /* to ensure that all the memory in really allocated immediately */ 48 | /* by the OS, and not deferred at later time. */ 49 | /* this improves performance, because we start with a fully initialized hashtable. */ 50 | hashtable->bucket = tommy_cast(tommy_hashtable_node**, tommy_malloc(hashtable->bucket_max * sizeof(tommy_hashtable_node*))); 51 | memset(hashtable->bucket, 0, hashtable->bucket_max * sizeof(tommy_hashtable_node*)); 52 | 53 | hashtable->count = 0; 54 | } 55 | 56 | void tommy_hashtable_done(tommy_hashtable* hashtable) 57 | { 58 | tommy_free(hashtable->bucket); 59 | } 60 | 61 | void tommy_hashtable_insert(tommy_hashtable* hashtable, tommy_hashtable_node* node, void* data, tommy_hash_t hash) 62 | { 63 | tommy_size_t pos = hash & hashtable->bucket_mask; 64 | 65 | tommy_list_insert_tail(&hashtable->bucket[pos], node, data); 66 | 67 | node->index = hash; 68 | 69 | ++hashtable->count; 70 | } 71 | 72 | void* tommy_hashtable_remove_existing(tommy_hashtable* hashtable, tommy_hashtable_node* node) 73 | { 74 | tommy_size_t pos = node->index & hashtable->bucket_mask; 75 | 76 | tommy_list_remove_existing(&hashtable->bucket[pos], node); 77 | 78 | --hashtable->count; 79 | 80 | return node->data; 81 | } 82 | 83 | void* tommy_hashtable_remove(tommy_hashtable* hashtable, tommy_search_func* cmp, const void* cmp_arg, tommy_hash_t hash) 84 | { 85 | tommy_size_t pos = hash & hashtable->bucket_mask; 86 | tommy_hashtable_node* node = hashtable->bucket[pos]; 87 | 88 | while (node) { 89 | /* we first check if the hash matches, as in the same bucket we may have multiples hash values */ 90 | if (node->index == hash && cmp(cmp_arg, node->data) == 0) { 91 | tommy_list_remove_existing(&hashtable->bucket[pos], node); 92 | 93 | --hashtable->count; 94 | 95 | return node->data; 96 | } 97 | node = node->next; 98 | } 99 | 100 | return 0; 101 | } 102 | 103 | void tommy_hashtable_foreach(tommy_hashtable* hashtable, tommy_foreach_func* func) 104 | { 105 | tommy_size_t bucket_max = hashtable->bucket_max; 106 | tommy_hashtable_node** bucket = hashtable->bucket; 107 | tommy_size_t pos; 108 | 109 | for (pos = 0; pos < bucket_max; ++pos) { 110 | tommy_hashtable_node* node = bucket[pos]; 111 | 112 | while (node) { 113 | void* data = node->data; 114 | node = node->next; 115 | func(data); 116 | } 117 | } 118 | } 119 | 120 | void tommy_hashtable_foreach_arg(tommy_hashtable* hashtable, tommy_foreach_arg_func* func, void* arg) 121 | { 122 | tommy_size_t bucket_max = hashtable->bucket_max; 123 | tommy_hashtable_node** bucket = hashtable->bucket; 124 | tommy_size_t pos; 125 | 126 | for (pos = 0; pos < bucket_max; ++pos) { 127 | tommy_hashtable_node* node = bucket[pos]; 128 | 129 | while (node) { 130 | void* data = node->data; 131 | node = node->next; 132 | func(arg, data); 133 | } 134 | } 135 | } 136 | 137 | tommy_size_t tommy_hashtable_memory_usage(tommy_hashtable* hashtable) 138 | { 139 | return hashtable->bucket_max * (tommy_size_t)sizeof(hashtable->bucket[0]) 140 | + tommy_hashtable_count(hashtable) * (tommy_size_t)sizeof(tommy_hashtable_node); 141 | } 142 | 143 | -------------------------------------------------------------------------------- /native/runtime/tommyds/tommyds/tommylist.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010, Andrea Mazzoleni. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 | * POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | 28 | #include "tommylist.h" 29 | #include "tommychain.h" 30 | 31 | /** \internal 32 | * Setup a list. 33 | */ 34 | tommy_inline void tommy_list_set(tommy_list* list, tommy_node* head, tommy_node* tail) 35 | { 36 | head->prev = tail; 37 | tail->next = 0; 38 | *list = head; 39 | } 40 | 41 | void tommy_list_sort(tommy_list* list, tommy_compare_func* cmp) 42 | { 43 | tommy_chain chain; 44 | tommy_node* head; 45 | 46 | if (tommy_list_empty(list)) 47 | return; 48 | 49 | head = tommy_list_head(list); 50 | 51 | /* create a chain from the list */ 52 | chain.head = head; 53 | chain.tail = head->prev; 54 | 55 | tommy_chain_mergesort(&chain, cmp); 56 | 57 | /* restore the list */ 58 | tommy_list_set(list, chain.head, chain.tail); 59 | } 60 | 61 | -------------------------------------------------------------------------------- /native/runtime/tommyds/tommyds/tommytree.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015, Andrea Mazzoleni. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 | * POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | 28 | #include "tommytree.h" 29 | 30 | #include /* for assert */ 31 | 32 | /******************************************************************************/ 33 | /* tree */ 34 | 35 | void tommy_tree_init(tommy_tree* tree, tommy_compare_func* cmp) 36 | { 37 | tree->root = 0; 38 | tree->count = 0; 39 | tree->cmp = cmp; 40 | } 41 | 42 | static tommy_ssize_t tommy_tree_delta(tommy_tree_node* root) 43 | { 44 | tommy_ssize_t left_height = root->prev ? root->prev->index : 0; 45 | tommy_ssize_t right_height = root->next ? root->next->index : 0; 46 | 47 | return left_height - right_height; 48 | } 49 | 50 | /* AVL tree operations */ 51 | static tommy_tree_node* tommy_tree_balance(tommy_tree_node*); 52 | 53 | static tommy_tree_node* tommy_tree_rotate_left(tommy_tree_node* root) 54 | { 55 | tommy_tree_node* next = root->next; 56 | 57 | root->next = next->prev; 58 | 59 | next->prev = tommy_tree_balance(root); 60 | 61 | return tommy_tree_balance(next); 62 | } 63 | 64 | static tommy_tree_node* tommy_tree_rotate_right(tommy_tree_node* root) 65 | { 66 | tommy_tree_node* prev = root->prev; 67 | 68 | root->prev = prev->next; 69 | 70 | prev->next = tommy_tree_balance(root); 71 | 72 | return tommy_tree_balance(prev); 73 | } 74 | 75 | static tommy_tree_node* tommy_tree_move_right(tommy_tree_node* root, tommy_tree_node* node) 76 | { 77 | if (!root) 78 | return node; 79 | 80 | root->next = tommy_tree_move_right(root->next, node); 81 | 82 | return tommy_tree_balance(root); 83 | } 84 | 85 | static tommy_tree_node* tommy_tree_balance(tommy_tree_node* root) 86 | { 87 | tommy_ssize_t delta = tommy_tree_delta(root); 88 | 89 | if (delta < -1) { 90 | if (tommy_tree_delta(root->next) > 0) 91 | root->next = tommy_tree_rotate_right(root->next); 92 | return tommy_tree_rotate_left(root); 93 | } 94 | 95 | if (delta > 1) { 96 | if (tommy_tree_delta(root->prev) < 0) 97 | root->prev = tommy_tree_rotate_left(root->prev); 98 | return tommy_tree_rotate_right(root); 99 | } 100 | 101 | /* recompute key */ 102 | root->index = 0; 103 | 104 | if (root->prev && root->prev->index > root->index) 105 | root->index = root->prev->index; 106 | 107 | if (root->next && root->next->index > root->index) 108 | root->index = root->next->index; 109 | 110 | /* count itself */ 111 | root->index += 1; 112 | 113 | return root; 114 | } 115 | 116 | static tommy_tree_node* tommy_tree_insert_node(tommy_compare_func* cmp, tommy_tree_node* root, tommy_tree_node** let) 117 | { 118 | int c; 119 | 120 | if (!root) 121 | return *let; 122 | 123 | c = cmp((*let)->data, root->data); 124 | 125 | if (c < 0) { 126 | root->prev = tommy_tree_insert_node(cmp, root->prev, let); 127 | return tommy_tree_balance(root); 128 | } 129 | 130 | if (c > 0) { 131 | root->next = tommy_tree_insert_node(cmp, root->next, let); 132 | return tommy_tree_balance(root); 133 | } 134 | 135 | /* already present, set the return pointer */ 136 | *let = root; 137 | 138 | return root; 139 | } 140 | 141 | void* tommy_tree_insert(tommy_tree* tree, tommy_tree_node* node, void* data) 142 | { 143 | tommy_tree_node* insert = node; 144 | 145 | insert->data = data; 146 | insert->prev = 0; 147 | insert->next = 0; 148 | insert->index = 0; 149 | 150 | tree->root = tommy_tree_insert_node(tree->cmp, tree->root, &insert); 151 | 152 | if (insert == node) 153 | ++tree->count; 154 | 155 | return insert->data; 156 | } 157 | 158 | static tommy_tree_node* tommy_tree_remove_node(tommy_compare_func* cmp, tommy_tree_node* root, void* data, tommy_tree_node** let) 159 | { 160 | int c; 161 | 162 | if (!root) 163 | return 0; 164 | 165 | c = cmp(data, root->data); 166 | 167 | if (c < 0) { 168 | root->prev = tommy_tree_remove_node(cmp, root->prev, data, let); 169 | return tommy_tree_balance(root); 170 | } 171 | 172 | if (c > 0) { 173 | root->next = tommy_tree_remove_node(cmp, root->next, data, let); 174 | return tommy_tree_balance(root); 175 | } 176 | 177 | /* found */ 178 | *let = root; 179 | 180 | return tommy_tree_move_right(root->prev, root->next); 181 | } 182 | 183 | void* tommy_tree_remove(tommy_tree* tree, void* data) 184 | { 185 | tommy_tree_node* node = 0; 186 | 187 | tree->root = tommy_tree_remove_node(tree->cmp, tree->root, data, &node); 188 | 189 | if (!node) 190 | return 0; 191 | 192 | --tree->count; 193 | 194 | return node->data; 195 | } 196 | 197 | static tommy_tree_node* tommy_tree_search_node(tommy_compare_func* cmp, tommy_tree_node* root, void* data) 198 | { 199 | int c; 200 | 201 | if (!root) 202 | return 0; 203 | 204 | c = cmp(data, root->data); 205 | 206 | if (c < 0) 207 | return tommy_tree_search_node(cmp, root->prev, data); 208 | 209 | if (c > 0) 210 | return tommy_tree_search_node(cmp, root->next, data); 211 | 212 | return root; 213 | } 214 | 215 | void* tommy_tree_search(tommy_tree* tree, void* data) 216 | { 217 | tommy_tree_node* node = tommy_tree_search_node(tree->cmp, tree->root, data); 218 | 219 | if (!node) 220 | return 0; 221 | 222 | return node->data; 223 | } 224 | 225 | void* tommy_tree_search_compare(tommy_tree* tree, tommy_compare_func* cmp, void* cmp_arg) 226 | { 227 | tommy_tree_node* node = tommy_tree_search_node(cmp, tree->root, cmp_arg); 228 | 229 | if (!node) 230 | return 0; 231 | 232 | return node->data; 233 | } 234 | 235 | void* tommy_tree_remove_existing(tommy_tree* tree, tommy_tree_node* node) 236 | { 237 | void* data = tommy_tree_remove(tree, node->data); 238 | 239 | assert(data != 0); 240 | 241 | return data; 242 | } 243 | 244 | static void tommy_tree_foreach_node(tommy_tree_node* root, tommy_foreach_func* func) 245 | { 246 | tommy_tree_node* next; 247 | 248 | if (!root) 249 | return; 250 | 251 | tommy_tree_foreach_node(root->prev, func); 252 | 253 | /* make a copy in case func is free() */ 254 | next = root->next; 255 | 256 | func(root->data); 257 | 258 | tommy_tree_foreach_node(next, func); 259 | } 260 | 261 | void tommy_tree_foreach(tommy_tree* tree, tommy_foreach_func* func) 262 | { 263 | tommy_tree_foreach_node(tree->root, func); 264 | } 265 | 266 | static void tommy_tree_foreach_arg_node(tommy_tree_node* root, tommy_foreach_arg_func* func, void* arg) 267 | { 268 | tommy_tree_node* next; 269 | 270 | if (!root) 271 | return; 272 | 273 | tommy_tree_foreach_arg_node(root->prev, func, arg); 274 | 275 | /* make a copy in case func is free() */ 276 | next = root->next; 277 | 278 | func(arg, root->data); 279 | 280 | tommy_tree_foreach_arg_node(next, func, arg); 281 | } 282 | 283 | void tommy_tree_foreach_arg(tommy_tree* tree, tommy_foreach_arg_func* func, void* arg) 284 | { 285 | tommy_tree_foreach_arg_node(tree->root, func, arg); 286 | } 287 | 288 | tommy_size_t tommy_tree_memory_usage(tommy_tree* tree) 289 | { 290 | return tommy_tree_count(tree) * sizeof(tommy_tree_node); 291 | } 292 | 293 | -------------------------------------------------------------------------------- /native/runtime/tommyds/tommyds/tommytree.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015, Andrea Mazzoleni. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 | * POSSIBILITY OF SUCH DAMAGE. 26 | */ 27 | 28 | /** \file 29 | * AVL tree. 30 | * 31 | * This tree is a standard AVL tree implementation that stores elements in the 32 | * order defined by the comparison function. 33 | * 34 | * As difference than other tommy containers, duplicate elements cannot be inserted. 35 | * 36 | * To initialize a tree you have to call tommy_tree_init() specifing a comparison 37 | * function that will define the order in the tree. 38 | * 39 | * \code 40 | * tommy_tree tree; 41 | * 42 | * tommy_tree_init(&tree, cmp); 43 | * \endcode 44 | * 45 | * To insert elements in the tree you have to call tommy_tree_insert() for 46 | * each element. 47 | * In the insertion call you have to specify the address of the node and the 48 | * address of the object. 49 | * The address of the object is used to initialize the tommy_node::data field 50 | * of the node. 51 | * 52 | * \code 53 | * struct object { 54 | * int value; 55 | * // other fields 56 | * tommy_tree_node node; 57 | * }; 58 | * 59 | * struct object* obj = malloc(sizeof(struct object)); // creates the object 60 | * 61 | * obj->value = ...; // initializes the object 62 | * 63 | * tommy_tree_insert(&tree, &obj->node, obj); // inserts the object 64 | * \endcode 65 | * 66 | * To find and element in the tree you have to call tommy_tree_search() providing 67 | * the key to search. 68 | * 69 | * \code 70 | * struct object value_to_find = { 1 }; 71 | * struct object* obj = tommy_tree_search(&tree, &value_to_find); 72 | * if (!obj) { 73 | * // not found 74 | * } else { 75 | * // found 76 | * } 77 | * \endcode 78 | * 79 | * To remove an element from the tree you have to call tommy_tree_remove() 80 | * providing the key to search and remove. 81 | * 82 | * \code 83 | * struct object value_to_remove = { 1 }; 84 | * struct object* obj = tommy_tree_remove(&tree, &value_to_remove); 85 | * if (obj) { 86 | * free(obj); // frees the object allocated memory 87 | * } 88 | * \endcode 89 | * 90 | * To destroy the tree you have to remove or destroy all the contained elements. 91 | * The tree itself doesn't have or need a deallocation function. 92 | * 93 | * If you need to iterate over all the elements in the tree, you can use 94 | * tommy_tree_foreach() or tommy_tree_foreach_arg(). 95 | * If you need a more precise control with a real iteration, you have to insert 96 | * all the elements also in a ::tommy_list, and use the list to iterate. 97 | * See the \ref multiindex example for more detail. 98 | */ 99 | 100 | #ifndef __TOMMYTREE_H 101 | #define __TOMMYTREE_H 102 | 103 | #include "tommytypes.h" 104 | 105 | /******************************************************************************/ 106 | /* tree */ 107 | 108 | /** 109 | * Tree node. 110 | * This is the node that you have to include inside your objects. 111 | */ 112 | typedef tommy_node tommy_tree_node; 113 | 114 | /** 115 | * Tree container type. 116 | * \note Don't use internal fields directly, but access the container only using functions. 117 | */ 118 | typedef struct tommy_tree_struct { 119 | tommy_tree_node* root; /**< Root node. */ 120 | tommy_compare_func* cmp; /**< Comparison function. */ 121 | tommy_size_t count; /**< Number of elements. */ 122 | } tommy_tree; 123 | 124 | /** 125 | * Initializes the tree. 126 | * \param cmp The comparison function that defines the orderin the tree. 127 | */ 128 | void tommy_tree_init(tommy_tree* tree, tommy_compare_func* cmp); 129 | 130 | /** 131 | * Inserts an element in the tree. 132 | * If the element is already present, it's not inserted again. 133 | * Check the return value to identify if the element was already present or not. 134 | * You have to provide the pointer of the node embedded into the object and 135 | * the pointer to the object. 136 | * \param node Pointer to the node embedded into the object to insert. 137 | * \param data Pointer to the object to insert. 138 | * \return The element in the tree. Either the already existing one, or the one just inserted. 139 | */ 140 | void* tommy_tree_insert(tommy_tree* tree, tommy_tree_node* node, void* data); 141 | 142 | /** 143 | * Searches and removes an element. 144 | * If the element is not found, 0 is returned. 145 | * \param data Element used for comparison. 146 | * \return The removed element, or 0 if not found. 147 | */ 148 | void* tommy_tree_remove(tommy_tree* tree, void* data); 149 | 150 | /** 151 | * Searches an element in the tree. 152 | * If the element is not found, 0 is returned. 153 | * \param data Element used for comparison. 154 | * \return The first element found, or 0 if none. 155 | */ 156 | void* tommy_tree_search(tommy_tree* tree, void* data); 157 | 158 | /** 159 | * Searches an element in the tree with a specific comparison function. 160 | * 161 | * Like tommy_tree_search() but you can specify a different comparison function. 162 | * Note that this function must define a suborder of the original one. 163 | * 164 | * The ::data argument will be the first argument of the comparison function, 165 | * and it can be of a different type of the objects in the tree. 166 | */ 167 | void* tommy_tree_search_compare(tommy_tree* tree, tommy_compare_func* cmp, void* cmp_arg); 168 | 169 | /** 170 | * Removes an element from the tree. 171 | * You must already have the address of the element to remove. 172 | * \return The tommy_node::data field of the node removed. 173 | */ 174 | void* tommy_tree_remove_existing(tommy_tree* tree, tommy_tree_node* node); 175 | 176 | /** 177 | * Calls the specified function for each element in the tree. 178 | * 179 | * The elements are processed in order. 180 | * 181 | * You cannot add or remove elements from the inside of the callback, 182 | * but can use it to deallocate them. 183 | * 184 | * \code 185 | * tommy_tree tree; 186 | * 187 | * // initializes the tree 188 | * tommy_tree_init(&tree, cmp); 189 | * 190 | * ... 191 | * 192 | * // creates an object 193 | * struct object* obj = malloc(sizeof(struct object)); 194 | * 195 | * ... 196 | * 197 | * // insert it in the tree 198 | * tommy_tree_insert(&tree, &obj->node, obj); 199 | * 200 | * ... 201 | * 202 | * // deallocates all the objects iterating the tree 203 | * tommy_tree_foreach(&tree, free); 204 | * \endcode 205 | */ 206 | void tommy_tree_foreach(tommy_tree* tree, tommy_foreach_func* func); 207 | 208 | /** 209 | * Calls the specified function with an argument for each element in the tree. 210 | */ 211 | void tommy_tree_foreach_arg(tommy_tree* tree, tommy_foreach_arg_func* func, void* arg); 212 | 213 | /** 214 | * Gets the number of elements. 215 | */ 216 | tommy_inline tommy_size_t tommy_tree_count(tommy_tree* tree) 217 | { 218 | return tree->count; 219 | } 220 | 221 | /** 222 | * Gets the size of allocated memory. 223 | * It includes the size of the ::tommy_tree_node of the stored elements. 224 | */ 225 | tommy_size_t tommy_tree_memory_usage(tommy_tree* tree); 226 | 227 | #endif 228 | 229 | -------------------------------------------------------------------------------- /native/runtime/tommyds/tommyweb-header.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | TommyDS 6 | 7 | 8 | 9 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /native/runtime/types.h: -------------------------------------------------------------------------------- 1 | #ifndef WEAVER_RUNTIME_TYPES_H 2 | #define WEAVER_RUNTIME_TYPES_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | typedef uint8_t WV_Byte; 10 | typedef uint8_t WV_U8; 11 | typedef uint16_t WV_U16; 12 | typedef uint32_t WV_U32; 13 | typedef uint64_t WV_U64; 14 | typedef int32_t WV_I32; 15 | typedef double WV_F; 16 | typedef void* WV_Any; 17 | 18 | const static WV_U32 WV_U32_MAX = 0xffffffff; 19 | 20 | static inline WV_U32 WV_SafeAdd32(WV_U32 a, WV_U32 b) 21 | { 22 | if (WV_U32_MAX - a < b) { 23 | return WV_U32_MAX; 24 | } 25 | return a + b; 26 | } 27 | 28 | static inline WV_U16 WV_NToH16(WV_U16 x) 29 | { 30 | return ntohs(x); 31 | } 32 | 33 | static inline WV_U32 WV_NToH32(WV_U32 x) 34 | { 35 | return ntohl(x); 36 | } 37 | 38 | typedef struct _WV_ByteSlice { 39 | const WV_Byte* cursor; 40 | WV_U32 length; 41 | } WV_ByteSlice; 42 | 43 | static const WV_ByteSlice WV_EMPTY = { .cursor = NULL, .length = 0 }; 44 | 45 | static inline WV_ByteSlice WV_SliceAfter(WV_ByteSlice slice, WV_U32 index) 46 | { 47 | if (slice.cursor == NULL) { 48 | return WV_EMPTY; 49 | } 50 | if (slice.length < index) { 51 | return (WV_ByteSlice){ .cursor = slice.cursor + slice.length, .length = 0 }; 52 | } 53 | WV_ByteSlice after = { .cursor = slice.cursor + index, .length = slice.length - index }; 54 | return after; 55 | } 56 | 57 | static inline WV_ByteSlice WV_SliceBefore(WV_ByteSlice slice, WV_U32 index) 58 | { 59 | if (slice.cursor == NULL) { 60 | return WV_EMPTY; 61 | } 62 | if (slice.length < index) { 63 | return slice; 64 | } 65 | return (WV_ByteSlice){ .cursor = slice.cursor, .length = index }; 66 | } 67 | 68 | static inline WV_U8 WV_UpdateV(WV_U8* v, WV_U8 expr) 69 | { 70 | if (expr) { 71 | *v = 1; 72 | } 73 | return *v; 74 | } 75 | 76 | #endif -------------------------------------------------------------------------------- /native/weaver.h: -------------------------------------------------------------------------------- 1 | // Weaver Runtime Header 2 | #ifndef WV_WEAVER_H 3 | #define WV_WEAVER_H 4 | 5 | #include "runtime/malloc.h" 6 | #include "runtime/profile.h" 7 | #include "runtime/seq.h" 8 | #include "runtime/types.h" 9 | #include "runtime/timer.h" 10 | #include 11 | #include 12 | 13 | // implemented by blackbox 14 | typedef struct _WV_Runtime WV_Runtime; 15 | WV_U8 WV_ProcessPacket(WV_ByteSlice, WV_Runtime *); 16 | WV_Runtime *WV_AllocRuntime(); 17 | WV_U8 WV_FreeRuntime(WV_Runtime *); 18 | WV_Profile *WV_GetProfile(WV_Runtime *); 19 | // blackbox implemented end 20 | 21 | // implemented by whitebox 22 | WV_U8 WV_Setup(); 23 | // whitebox implemented end 24 | 25 | #endif -------------------------------------------------------------------------------- /rubik/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ants-xjtu/rubik/107cc58254f56c02902f55c2b87054f5ebfcd502/rubik/__init__.py -------------------------------------------------------------------------------- /rubik/__main__.py: -------------------------------------------------------------------------------- 1 | from sys import argv 2 | from importlib import import_module 3 | 4 | from rubik.compile import compile5a_layer, OptimizeDriver 5 | from rubik.compile2 import compile7_stack, compile7w_stack 6 | 7 | 8 | stack = import_module(argv[1]).stack 9 | block_map = { 10 | # layer.layer.context.layer_id: compile5a_layer(layer.layer) 11 | layer.layer.context.layer_id: compile5a_layer(layer.layer).optimize( 12 | OptimizeDriver(layer.layer.context) 13 | ) 14 | for layer in stack.name_map.values() 15 | } 16 | inst_decls = { 17 | layer.layer.context.layer_id: layer.layer.context.inst.decl(layer.layer.context) 18 | for layer in stack.name_map.values() 19 | if layer.layer.context.inst is not None 20 | } 21 | 22 | print("/* Weaver Whitebox Code Template */") 23 | print(compile7w_stack(stack.context)) 24 | print("/* Weaver Auto-generated Blackbox Code */") 25 | print( 26 | compile7_stack( 27 | stack.context, block_map, inst_decls, stack.entry.layer.context.layer_id, 28 | {layer.layer.context.layer_id: layer.layer.context for layer in stack.name_map.values()} 29 | ) 30 | ) 31 | -------------------------------------------------------------------------------- /rubik/util.py: -------------------------------------------------------------------------------- 1 | def make_block(text: str) -> str: 2 | if text: 3 | text = ("\n" + text).replace("\n", "\n ") + "\n" 4 | return "{" + text + "}" 5 | 6 | 7 | def indent_join(text_list): 8 | return make_block("\n".join(text_list)) 9 | 10 | 11 | def code_comment(text, comment): 12 | return "// " + comment.replace("\n", "\n// ") + f"\n{text}" 13 | 14 | 15 | def comment_only(comment): 16 | return f"// LOGICALLY " + comment.replace("\n", "\n// ") 17 | 18 | -------------------------------------------------------------------------------- /stock/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ants-xjtu/rubik/107cc58254f56c02902f55c2b87054f5ebfcd502/stock/__init__.py -------------------------------------------------------------------------------- /stock/gtp.py: -------------------------------------------------------------------------------- 1 | from rubik.lang import Stack, Predicate 2 | from stock.protocols import eth_parser, ip_parser, udp_parser, gtp_parser, tcp_parser 3 | 4 | 5 | stack = Stack() 6 | stack.eth = eth_parser() 7 | stack.ip1 = ip_parser() 8 | stack.udp = udp_parser() 9 | stack.gtp = gtp_parser() 10 | stack.ip2 = ip_parser() 11 | stack.tcp = tcp_parser(stack.ip2) 12 | 13 | stack += (stack.eth >> stack.ip1) + Predicate(1) 14 | stack += (stack.ip1 >> stack.udp) + Predicate( 15 | (stack.ip1.psm.dump | stack.ip1.psm.last) & (stack.ip1.header.protocol == 17) 16 | ) 17 | stack += (stack.udp >> stack.gtp) + Predicate(1) 18 | stack += (stack.gtp >> stack.ip2) + Predicate(stack.gtp.header.MT == 255) 19 | stack += (stack.ip2 >> stack.tcp) + Predicate( 20 | (stack.ip2.psm.dump | stack.ip2.psm.last) & (stack.ip2.header.protocol == 6) 21 | ) 22 | -------------------------------------------------------------------------------- /stock/pptp.py: -------------------------------------------------------------------------------- 1 | # pylint: disable = unused-wildcard-import 2 | from rubik.lang import * 3 | 4 | from stock.protocols.eth import eth_parser 5 | from stock.protocols.tcp import tcp_parser 6 | from stock.protocols.ip import ip_parser 7 | from stock.protocols.udp import udp_parser 8 | from stock.protocols.gre import gre_parser 9 | from stock.protocols.ppp import ppp_parser 10 | from stock.protocols.pptp import pptp_parser 11 | from stock.protocols.tcp import tcp_parser 12 | 13 | 14 | stack = Stack() 15 | stack.eth = eth_parser() 16 | stack.ip1 = ip_parser() 17 | stack.tcp_ctl = tcp_parser(stack.ip1) 18 | stack.pptp = pptp_parser(stack.ip1) 19 | stack.gre = gre_parser(stack.ip1) 20 | stack.ppp = ppp_parser(stack.ip1, stack.gre) 21 | stack.ip2 = ip_parser() 22 | stack.udp = udp_parser() 23 | stack.tcp = tcp_parser(stack.ip2) 24 | 25 | stack += (stack.eth >> stack.ip1) + Predicate(1) 26 | stack += (stack.ip1 >> stack.tcp_ctl) + Predicate( 27 | (stack.ip1.psm.dump | stack.ip1.psm.last) & (stack.ip1.header.protocol == 6) 28 | ) 29 | stack += (stack.tcp_ctl >> stack.pptp) + Predicate( 30 | stack.tcp_ctl.psm.buffering & (stack.tcp_ctl.sdu.length != 0) 31 | ) 32 | stack += (stack.ip1 >> stack.gre) + Predicate( 33 | (stack.ip1.psm.last | stack.ip1.psm.dump) & (stack.ip1.header.protocol == 47) 34 | ) 35 | stack += (stack.gre >> stack.ppp) + Predicate( 36 | (stack.gre.header.protocol == 0x880B) & stack.gre.psm.tunneling 37 | ) 38 | stack += (stack.ppp >> stack.ip2) + Predicate( 39 | (stack.ppp.temp.protocol == 0x0021) & stack.ppp.psm.tunneling 40 | ) 41 | stack += (stack.ip2 >> stack.tcp) + Predicate(stack.ip2.header.protocol == 6) 42 | stack += (stack.ip2 >> stack.udp) + Predicate(stack.ip2.header.protocol == 17) 43 | -------------------------------------------------------------------------------- /stock/protocols/__init__.py: -------------------------------------------------------------------------------- 1 | from stock.protocols.eth import eth_parser 2 | from stock.protocols.ip import ip_parser 3 | from stock.protocols.tcp import tcp_parser 4 | from stock.protocols.udp import udp_parser 5 | from stock.protocols.gtp import gtp_parser 6 | from stock.protocols.sctp import sctp_parser 7 | -------------------------------------------------------------------------------- /stock/protocols/eth.py: -------------------------------------------------------------------------------- 1 | from rubik.lang import layout, Bit, Connectionless 2 | 3 | 4 | class ethernet_hdr(layout): 5 | blank1 = Bit(64) 6 | blank2 = Bit(32) 7 | blank3 = Bit(16) 8 | 9 | 10 | def eth_parser(): 11 | eth = Connectionless() 12 | eth.header = ethernet_hdr 13 | return eth 14 | -------------------------------------------------------------------------------- /stock/protocols/gre.py: -------------------------------------------------------------------------------- 1 | # pylint: disable = unused-wildcard-import 2 | from rubik.lang import * 3 | 4 | 5 | class GRE_header(layout): 6 | C = Bit(1) 7 | R = Bit(1) 8 | K = Bit(1) 9 | S = Bit(1) 10 | strict_source_route = Bit(1) 11 | recursion_control = Bit(3) 12 | A = Bit(1) 13 | reserved = Bit(4) 14 | version = Bit(3) 15 | protocol = UInt(16) 16 | payload_length = UInt(16) 17 | call_ID = Bit(16) 18 | 19 | 20 | class GRE_sequence_number(layout): 21 | sequence_number = UInt(32) 22 | 23 | 24 | class GRE_ack_number(layout): 25 | ack_number = UInt(32) 26 | 27 | 28 | class GRE_perm(layout): 29 | short_PPP = Bit(8, init=0) 30 | active_offset = Bit(32, init=0) 31 | passive_offset = Bit(32, init=0) 32 | 33 | 34 | class GRE_temp(layout): 35 | offset = Bit(64) 36 | 37 | 38 | def gre_parser(ip): 39 | gre = ConnectionOriented() 40 | 41 | gre.header = GRE_header 42 | gre.header += If(gre.header.S) >> GRE_sequence_number 43 | gre.header += If(gre.header.A) >> GRE_ack_number 44 | 45 | gre.selector = ([ip.header.saddr], [ip.header.daddr]) 46 | 47 | gre.perm = GRE_perm 48 | gre.temp = GRE_temp 49 | 50 | gre.prep = ( 51 | If((gre.header.payload_length != 0) & (gre.to_active)) 52 | >> Assign(gre.temp.offset, gre.perm.passive_offset) 53 | + Assign( 54 | gre.perm.passive_offset, gre.perm.passive_offset + gre.header.payload_length 55 | ) 56 | ) + ( 57 | If((gre.header.payload_length != 0) & (gre.to_passive)) 58 | >> Assign(gre.temp.offset, gre.perm.active_offset) 59 | + Assign( 60 | gre.perm.active_offset, gre.perm.active_offset + gre.header.payload_length 61 | ) 62 | ) 63 | 64 | # gre.seq = Sequence(meta=gre.temp.offset, data=gre.payload, data_len=gre.payload_len) 65 | 66 | dump = PSMState(start=True) 67 | nothing = PSMState(accept=True) 68 | gre.psm = PSM(dump, nothing) 69 | gre.psm.tunneling = (dump >> dump) + Predicate(gre.header.payload_length != 0) 70 | gre.psm.only_ack = (dump >> dump) + Predicate(gre.header.payload_length == 0) 71 | 72 | # gre.event.asm = If(gre.psm.tunneling) >> Assemble() 73 | 74 | return gre 75 | -------------------------------------------------------------------------------- /stock/protocols/gtp.py: -------------------------------------------------------------------------------- 1 | from rubik.lang import layout, Bit, Connectionless, If, AnyUntil 2 | 3 | 4 | class gtp_hdr(layout): 5 | version = Bit(3) 6 | PT = Bit(1) 7 | Reserved = Bit(1) 8 | E = Bit(1) 9 | S = Bit(1) 10 | PN = Bit(1) 11 | MT = Bit(8) 12 | Total_length = Bit(16) 13 | TEID = Bit(32) 14 | 15 | 16 | class var_header(layout): 17 | seq_no_upper = Bit(16) 18 | npdu_no = Bit(8) 19 | next_header_type = Bit(8) 20 | 21 | 22 | class gtp_extension_header(layout): 23 | extension_length = Bit(8) 24 | extension_data = Bit(((extension_length << 2) - 2) << 3) 25 | next_type = Bit(8) 26 | 27 | 28 | def gtp_parser(): 29 | gtp = Connectionless() 30 | gtp.header = gtp_hdr 31 | gtp.header += ( 32 | If((gtp.header.E == 1) | (gtp.header.S == 1) | (gtp.header.PN == 1)) 33 | >> var_header 34 | ) 35 | gtp.header += If(gtp.header.E == 1) >> AnyUntil( 36 | [gtp_extension_header], 37 | (gtp_extension_header.next_type != 0) & (gtp.payload_len != 0), 38 | ) 39 | return gtp 40 | -------------------------------------------------------------------------------- /stock/protocols/ip.py: -------------------------------------------------------------------------------- 1 | from rubik.lang import ( 2 | Connectionless, 3 | layout, 4 | Bit, 5 | UInt, 6 | Assign, 7 | Sequence, 8 | PSM, 9 | PSMState, 10 | Pred, 11 | If, 12 | Assemble, 13 | ) 14 | 15 | 16 | class ip_hdr(layout): 17 | version = Bit(4) 18 | ihl = Bit(4) 19 | tos = Bit(8) 20 | tot_len = UInt(16) 21 | id = Bit(16) 22 | blank = Bit(1) 23 | dont_frag = Bit(1) 24 | more_frag = Bit(1) 25 | f1 = Bit(5) 26 | f2 = Bit(8) 27 | ttl = Bit(8) 28 | protocol = Bit(8) 29 | check = Bit(16) 30 | saddr = Bit(32) 31 | daddr = Bit(32) 32 | 33 | 34 | class ip_temp(layout): 35 | offset = Bit(16) 36 | length = Bit(16) 37 | 38 | 39 | def ip_parser(): 40 | ip = Connectionless() 41 | 42 | ip.header = ip_hdr 43 | ip.selector = [ip.header.saddr, ip.header.daddr] 44 | 45 | ip.temp = ip_temp 46 | ip.prep = Assign( 47 | ip.temp.offset, ((ip.header.f1 << 8) + ip.header.f2) << 3 48 | ) + Assign(ip.temp.length, ip.header.tot_len - (ip.header.ihl << 2)) 49 | 50 | ip.seq = Sequence(meta=ip.temp.offset, data=ip.payload[: ip.temp.length]) 51 | 52 | DUMP = PSMState(start=True, accept=True) 53 | FRAG = PSMState() 54 | ip.psm = PSM(DUMP, FRAG) 55 | ip.psm.dump = (DUMP >> DUMP) + Pred( 56 | ((ip.header.dont_frag == 1) & (ip.temp.offset == 0)) 57 | | ((ip.header.more_frag == 0) & (ip.temp.offset == 0)) 58 | ) 59 | ip.psm.frag = (DUMP >> FRAG) + Pred( 60 | (ip.header.more_frag == 1) | (ip.temp.offset != 0) 61 | ) 62 | ip.psm.more = (FRAG >> FRAG) + Pred(ip.header.more_frag == 1) 63 | ip.psm.last = (FRAG >> DUMP) + Pred(ip.v.header.more_frag == 0) 64 | 65 | ip.event.asm = If(ip.psm.dump | ip.psm.last) >> Assemble() 66 | 67 | return ip 68 | -------------------------------------------------------------------------------- /stock/protocols/loopback.py: -------------------------------------------------------------------------------- 1 | from rubik.lang import layout, Bit, Connectionless 2 | 3 | class loopback_hdr(layout): 4 | family = Bit(32) 5 | 6 | def loopback_parser(): 7 | loopback = Connectionless() 8 | loopback.header = loopback_hdr 9 | return loopback 10 | -------------------------------------------------------------------------------- /stock/protocols/ppp.py: -------------------------------------------------------------------------------- 1 | # pylint: disable = unused-wildcard-import 2 | from rubik.lang import * 3 | 4 | 5 | class PPP_short_header(layout): 6 | short_protocol = Bit(1 << 3) 7 | 8 | 9 | class PPP_header(layout): 10 | address = Bit(8) 11 | control = Bit(8) 12 | protocol = UInt(2 << 3) 13 | 14 | 15 | class PPP_LCP_header(layout): 16 | LCP_code = Bit(8) 17 | LCP_ID = Bit(8) 18 | LCP_length = UInt(2 << 3) 19 | 20 | 21 | class PPP_IPCP_header(layout): 22 | PPP_IPCP_code = Bit(8) 23 | PPP_IPCP_ID = Bit(8) 24 | PPP_IPCP_length = UInt(2 << 3) 25 | PPP_IPCP_data = Bit(PPP_IPCP_length - 4) 26 | 27 | 28 | class PPP_CHAP_header(layout): 29 | CHAP_code = Bit(8) 30 | CHAP_ID = Bit(8) 31 | CHAP_length = UInt(2 << 3) 32 | CHAP_data = Bit(CHAP_length - 4) 33 | 34 | 35 | class PPP_CCP_header(layout): 36 | CCP_code = Bit(8) 37 | CCP_ID = Bit(8) 38 | CCP_length = UInt(2 << 3) 39 | CCP_data = Bit(CCP_length - 4) 40 | 41 | 42 | class PPP_IPV6CP_header(layout): 43 | IPV6CP_code = Bit(8) 44 | IPV6CP_ID = Bit(8) 45 | IPV6CP_length = UInt(2 << 3) 46 | IPV6CP_data = Bit(IPV6CP_length - 4) 47 | 48 | 49 | class PPP_LCP_ACCM_option(layout): 50 | ACCM_type = Bit(8, const=2) 51 | ACCM_length = Bit(8) 52 | ACCM_value = Bit((ACCM_length - 2) << 3) 53 | 54 | 55 | class PPP_LCP_AP_option(layout): 56 | AP_type = Bit(8, const=3) 57 | AP_length = Bit(8) 58 | AP_value = Bit((AP_length - 2) << 3) 59 | 60 | 61 | class PPP_LCP_MN_option(layout): 62 | MN_type = Bit(8, const=5) 63 | MN_length = Bit(8) 64 | MN_value = Bit((MN_length - 2) << 3) 65 | 66 | 67 | class PPP_LCP_PFC_option(layout): 68 | PFC_type = Bit(8, const=7) 69 | PFC_length = Bit(8) 70 | 71 | 72 | class PPP_LCP_ACFC_option(layout): 73 | ACFC_type = Bit(8, const=8) 74 | ACFC_length = Bit(8) 75 | 76 | 77 | class PPP_LCP_MRU_option(layout): 78 | MRU_type = Bit(8, const=1) 79 | MRU_length = Bit(8) 80 | MRU_value = Bit((MRU_length - 2) << 3) 81 | 82 | 83 | class PPP_temp_data(layout): 84 | protocol = Bit(32) 85 | 86 | 87 | def ppp_parser(ip, gre): 88 | PPP = ConnectionOriented() 89 | ppp_long = PPP_header 90 | ppp_long += If(ppp_long.protocol == 0xC223) >> PPP_CHAP_header 91 | ppp_long += If(ppp_long.protocol == 0x80FD) >> PPP_CCP_header 92 | ppp_long += If(ppp_long.protocol == 0x8021) >> PPP_IPCP_header 93 | ppp_long += If(ppp_long.protocol == 0x8057) >> PPP_IPV6CP_header 94 | ppp_long += If(ppp_long.protocol == 0xC021) >> PPP_LCP_header + AnyUntil( 95 | [ 96 | PPP_LCP_ACCM_option, 97 | PPP_LCP_AP_option, 98 | PPP_LCP_MN_option, 99 | PPP_LCP_PFC_option, 100 | PPP_LCP_ACFC_option, 101 | PPP_LCP_MRU_option, 102 | ], 103 | PPP.cursor < PPP_LCP_header.LCP_length + 4, 104 | ) 105 | ppp_short = PPP_short_header 106 | ppp_short += If(ppp_short.short_protocol == 0xC223) >> PPP_CHAP_header 107 | ppp_short += If(ppp_short.short_protocol == 0x80FD) >> PPP_CCP_header 108 | ppp_short += If(ppp_short.short_protocol == 0x8021) >> PPP_IPCP_header 109 | ppp_short += If(ppp_short.short_protocol == 0x8057) >> PPP_IPV6CP_header 110 | ppp_short += If(ppp_short.short_protocol == 0xC021) >> PPP_LCP_header 111 | 112 | class blank(layout): 113 | pass 114 | 115 | PPP.header = blank + (If(gre.perm.short_PPP == 0) >> ppp_long) 116 | PPP.header += If(gre.perm.short_PPP == 1) >> ppp_short 117 | 118 | PPP.temp = PPP_temp_data 119 | 120 | PPP.prep = ( 121 | If(gre.perm.short_PPP) >> Assign(PPP.temp.protocol, PPP.header.short_protocol) 122 | ) + (If(gre.perm.short_PPP == 0) >> Assign(PPP.temp.protocol, PPP.header.protocol)) 123 | 124 | PPP.selector = ([ip.header.saddr], [ip.header.daddr]) 125 | 126 | SESSION = PSMState(start=True, accept=True) 127 | ( 128 | active_passive_sent_ACFC, 129 | passive_sent_ACFC_active_no, 130 | active_ACFC_acked_passive_no, 131 | ) = make_psm_state(3) 132 | 133 | PPP.psm = PSM( 134 | SESSION, 135 | active_passive_sent_ACFC, 136 | passive_sent_ACFC_active_no, 137 | active_ACFC_acked_passive_no, 138 | ) 139 | 140 | PPP.psm.config = (SESSION >> SESSION) + Predicate( 141 | (PPP.temp.protocol != 0x0021) & (PPP.header_contain(PPP_LCP_ACFC_option) == 0) 142 | ) 143 | 144 | PPP.psm.p_sent_AFCF = (SESSION >> passive_sent_ACFC_active_no) + Predicate( 145 | PPP.to_passive & PPP.header_contain(PPP_LCP_ACFC_option) 146 | ) 147 | PPP.psm.a_after_p_sent_AFCF = ( 148 | passive_sent_ACFC_active_no >> active_passive_sent_ACFC 149 | ) + Predicate(PPP.to_active & PPP.header_contain(PPP_LCP_ACFC_option)) 150 | 151 | PPP.psm.a_AFCF_acked = ( 152 | active_passive_sent_ACFC >> active_ACFC_acked_passive_no 153 | ) + Predicate(PPP.to_passive) 154 | 155 | PPP.psm.a_p_AFCF_acked = (active_ACFC_acked_passive_no >> SESSION) + Predicate( 156 | PPP.to_active 157 | ) 158 | 159 | PPP.psm.tunneling = (SESSION >> SESSION) + Predicate(PPP.temp.protocol == 0x0021) 160 | 161 | PPP.event.switch_to_short = If(PPP.psm.a_p_AFCF_acked) >> Assign( 162 | gre.perm.short_PPP, 1 163 | ) 164 | 165 | return PPP 166 | -------------------------------------------------------------------------------- /stock/protocols/pptp.py: -------------------------------------------------------------------------------- 1 | # pylint: disable = unused-wildcard-import 2 | from rubik.lang import * 3 | 4 | 5 | class pptp_general(layout): 6 | length = UInt(16) 7 | pptp_message_type = UInt(16) 8 | magic_cookie = Bit(32) 9 | 10 | 11 | class start_control_connection_request(layout): 12 | SCCRQ_type = Bit(16, const=256) 13 | SCCRQ_reserved0 = Bit(16) 14 | SCCRQ_protocol_version = Bit(16) 15 | SCCRQ_reserved1 = Bit(16) 16 | SCCRQ_framing_capabilities = Bit(32) 17 | SCCRQ_bearer_capabilities = Bit(32) 18 | SCCRQ_maximum_channels = Bit(16) 19 | SCCRQ_firmware_revision = Bit(16) 20 | SCCRQ_host_name = Bit(64) 21 | SCCRQ_vandor_name = Bit(64) 22 | 23 | 24 | class start_control_connection_reply(layout): 25 | SCCRP_type = Bit(16, const=512) 26 | SCCRP_reserved0 = Bit(16) 27 | SCCRP_protocol_version = Bit(16) 28 | SCCRP_result_code = Bit(8) 29 | SCCRP_error_code = Bit(8) 30 | SCCRP_framing_capabilities = Bit(32) 31 | SCCRP_bearer_capabilities = Bit(32) 32 | SCCRP_maximum_channels = Bit(16) 33 | SCCRP_firmware_revision = Bit(16) 34 | SCCRP_host_name = Bit(64) 35 | SCCRP_vandor_name = Bit(64) 36 | 37 | 38 | class outgoing_call_request(layout): 39 | OCRQ_type = Bit(16, const=1792) 40 | OCRQ_reserved0 = Bit(16) 41 | OCRQ_call_ID = Bit(16) 42 | OCRQ_call_serial_number = Bit(16) 43 | OCRQ_minimum_BPS = Bit(32) 44 | OCRQ_maximum_BPS = Bit(32) 45 | OCRQ_bearer_type = Bit(32) 46 | OCRQ_framing_type = Bit(32) 47 | OCRQ_packet_recv_ws = Bit(16) 48 | OCRQ_packet_processing_delay = Bit(16) 49 | OCRQ_phone_number_length = Bit(16) 50 | OCRQ_reserved = Bit(16) 51 | OCRQ_phone_number = Bit(64) 52 | OCRQ_subaddress = Bit(64) 53 | 54 | 55 | class outgoing_call_reply(layout): 56 | OCRP_type = Bit(16, const=2048) 57 | OCRP_reserved0 = Bit(16) 58 | OCRP_call_ID = Bit(16) 59 | OCRP_peer_call_ID = Bit(16) 60 | OCRP_result_code = Bit(8) 61 | OCRP_error_code = Bit(8) 62 | OCRP_cause_code = Bit(16) 63 | OCRP_connect_speed = Bit(32) 64 | OCRP_packet_recv_ws = Bit(16) 65 | OCRP_packet_processing_delay = Bit(16) 66 | OCRP_physical_channel_ID = Bit(32) 67 | 68 | 69 | class set_link_info(layout): 70 | SLI_type = Bit(16, const=3840) 71 | SLI_reserved0 = Bit(16) 72 | SLI_peer_call_ID = Bit(16) 73 | SLI_reserved = Bit(16) 74 | SLI_send_ACCM = Bit(32) 75 | SLI_recv_ACCM = Bit(32) 76 | 77 | 78 | class echo_request(layout): 79 | ERQ_type = Bit(16, const=1280) 80 | ERQ_reserved0 = Bit(16) 81 | ERQ_identifier = Bit(32) 82 | 83 | 84 | class echo_reply(layout): 85 | ERP_type = Bit(16, const=1536) 86 | ERP_reserved0 = Bit(16) 87 | ERP_identifier = Bit(32) 88 | ERP_result_code = Bit(8) 89 | ERP_error_code = Bit(8) 90 | ERP_reserved1 = Bit(16) 91 | 92 | 93 | def pptp_parser(ip): 94 | pptp = ConnectionOriented() 95 | pptp.header = pptp_general + AnyUntil( 96 | [ 97 | start_control_connection_request, 98 | start_control_connection_reply, 99 | outgoing_call_request, 100 | outgoing_call_reply, 101 | set_link_info, 102 | echo_request, 103 | echo_reply, 104 | ], 105 | Const(0), 106 | ) 107 | pptp.selector = ([ip.header.saddr], [ip.header.daddr]) 108 | 109 | CLOSED = PSMState(start=True, accept=True) 110 | ( 111 | REQUEST_CONNECT, 112 | CONNECTION_ESTABLISHED, 113 | REQUEST_SESSION, 114 | SESSION_ESTABLISHED, 115 | ECHO_SENT, 116 | ) = make_psm_state(5) 117 | 118 | pptp.psm = PSM( 119 | CLOSED, 120 | REQUEST_CONNECT, 121 | CONNECTION_ESTABLISHED, 122 | REQUEST_SESSION, 123 | SESSION_ESTABLISHED, 124 | ECHO_SENT, 125 | ) 126 | 127 | pptp.psm.SCCRQ_sent = (CLOSED >> REQUEST_CONNECT) + Predicate( 128 | pptp.to_passive & pptp.header_contain(start_control_connection_request) 129 | ) 130 | 131 | pptp.psm.SCCRP_sent = (REQUEST_CONNECT >> CONNECTION_ESTABLISHED) + Predicate( 132 | pptp.to_active & pptp.header_contain(start_control_connection_reply) 133 | ) 134 | 135 | pptp.psm.OCRQ_sent = (CONNECTION_ESTABLISHED >> REQUEST_SESSION) + Predicate( 136 | pptp.to_passive & pptp.header_contain(outgoing_call_request) 137 | ) 138 | 139 | pptp.psm.OCRP_sent = (REQUEST_SESSION >> SESSION_ESTABLISHED) + Predicate( 140 | pptp.to_active & pptp.header_contain(outgoing_call_reply) 141 | ) 142 | 143 | pptp.psm.session_config_passive = ( 144 | SESSION_ESTABLISHED >> SESSION_ESTABLISHED 145 | ) + Predicate(pptp.to_active & pptp.header_contain(set_link_info)) 146 | 147 | pptp.psm.session_config_active = ( 148 | SESSION_ESTABLISHED >> SESSION_ESTABLISHED 149 | ) + Predicate(pptp.to_passive & pptp.header_contain(set_link_info)) 150 | 151 | pptp.psm.keep_alive = (SESSION_ESTABLISHED >> ECHO_SENT) + Predicate( 152 | pptp.to_passive & pptp.header_contain(echo_request) 153 | ) 154 | 155 | pptp.psm.maintain_connection = (ECHO_SENT >> SESSION_ESTABLISHED) + Predicate( 156 | pptp.to_active & pptp.header_contain(echo_reply) 157 | ) 158 | 159 | return pptp 160 | -------------------------------------------------------------------------------- /stock/protocols/quic_frame.py: -------------------------------------------------------------------------------- 1 | # pylint: disable = unused-wildcard-import 2 | from rubik.lang import * 3 | 4 | 5 | class QUICFrame(layout): 6 | frame_type = Bit(8) 7 | 8 | 9 | class StreamID(layout): 10 | stream_id = Bit(8) 11 | 12 | 13 | def tail_bit(head): 14 | return Bit(((Const(1) << ((head & 0xC0) >> 6)) - 1) << 3) 15 | 16 | 17 | class StreamIDTail(layout): 18 | stream_id_tail = tail_bit(StreamID.stream_id) 19 | 20 | 21 | class FrameOffset(layout): 22 | frame_offset = Bit(8) 23 | 24 | 25 | class FrameOffsetTail(layout): 26 | frame_offset_tail = tail_bit(FrameOffset.frame_offset) 27 | 28 | 29 | class FrameLength(layout): 30 | frame_length = Bit(8) 31 | 32 | 33 | class FrameLengthTail(layout): 34 | frame_length_tail = tail_bit(FrameLength.frame_length) 35 | 36 | 37 | class ConnectionClose(layout): 38 | error_code = Bit(16) 39 | reason_phrase_length = Bit(8) 40 | reason_phrase = Bit(reason_phrase_length << 3) 41 | 42 | 43 | class QUICACK(layout): 44 | last_ack_upper = Bit(2) 45 | last_ack_lower = Bit(6) 46 | last_ack_extra = Bit(((Const(1) << last_ack_upper) - 1) << 3) 47 | ack_delay_upper = Bit(2) 48 | ack_delay_lower = Bit(6) 49 | ack_delay_extra = Bit(((Const(1) << ack_delay_upper) - 1) << 3) 50 | ack_block_count_upper = Bit(2) 51 | ack_block_count_lower = Bit(6) 52 | ack_block_count_extra = Bit(((Const(1) << ack_block_count_upper) - 1) << 3) 53 | ack_block_upper = Bit(2) 54 | ack_block_lower = Bit(6) 55 | ack_block_extra = Bit(((Const(1) << ack_block_upper) - 1) << 3) 56 | 57 | 58 | class MaxStreamData(layout): 59 | max_data_stream_id_upper = Bit(2) 60 | max_data_stream_id_lower = Bit(6) 61 | max_data_stream_id_extra = Bit(((Const(1) << max_data_stream_id_upper) - 1) << 3) 62 | max_stream_data_upper = Bit(2) 63 | max_stream_data_lower = Bit(6) 64 | max_stream_data_extra = Bit(((Const(1) << max_stream_data_upper) - 1) << 3) 65 | 66 | 67 | class MaxData(layout): 68 | maximum_data = Bit(8) 69 | maximum_date_lower = tail_bit(maximum_data) 70 | 71 | 72 | class MaxStreamID(layout): 73 | max_stream_id = Bit(16) 74 | 75 | 76 | class PathChallenge(layout): 77 | path_challenge = Bit(64) 78 | 79 | 80 | class PathResponse(layout): 81 | path_response = Bit(64) 82 | 83 | 84 | class StopSending(layout): 85 | stop_stream_id = Bit(8) 86 | stop_stream_id_tail = tail_bit(stop_stream_id) 87 | application_error = Bit(16) 88 | 89 | 90 | class RSTStream(layout): 91 | rst_var_1 = Bit(8) 92 | rst_var_2 = tail_bit(rst_var_1) 93 | rst_err_code = Bit(16) 94 | rst_var_3 = Bit(8) 95 | rst_var_4 = tail_bit(rst_var_3) 96 | 97 | 98 | class blank(layout): 99 | pass 100 | 101 | 102 | class QUICFrameTempData(layout): 103 | length = Bit(64) 104 | offset = Bit(64) 105 | payload_len = Bit(64) 106 | real_payload_len = Bit(64) 107 | data = Bit() 108 | 109 | 110 | def quic_frame_protocol_parser(stack): 111 | quic_frame_protocol = Connectionless() 112 | quic_frame_protocol.header = QUICFrame 113 | global_frames = blank 114 | for frame_type, layout in [ 115 | (0x01, RSTStream), 116 | (0x02, ConnectionClose), 117 | (0x06, MaxStreamID), 118 | (0x0C, StopSending), 119 | (0x0F, PathResponse), 120 | (0x0E, PathChallenge), 121 | (0x0D, QUICACK), 122 | (0x04, MaxData), 123 | (0x05, MaxStreamData), 124 | ]: 125 | global_frames += If(QUICFrame.frame_type == frame_type) >> layout 126 | 127 | def head_tail(head_layout, head_var, tail_layout): 128 | return head_layout + (If(head_var & 0xC0 != 0) >> tail_layout) 129 | 130 | frame_type = quic_frame_protocol.header.frame_type 131 | quic_frame_protocol.header += ( 132 | blank 133 | + (If(frame_type & 0xF0 == 0) >> global_frames) 134 | + ( 135 | If(frame_type & 0xF8 == 0x10) 136 | >> head_tail(StreamID, StreamID.stream_id, StreamIDTail) 137 | + ( 138 | If(frame_type & 0x04 != 0) 139 | >> head_tail(FrameOffset, FrameOffset.frame_offset, FrameOffsetTail) 140 | ) 141 | + ( 142 | If(frame_type & 0x02 != 0) 143 | >> head_tail(FrameLength, FrameLength.frame_length, FrameLengthTail) 144 | ) 145 | ) 146 | ) 147 | 148 | quic_frame_protocol.temp = QUICFrameTempData 149 | quic_frame_protocol.prep = ( 150 | ( 151 | If(quic_frame_protocol.header.frame_type & 0xF0 != 0) 152 | >> ( 153 | ( 154 | If(quic_frame_protocol.header.frame_type & 0x04 == 0) 155 | >> Assign(quic_frame_protocol.temp.offset, 0) 156 | >> Else() 157 | >> AssignQUICUInt( 158 | quic_frame_protocol.temp.offset, 159 | quic_frame_protocol.header.frame_offset, 160 | quic_frame_protocol.header.frame_offset_tail, 161 | ) 162 | ) 163 | + ( 164 | If(quic_frame_protocol.header.frame_type & 0x02 != 0) 165 | >> ( 166 | AssignQUICUInt( 167 | quic_frame_protocol.temp.payload_len, 168 | quic_frame_protocol.header.frame_length, 169 | quic_frame_protocol.header.frame_length_tail, 170 | ) 171 | + Assign( 172 | quic_frame_protocol.temp.length, 173 | quic_frame_protocol.temp.payload_len, 174 | ) 175 | ) 176 | >> Else() 177 | >> Assign(quic_frame_protocol.temp.length, 0) 178 | ) 179 | ) 180 | >> Else() 181 | >> ( 182 | If(frame_type == 0) 183 | >> Assign( 184 | quic_frame_protocol.temp.length, quic_frame_protocol.payload_len 185 | ) 186 | >> Else() 187 | >> Assign(quic_frame_protocol.temp.length, 0) 188 | ) 189 | ) 190 | + ( 191 | If(quic_frame_protocol.header_contain(StreamID) == 0) 192 | >> ( 193 | Assign(quic_frame_protocol.temp.offset, 0) 194 | + Assign(quic_frame_protocol.temp.data, NoData()) 195 | + Assign(quic_frame_protocol.temp.payload_len, 0) 196 | ) 197 | >> Else() 198 | >> Assign(quic_frame_protocol.temp.data, quic_frame_protocol.payload) 199 | ) 200 | + Assign( 201 | quic_frame_protocol.temp.real_payload_len, quic_frame_protocol.payload_len 202 | ) 203 | ) 204 | 205 | quic_frame_protocol.selector = [ 206 | stack.ip.header.saddr, 207 | stack.udp.header.src_port, 208 | stack.ip.header.daddr, 209 | stack.udp.header.dst_port, 210 | quic_frame_protocol.header.stream_id, 211 | SliceBeforeOp(quic_frame_protocol.header.stream_id_tail, Const(7)), 212 | ] 213 | 214 | # TODO: states for all data has passed middlebox 215 | 216 | quic_frame_protocol.seq = Sequence( 217 | meta=quic_frame_protocol.temp.offset, 218 | data=SliceBeforeOp( 219 | quic_frame_protocol.temp.data, quic_frame_protocol.temp.payload_len 220 | ), 221 | ) 222 | 223 | dump = PSMState(start=True, accept=True) 224 | frag = PSMState() 225 | quic_frame_protocol.psm = PSM(dump, frag) 226 | quic_frame_protocol.psm.other_frame = (dump >> dump) + Predicate( 227 | quic_frame_protocol.header_contain(StreamID) == 0 228 | ) 229 | quic_frame_protocol.psm.frag_other_frame = (frag >> frag) + Predicate( 230 | quic_frame_protocol.header_contain(StreamID) == 0 231 | ) 232 | quic_frame_protocol.psm.more_frag = (dump >> frag) + Predicate( 233 | quic_frame_protocol.header_contain(StreamID) & (frame_type & 0x01 == 0) 234 | ) 235 | quic_frame_protocol.psm.more_normal_frag = (frag >> frag) + Predicate( 236 | quic_frame_protocol.header_contain(StreamID) & (frame_type & 0x01 == 0) 237 | ) 238 | quic_frame_protocol.psm.receiving_all = (frag >> dump) + Predicate( 239 | quic_frame_protocol.header_contain(StreamID) 240 | & (quic_frame_protocol.v.header.frame_type & 0x01 != 0) 241 | ) 242 | 243 | quic_frame_protocol.event.asm = ( 244 | If( 245 | quic_frame_protocol.psm.receiving_all 246 | | quic_frame_protocol.psm.more_frag 247 | | quic_frame_protocol.psm.more_normal_frag 248 | ) 249 | >> Assemble() 250 | ) 251 | 252 | quic_frame_protocol.event.sdu = If(1) >> ( 253 | AssignSDU(quic_frame_protocol.payload[quic_frame_protocol.temp.length :]) 254 | ) 255 | 256 | quic_frame_protocol.event += ( 257 | quic_frame_protocol.event.asm, 258 | quic_frame_protocol.event.sdu, 259 | ) 260 | return quic_frame_protocol 261 | -------------------------------------------------------------------------------- /stock/protocols/quic_header.py: -------------------------------------------------------------------------------- 1 | # pylint: disable = unused-wildcard-import 2 | from rubik.lang import * 3 | 4 | 5 | def quic_header_protocol_parser(stack): 6 | class PacketType(layout): 7 | type = Bit(8) 8 | 9 | class Part1(layout): 10 | version = Bit(32) 11 | id_len = Bit(8) 12 | 13 | class DstConnID(layout): 14 | dstid = Bit((((Part1.id_len & 0xF0) >> 4) + 3) << 3) 15 | 16 | class SrcConnID(layout): 17 | srcid = Bit(((Part1.id_len & 0x0F) + 3) << 3) 18 | 19 | class Part2(layout): 20 | payload_length1 = Bit(8) 21 | 22 | class PayloadLengthTail(layout): 23 | payload_length2 = Bit( 24 | ((Const(1) << ((Part2.payload_length1 & 0xC0) >> 6)) - 1) << 3 25 | ) 26 | 27 | class Part3(layout): 28 | packet_number = Bit(32) 29 | 30 | class ShortHeader(layout): 31 | dst_conn_id = Bit(stack.udp.perm.dst_conn_id_len << 3) 32 | # packet_number = \ 33 | # byte(1 << (QUICHeaderFormPacketType.header_form_packet_type & 0x03)) 34 | packet_number = Bit(8) 35 | 36 | quic_header_protocol = ConnectionOriented() 37 | quic_header_protocol.header = PacketType 38 | long_header = Part1 39 | long_header += If(long_header.id_len & 0xF0 != 0) >> DstConnID 40 | long_header += If(long_header.id_len & 0x0F != 0) >> SrcConnID 41 | long_header += Part2 42 | long_header += If(long_header.payload_length1 & 0xC0 != 0) >> PayloadLengthTail 43 | long_header += Part3 44 | quic_header_protocol.header += ( 45 | If(quic_header_protocol.header.type & 0x80 != 0) >> long_header 46 | ) 47 | quic_header_protocol.header += ( 48 | If(quic_header_protocol.header.type & 0x80 == 0) >> ShortHeader 49 | ) 50 | quic_header_protocol.prep = If( 51 | quic_header_protocol.header_contain(Part1) 52 | ) >> Assign( 53 | stack.udp.perm.dst_conn_id_len, 54 | ((quic_header_protocol.header.id_len & 0xF0) >> 4) + 3, 55 | ) 56 | 57 | # TODO: change to Connection ID 58 | quic_header_protocol.selector = ( 59 | [stack.ip.header.saddr, stack.udp.header.src_port], 60 | [stack.ip.header.daddr, stack.udp.header.dst_port], 61 | ) 62 | 63 | # TODO: drive this state machine with `quic_frame`'s packets 64 | start = PSMState(start=True, accept=True) 65 | quic_header_protocol.psm = PSM(start) 66 | 67 | quic_header_protocol.psm.loop = (start >> start) + Predicate(1) 68 | return quic_header_protocol 69 | -------------------------------------------------------------------------------- /stock/protocols/quic_udp.py: -------------------------------------------------------------------------------- 1 | from rubik.lang import layout, Bit, Connectionless, Const, PSMState, PSM, Predicate 2 | 3 | class udp_hdr(layout): 4 | src_port = Bit(16) 5 | dst_port = Bit(16) 6 | pkt_length = Bit(16) 7 | checksum = Bit(16) 8 | 9 | class udp_perm(layout): 10 | dst_conn_id_len = Bit(8, init = Const(0)) 11 | 12 | def udp_parser(): 13 | udp = Connectionless() 14 | udp.header = udp_hdr 15 | udp.perm = udp_perm 16 | udp.selector = [udp.header.src_port, udp.header.dst_port] 17 | start = PSMState(start = True) 18 | nothing = PSMState(accept = True) 19 | 20 | udp.psm = PSM(start, nothing) 21 | udp.psm.dump = (start >> start) + Predicate(1) 22 | 23 | 24 | return udp 25 | -------------------------------------------------------------------------------- /stock/protocols/sctp.py: -------------------------------------------------------------------------------- 1 | from rubik.lang import ( 2 | layout, 3 | Bit, 4 | UInt, 5 | ConnectionOriented, 6 | If, 7 | Assign, 8 | AnyUntil, 9 | PSMState, 10 | make_psm_state, 11 | Else, 12 | Sequence, 13 | PSM, 14 | Pred, 15 | Assemble, 16 | NoData, 17 | Const, 18 | ) 19 | 20 | 21 | class sctp_common_hdr(layout): 22 | sport = UInt(16) 23 | dport = UInt(16) 24 | veri_tag = Bit(32) 25 | chk_sum = Bit(32) 26 | 27 | 28 | class sctp_data_hdr(layout): 29 | data_type = Bit(8, const=0) 30 | reserved = Bit(5) 31 | U = Bit(1) 32 | B = Bit(1) 33 | E = Bit(1) 34 | length = UInt(16) 35 | TSN = UInt(32) 36 | stream_id = Bit(16) 37 | stream_seq_num = Bit(16) 38 | payload_id = Bit(32) 39 | 40 | 41 | class sctp_init_hdr(layout): 42 | init_type = Bit(8, const=1) 43 | init_chunk_flags = Bit(8) 44 | init_chunk_length = UInt(16) 45 | initiate_tag = Bit(32) 46 | a_rwnd = Bit(32) 47 | number_outbound_stream = UInt(16) 48 | number_inbound_stream = UInt(16) 49 | initiate_TSN = UInt(32) 50 | 51 | 52 | class sctp_init_ack_hdr(layout): 53 | init_ack_type = Bit(8, const=2) 54 | init_ack_chunk_flags = Bit(8) 55 | init_ack_chunk_length = UInt(16) 56 | init_ack_initiate_tag = Bit(32) 57 | init_ack_a_rwnd = Bit(32) 58 | init_ack_number_outbound_stream = UInt(16) 59 | init_ack_number_inbound_stream = UInt(16) 60 | init_ack_initiate_TSN = UInt(32) 61 | 62 | 63 | # class sctp_init_ack_hdr(layout): 64 | class sctp_sack_hdr(layout): 65 | sack_type = Bit(8, const=3) 66 | sack_chunk_flag = Bit(8) 67 | sack_chunk_length = UInt(16) 68 | cumu_TSN_ack = UInt(32) 69 | a_rwnd = Bit(32) 70 | number_of_gap = Bit(16) 71 | number_of_dup = Bit(16) 72 | 73 | 74 | class sctp_cookie_echo_hdr(layout): 75 | cookie_echo_type = Bit(8, const=10) 76 | cookie_echo_chunk_flag = Bit(8) 77 | cookie_echo_chunk_length = UInt(16) 78 | 79 | 80 | class sctp_cookie_ack_hdr(layout): 81 | cookie_ack_chunk_type = Bit(8, const=11) 82 | cookie_ack_chunk_flag = Bit(8) 83 | cookie_ack_chunk_length = UInt(16) 84 | 85 | 86 | class sctp_shutdown_hdr(layout): 87 | shutdown_chunk_type = Bit(8, const=7) 88 | shutdown_chunk_flag = Bit(8) 89 | shutdown_chunk_length = Bit(8) 90 | shudown_cumu_TSN_ack = Bit(32) 91 | 92 | 93 | class sctp_shutdown_ack_hdr(layout): 94 | shutdown_ack_chunk_type = Bit(8, const=8) 95 | shutdown_ack_chunk_flag = Bit(8) 96 | shutdown_ack_chunk_length = Bit(8) 97 | 98 | 99 | class sctp_shutdown_complete_hdr(layout): 100 | shutdown_complete_chunk_type = Bit(8, const=14) 101 | shutdown_complete_chunk_flag = Bit(8) 102 | shutdown_complete_chunk_length = Bit(8) 103 | 104 | 105 | class sctp_abort_hdr(layout): 106 | abort_chunk_type = Bit(8, const=6) 107 | abort_chunk_flag = Bit(8) 108 | abort_chunk_length = Bit(8) 109 | 110 | 111 | class sctp_perm(layout): 112 | active_seq = Bit(32, init=0) 113 | passive_seq = Bit(32, init=0) 114 | 115 | 116 | class sctp_temp(layout): 117 | seq = Bit(32) 118 | data_len = Bit(16) 119 | 120 | 121 | def sctp_parser(ip): 122 | sctp = ConnectionOriented() 123 | 124 | sctp.header = sctp_common_hdr + AnyUntil( 125 | [ 126 | sctp_data_hdr, 127 | sctp_init_hdr, 128 | sctp_init_ack_hdr, 129 | sctp_sack_hdr, 130 | sctp_cookie_echo_hdr, 131 | sctp_cookie_ack_hdr, 132 | sctp_shutdown_hdr, 133 | sctp_shutdown_ack_hdr, 134 | sctp_shutdown_complete_hdr, 135 | sctp_abort_hdr, 136 | ], 137 | Const(0), 138 | ) 139 | 140 | sctp.selector = ( 141 | [ip.header.saddr, sctp.header.sport], 142 | [ip.header.daddr, sctp.header.dport], 143 | ) 144 | 145 | sctp.temp = sctp_temp 146 | sctp.perm = sctp_perm 147 | 148 | CLOSED = PSMState(start=True) 149 | ( 150 | INIT_SENT, 151 | INIT_ACK_SENT, 152 | COOKIE_ECHO_SENT, 153 | ESTABLISHED, 154 | MORE_FRAG, 155 | SHUTDOWN_SENT, 156 | SHUTDOWN_ACK_SENT, 157 | ) = make_psm_state(7) 158 | TERMINATE = PSMState(accept=True) 159 | 160 | sctp.prep = ( 161 | ( 162 | If(sctp.header_contain(sctp_data_hdr)) 163 | >> ( 164 | If(sctp.to_active) 165 | >> ( 166 | Assign( 167 | sctp.perm.passive_seq, sctp.header.TSN + sctp.header.length - 16 168 | ) 169 | + Assign(sctp.temp.seq, sctp.header.TSN) 170 | + Assign(sctp.temp.data_len, sctp.header.length - 16) 171 | ) 172 | >> Else() 173 | >> ( 174 | Assign( 175 | sctp.perm.active_seq, sctp.header.TSN + sctp.header.length - 16 176 | ) 177 | + Assign(sctp.temp.seq, sctp.header.TSN) 178 | + Assign(sctp.temp.data_len, sctp.header.length - 16) 179 | ) 180 | ) 181 | ) 182 | + ( 183 | If(sctp.header_contain(sctp_init_hdr)) 184 | >> Assign(sctp.perm.active_seq, sctp.header.initiate_TSN) 185 | ) 186 | + ( 187 | If(sctp.header_contain(sctp_init_ack_hdr)) 188 | >> Assign(sctp.perm.passive_seq, sctp.header.init_ack_initiate_TSN) 189 | ) 190 | + ( 191 | If( 192 | sctp.header_contain(sctp_cookie_echo_hdr) 193 | | sctp.header_contain(sctp_cookie_ack_hdr) 194 | | sctp.header_contain(sctp_init_hdr) 195 | | sctp.header_contain(sctp_init_ack_hdr) 196 | | sctp.header_contain(sctp_sack_hdr) 197 | ) 198 | >> ( 199 | Assign(sctp.temp.data_len, 0) 200 | + ( 201 | (If(sctp.to_active) >> Assign(sctp.temp.seq, sctp.perm.passive_seq)) 202 | + ( 203 | If(sctp.to_passive) 204 | >> Assign(sctp.temp.seq, sctp.perm.active_seq) 205 | ) 206 | ) 207 | ) 208 | ) 209 | ) 210 | 211 | sctp.seq = Sequence( 212 | meta=sctp.temp.seq, 213 | zero_based=False, 214 | data=sctp.payload[: sctp.temp.data_len], 215 | data_len=sctp.temp.data_len, 216 | window=None, 217 | ) 218 | 219 | sctp.psm = PSM( 220 | CLOSED, 221 | INIT_SENT, 222 | INIT_ACK_SENT, 223 | COOKIE_ECHO_SENT, 224 | ESTABLISHED, 225 | MORE_FRAG, 226 | SHUTDOWN_SENT, 227 | SHUTDOWN_ACK_SENT, 228 | TERMINATE, 229 | ) 230 | 231 | sctp.psm.hs1 = (CLOSED >> INIT_SENT) + Pred(sctp.header_contain(sctp_init_hdr)) 232 | sctp.psm.hs2 = (INIT_SENT >> INIT_ACK_SENT) + Pred( 233 | sctp.header_contain(sctp_init_ack_hdr) 234 | ) 235 | sctp.psm.hs3 = (INIT_ACK_SENT >> COOKIE_ECHO_SENT) + Pred( 236 | sctp.header_contain(sctp_cookie_echo_hdr) 237 | ) 238 | sctp.psm.hs4 = (COOKIE_ECHO_SENT >> ESTABLISHED) + Pred( 239 | sctp.header_contain(sctp_cookie_ack_hdr) 240 | ) 241 | 242 | sctp.psm.sack = (ESTABLISHED >> ESTABLISHED) + Pred( 243 | sctp.header_contain(sctp_sack_hdr) 244 | ) 245 | sctp.psm.data_start = (ESTABLISHED >> MORE_FRAG) + Pred( 246 | sctp.header_contain(sctp_data_hdr) 247 | & (sctp.header.U == 0) 248 | & (sctp.header.B == 1) 249 | & (sctp.header.E == 0) 250 | ) 251 | 252 | sctp.psm.more_data = (MORE_FRAG >> MORE_FRAG) + Pred( 253 | sctp.header_contain(sctp_data_hdr) 254 | & (sctp.header.U == 0) 255 | & (sctp.header.B == 0) 256 | & (sctp.header.E == 0) 257 | ) 258 | 259 | sctp.psm.data_end = (MORE_FRAG >> ESTABLISHED) + Pred( 260 | sctp.header_contain(sctp_data_hdr) 261 | & (sctp.header.U == 0) 262 | & (sctp.header.B == 0) 263 | & (sctp.header.E == 1) 264 | ) 265 | 266 | sctp.psm.single_frag = (ESTABLISHED >> ESTABLISHED) + Pred( 267 | sctp.header_contain(sctp_data_hdr) 268 | & (sctp.header.U == 0) 269 | & (sctp.header.B == 1) 270 | & (sctp.header.E == 1) 271 | ) 272 | 273 | sctp.psm.unordered = (ESTABLISHED >> ESTABLISHED) + Pred( 274 | sctp.header_contain(sctp_data_hdr) & (sctp.header.U == 1) 275 | ) 276 | 277 | sctp.psm.abort1 = (ESTABLISHED >> TERMINATE) + Pred( 278 | sctp.header_contain(sctp_abort_hdr) 279 | ) 280 | 281 | sctp.psm.abort2 = (CLOSED >> TERMINATE) + Pred(sctp.header_contain(sctp_abort_hdr)) 282 | 283 | sctp.psm.wv1 = (ESTABLISHED >> SHUTDOWN_SENT) + Pred( 284 | sctp.header_contain(sctp_shutdown_hdr) 285 | ) 286 | 287 | sctp.psm.wv2 = (SHUTDOWN_SENT >> SHUTDOWN_ACK_SENT) + Pred( 288 | sctp.header_contain(sctp_shutdown_ack_hdr) 289 | ) 290 | 291 | sctp.psm.wv3 = (SHUTDOWN_ACK_SENT >> TERMINATE) + Pred( 292 | sctp.header_contain(sctp_shutdown_complete_hdr) 293 | ) 294 | 295 | sctp.event.asm = ( 296 | If(sctp.psm.unordered | sctp.psm.single_frag | sctp.psm.data_end) >> Assemble() 297 | ) 298 | return sctp 299 | -------------------------------------------------------------------------------- /stock/protocols/tcp.py: -------------------------------------------------------------------------------- 1 | from rubik.lang import ( 2 | layout, 3 | Bit, 4 | UInt, 5 | ConnectionOriented, 6 | If, 7 | Assign, 8 | AnyUntil, 9 | PSMState, 10 | make_psm_state, 11 | Else, 12 | Sequence, 13 | PSM, 14 | Pred, 15 | Assemble, 16 | NoData, 17 | ) 18 | 19 | 20 | class tcp_hdr(layout): 21 | sport = UInt(16) 22 | dport = UInt(16) 23 | seq_num = UInt(32) 24 | ack_num = UInt(32) 25 | hdr_len = Bit(4) 26 | blank = Bit(4) 27 | cwr = Bit(1) 28 | ece = Bit(1) 29 | urg = Bit(1) 30 | ack = Bit(1) 31 | psh = Bit(1) 32 | rst = Bit(1) 33 | syn = Bit(1) 34 | fin = Bit(1) 35 | window_size = UInt(16) 36 | checksum = Bit(16) 37 | urgent_pointer = Bit(16) 38 | 39 | 40 | class tcp_eol(layout): 41 | eol_type = Bit(8, const=0) 42 | 43 | 44 | class tcp_nop(layout): 45 | nop_type = Bit(8, const=1) 46 | 47 | 48 | class tcp_mss(layout): 49 | mss_type = Bit(8, const=2) 50 | mss_len = Bit(8) 51 | mss_value = Bit(16) 52 | 53 | 54 | class tcp_ws(layout): 55 | ws_type = Bit(8, const=3) 56 | ws_len = Bit(8) 57 | ws_value = Bit(8) 58 | 59 | 60 | class tcp_SACK_permitted(layout): 61 | SCAK_permitted_type = Bit(8, const=4) 62 | SCAK_permitted_len = Bit(8) 63 | 64 | 65 | class tcp_SACK(layout): 66 | SACK_type = Bit(8, const=5) 67 | SACK_len = Bit(8) 68 | SACK_value = Bit((SACK_len - 2) << 3) 69 | 70 | 71 | class tcp_TS(layout): 72 | TS_type = Bit(8, const=8) 73 | TS_len = Bit(8) 74 | TS_value = Bit(32) 75 | TS_echo_reply = Bit(32) 76 | 77 | 78 | class tcp_cc_new(layout): 79 | cc_new_type = Bit(8, const=12) 80 | cc_new_len = Bit(8) 81 | cc_new_value = Bit(32) 82 | 83 | 84 | class tcp_blank(layout): 85 | blank_type = Bit(8) 86 | blank_len = Bit(8) 87 | blank_value = Bit((blank_len - 2) << 3) 88 | 89 | 90 | class tcp_data(layout): 91 | active_lwnd = Bit(32, init=0) 92 | passive_lwnd = Bit(32, init=0) 93 | active_wscale = Bit(32, init=0) 94 | passive_wscale = Bit(32, init=0) 95 | active_wsize = Bit(32, init=(1 << 32) - 1) 96 | passive_wsize = Bit(32, init=(1 << 32) - 1) 97 | fin_seq1 = Bit(32, init=0) 98 | fin_seq2 = Bit(32, init=0) 99 | 100 | 101 | class tcp_temp(layout): 102 | wnd = Bit(32) 103 | wnd_size = Bit(32) 104 | data_len = Bit(32) 105 | 106 | 107 | def tcp_parser(ip): 108 | tcp = ConnectionOriented() 109 | 110 | tcp.header = tcp_hdr 111 | tcp.header += If(tcp.cursor < tcp.header.hdr_len << 2) >> AnyUntil( 112 | [ 113 | tcp_eol, 114 | tcp_nop, 115 | tcp_mss, 116 | tcp_ws, 117 | tcp_SACK_permitted, 118 | tcp_SACK, 119 | tcp_TS, 120 | tcp_cc_new, 121 | tcp_blank, 122 | ], 123 | (tcp.cursor < tcp.header.hdr_len << 2) & (tcp.payload_len != 0), 124 | ) 125 | 126 | tcp.selector = ( 127 | [ip.header.saddr, tcp.header.sport], 128 | [ip.header.daddr, tcp.header.dport], 129 | ) 130 | 131 | tcp.perm = tcp_data 132 | tcp.temp = tcp_temp 133 | 134 | CLOSED = PSMState(start=True) 135 | SYN_SENT, SYN_RCV, EST, FIN_WAIT_1, CLOSE_WAIT, LAST_ACK = make_psm_state(6) 136 | TERMINATE = PSMState(accept=True) 137 | 138 | tcp.prep = Assign(tcp.temp.data_len, tcp.payload_len) 139 | tcp.prep = ( 140 | If(tcp.header.syn == 1) >> Assign(tcp.temp.data_len, 1) >> Else() >> tcp.prep 141 | ) 142 | tcp.prep = ( 143 | If(tcp.header.fin == 1) 144 | >> Assign(tcp.temp.data_len, tcp.payload_len + 1) 145 | + ( 146 | If(tcp.current_state == EST) 147 | >> Assign(tcp.perm.fin_seq1, tcp.header.seq_num + tcp.payload_len) 148 | >> Else() 149 | >> Assign(tcp.perm.fin_seq2, tcp.header.seq_num) 150 | ) 151 | >> Else() 152 | >> tcp.prep 153 | ) 154 | 155 | def update_wnd(oppo_lwnd, oppo_wscale, oppo_wsize, cur_lwnd, cur_wscale, cur_wsize): 156 | x = If(tcp.header_contain(tcp_ws)) >> Assign(oppo_wscale, tcp.header.ws_value) 157 | x += Assign(oppo_wsize, tcp.header.window_size) 158 | x += Assign(oppo_lwnd, tcp.header.ack_num) 159 | x += Assign(tcp.temp.wnd, cur_lwnd) 160 | x += Assign(tcp.temp.wnd_size, cur_wsize << cur_wscale) 161 | return x 162 | 163 | tcp.prep += If(tcp.to_active == 1) >> update_wnd( 164 | tcp.perm.passive_lwnd, 165 | tcp.perm.passive_wscale, 166 | tcp.perm.passive_wsize, 167 | tcp.perm.active_lwnd, 168 | tcp.perm.active_wscale, 169 | tcp.perm.active_wsize, 170 | ) 171 | tcp.prep += If(tcp.to_passive == 1) >> update_wnd( 172 | tcp.perm.active_lwnd, 173 | tcp.perm.active_wscale, 174 | tcp.perm.active_wsize, 175 | tcp.perm.passive_lwnd, 176 | tcp.perm.passive_wscale, 177 | tcp.perm.passive_wsize, 178 | ) 179 | 180 | tcp.seq = Sequence( 181 | meta=tcp.header.seq_num, 182 | zero_based=False, 183 | data=tcp.payload[: tcp.temp.data_len], 184 | data_len=tcp.temp.data_len, 185 | window=(tcp.temp.wnd, tcp.temp.wnd + tcp.temp.wnd_size), 186 | ) 187 | 188 | tcp.psm = PSM( 189 | CLOSED, SYN_SENT, SYN_RCV, EST, FIN_WAIT_1, CLOSE_WAIT, LAST_ACK, TERMINATE 190 | ) 191 | 192 | tcp.psm.fake = (CLOSED >> TERMINATE) + Pred(tcp.header.syn == 0) 193 | tcp.psm.hs1 = (CLOSED >> SYN_SENT) + Pred( 194 | (tcp.header.syn == 1) & (tcp.header.ack == 0) 195 | ) 196 | tcp.psm.hs2 = (SYN_SENT >> SYN_RCV) + Pred( 197 | (tcp.to_active == 1) & (tcp.header.syn == 1) & (tcp.header.ack == 1) 198 | ) 199 | tcp.psm.hs3 = (SYN_RCV >> EST) + Pred(tcp.v.header.ack == 1) 200 | 201 | tcp.psm.buffering = (EST >> EST) + Pred( 202 | (tcp.header.fin == 0) & (tcp.header.rst == 0) 203 | ) 204 | 205 | tcp.psm.wv1 = (EST >> FIN_WAIT_1) + Pred(tcp.v.header.fin == 1) 206 | tcp.psm.wv2 = (FIN_WAIT_1 >> CLOSE_WAIT) + Pred( 207 | (tcp.v.header.ack == 1) 208 | & (tcp.v.header.fin == 0) 209 | & (tcp.perm.fin_seq1 + 1 == tcp.v.header.ack_num) 210 | ) 211 | tcp.psm.wv2_fast = (FIN_WAIT_1 >> LAST_ACK) + Pred( 212 | (tcp.v.header.ack == 1) 213 | & (tcp.v.header.fin == 1) 214 | & (tcp.perm.fin_seq1 + 1 == tcp.v.header.ack_num) 215 | ) 216 | tcp.psm.wv3 = (CLOSE_WAIT >> LAST_ACK) + Pred(tcp.v.header.fin == 1) 217 | tcp.psm.wv4 = (LAST_ACK >> TERMINATE) + Pred( 218 | (tcp.v.header.ack == 1) & (tcp.perm.fin_seq2 + 1 == tcp.v.header.ack_num) 219 | ) 220 | 221 | for i, state in enumerate(tcp.psm.states()): 222 | setattr(tcp.psm, f"rst{i}", (state >> TERMINATE) + Pred(tcp.header.rst == 1)) 223 | 224 | tcp.event.asm = If(tcp.psm.buffering) >> Assemble() 225 | return tcp 226 | -------------------------------------------------------------------------------- /stock/protocols/udp.py: -------------------------------------------------------------------------------- 1 | from rubik.lang import layout, Bit, Connectionless 2 | 3 | 4 | class udp_hdr(layout): 5 | src_port = Bit(16) 6 | dst_port = Bit(16) 7 | pkt_length = Bit(16) 8 | checksum = Bit(16) 9 | 10 | 11 | def udp_parser(): 12 | udp = Connectionless() 13 | udp.header = udp_hdr 14 | return udp 15 | -------------------------------------------------------------------------------- /stock/quic.py: -------------------------------------------------------------------------------- 1 | # pylint: disable = unused-wildcard-import 2 | from rubik.lang import * 3 | 4 | from stock.protocols.loopback import loopback_parser 5 | from stock.protocols.quic_udp import udp_parser 6 | from stock.protocols.ip import ip_parser 7 | from stock.protocols.quic_header import quic_header_protocol_parser 8 | from stock.protocols.quic_frame import quic_frame_protocol_parser 9 | 10 | # includes loopback, IP and UDP 11 | 12 | stack = Stack() 13 | stack.loopback = loopback_parser() 14 | stack.ip = ip_parser() 15 | stack.udp = udp_parser() 16 | stack.quic_header_protocol = quic_header_protocol_parser(stack) 17 | stack.quic_frame_protocol = quic_frame_protocol_parser(stack) 18 | 19 | 20 | stack += (stack.loopback >> stack.ip) + Predicate(1) 21 | stack += (stack.ip >> stack.udp) + Predicate( 22 | (stack.ip.psm.dump | stack.ip.psm.last) & (stack.ip.header.protocol == 17) 23 | ) 24 | stack += (stack.udp >> stack.quic_header_protocol) + Predicate(1) 25 | 26 | stack += (stack.quic_header_protocol >> stack.quic_frame_protocol) + Predicate( 27 | stack.quic_header_protocol.psm.loop 28 | ) 29 | stack += (stack.quic_frame_protocol >> stack.quic_frame_protocol) + Predicate( 30 | stack.quic_frame_protocol.temp.real_payload_len > stack.quic_frame_protocol.temp.length 31 | ) 32 | -------------------------------------------------------------------------------- /stock/sctp.py: -------------------------------------------------------------------------------- 1 | from rubik.lang import Stack, Predicate, layout 2 | from stock.protocols import ip_parser, eth_parser, sctp_parser 3 | 4 | 5 | stack = Stack() 6 | stack.eth = eth_parser() 7 | stack.ip = ip_parser() 8 | stack.sctp = sctp_parser(stack.ip) 9 | 10 | stack += (stack.eth >> stack.ip) + Predicate(1) 11 | stack += (stack.ip >> stack.sctp) + Predicate( 12 | (stack.ip.psm.dump | stack.ip.psm.last) & (stack.ip.header.protocol == 132)) 13 | -------------------------------------------------------------------------------- /stock/tcp_ip.py: -------------------------------------------------------------------------------- 1 | from rubik.lang import Stack, Predicate, layout, Bit, If, Assign, Call 2 | from stock.protocols import ip_parser, tcp_parser, eth_parser, udp_parser 3 | 4 | 5 | stack = Stack() 6 | stack.eth = eth_parser() 7 | stack.ip = ip_parser() 8 | stack.tcp = tcp_parser(stack.ip) 9 | stack.udp = udp_parser() 10 | 11 | stack += (stack.eth >> stack.ip) + Predicate(1) 12 | stack += (stack.ip >> stack.tcp) + Predicate( 13 | (stack.ip.psm.dump | stack.ip.psm.last) & (stack.ip.header.protocol == 6) 14 | ) 15 | stack += (stack.ip >> stack.udp) + Predicate( 16 | (stack.ip.psm.dump | stack.ip.psm.last) & (stack.ip.header.protocol == 17) 17 | ) 18 | 19 | # stack.tcp.layer.context.buffer_data = False --------------------------------------------------------------------------------