├── test ├── messagepack │ ├── data-1-1.mp │ ├── data-1-2.mp │ ├── data-1-3.mp │ ├── test-file-ext.mp │ └── test-file-noext.mp ├── unit │ ├── ci-unix.sh │ ├── ci-windows.bat │ └── src │ │ ├── test-common.h │ │ ├── test-builder.h │ │ ├── test-expect.h │ │ ├── test-file.h │ │ ├── test-buffer.h │ │ ├── test-system.h │ │ ├── test.c │ │ ├── test-write.h │ │ ├── mpack-config.h │ │ ├── test-node.h │ │ ├── test-reader.h │ │ ├── test-reader.c │ │ ├── test-system.c │ │ ├── test.h │ │ ├── test-builder.c │ │ └── test-buffer.c ├── pseudojson │ ├── test-file-noext.debug │ └── test-file-ext.debug ├── fuzz │ ├── Makefile │ ├── fuzz-config.h │ └── fuzz.c ├── avr │ └── Makefile └── README.md ├── tools ├── clean.sh ├── scan-build.sh ├── clean.bat ├── unit.sh ├── update-gh-pages.sh ├── getversion.sh ├── gendocs.sh ├── vs2017_x86.bat ├── vs2017_x64.bat ├── afl.sh ├── coverage.sh ├── valgrind-suppressions ├── package.sh ├── unit.bat └── amalgamate.sh ├── .editorconfig ├── .gitattributes ├── .gitignore ├── docs ├── doxygen-mpack-css.css ├── doxygen-layout.xml ├── doxyfile ├── node.md ├── features.md ├── expect.md ├── protocol.md └── reader.md ├── examples ├── Makefile ├── sax-example.h └── sax-example.c ├── LICENSE ├── AUTHORS.md ├── src └── mpack │ ├── mpack.h │ └── mpack-platform.c ├── .github └── workflows │ └── unit.yml ├── README.md └── CHANGELOG.md /test/messagepack/data-1-1.mp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ludocode/mpack/HEAD/test/messagepack/data-1-1.mp -------------------------------------------------------------------------------- /test/messagepack/data-1-2.mp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ludocode/mpack/HEAD/test/messagepack/data-1-2.mp -------------------------------------------------------------------------------- /test/messagepack/data-1-3.mp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ludocode/mpack/HEAD/test/messagepack/data-1-3.mp -------------------------------------------------------------------------------- /test/messagepack/test-file-ext.mp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ludocode/mpack/HEAD/test/messagepack/test-file-ext.mp -------------------------------------------------------------------------------- /test/messagepack/test-file-noext.mp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ludocode/mpack/HEAD/test/messagepack/test-file-noext.mp -------------------------------------------------------------------------------- /tools/clean.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | rm -rf .build 3 | rm -f vgcore.* mpack-*.tar.gz *.gcov *.gcno src/mpack/*.o README.html 4 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*.{c,cpp}] 4 | end_of_line = lf 5 | insert_final_newline = true 6 | indent_style = space 7 | indent_size = 4 8 | -------------------------------------------------------------------------------- /tools/scan-build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | scan-build -o analysis --status-bugs bash -c 'test/unit/configure.py && ninja -f .build/unit/build.ninja' 3 | -------------------------------------------------------------------------------- /tools/clean.bat: -------------------------------------------------------------------------------- 1 | if exist test\.build rmdir /s /q test\.build 2 | if exist .ninja_deps del /AH .ninja_deps 3 | if exist .ninja_log del /AH .ninja_log 4 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # git must not convert line endings in our reference test file 2 | # https://github.com/ludocode/mpack/issues/64 3 | *.debug text eol=lf 4 | -------------------------------------------------------------------------------- /tools/unit.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Builds and runs the unit test suite. 4 | # Set CC before calling this to use a different compiler. 5 | # Pass a configuration to run or pass "all" to run all configurations. 6 | 7 | set -e 8 | cd "$(dirname $0)/.." 9 | test/unit/configure.py 10 | ninja -f .build/unit/build.ninja $@ 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # build files 3 | /.build/ 4 | test/.build/ 5 | coverage/ 6 | *.swp 7 | *.gcov 8 | mpack-test-file 9 | mpack-test-blank-file 10 | /mpack-test-dir/ 11 | /analysis/ 12 | /docs/html/ 13 | /vgcore.* 14 | /.ninja_deps 15 | /.ninja_log 16 | 17 | # other junk 18 | .directory 19 | /tags 20 | /examples/sax-example 21 | -------------------------------------------------------------------------------- /docs/doxygen-mpack-css.css: -------------------------------------------------------------------------------- 1 | table.doxtable th { 2 | background-color: #e0e7ff; 3 | color: #000; 4 | } 5 | .textblock code, 6 | .contents p code, .contents p em, 7 | .contents dd code, .contents dd em { 8 | padding: 1pt 2pt 1pt 2pt; 9 | background-color: #f5f5f5; 10 | } 11 | .textblock li { 12 | margin: 3pt 0 3pt 0; 13 | } 14 | -------------------------------------------------------------------------------- /examples/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: run-sax-example 2 | run-sax-example: sax-example 3 | ./sax-example ../test/messagepack/data-1-2.mp 4 | 5 | sax-example: Makefile sax-example.c sax-example.h $(shell echo ../src/*) 6 | cc -g -Wall -Werror -DMPACK_EXTENSIONS=1 sax-example.c ../src/mpack/*.c -I../src -o sax-example 7 | 8 | clean: 9 | rm -f sax-example 10 | -------------------------------------------------------------------------------- /tools/update-gh-pages.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # updates documentation in gh-pages 3 | 4 | set -e 5 | 6 | "`dirname $0`"/clean.sh 7 | . "`dirname $0`"/gendocs.sh 8 | cp -ar .build/docs/html docs-html 9 | "`dirname $0`"/clean.sh 10 | 11 | git checkout gh-pages || exit $? 12 | 13 | rm -r *.{html,png,js,css} search 14 | cp -r docs-html/* . 15 | rm -r docs-html 16 | 17 | git add *.{html,png,js,css} search 18 | git commit -am "Updated documentation to version $VERSION" 19 | git show --stat 20 | git status 21 | -------------------------------------------------------------------------------- /test/unit/ci-unix.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Builds and runs the unit test suite under UNIX. 4 | # 5 | # Pass environment variable CC to specify the compiler. 6 | # 7 | # This script is run by the continuous integration server to test MPack on UNIX 8 | # systems. 9 | 10 | set -e 11 | 12 | # Amalgamate if necessary 13 | if [[ "$AMALGAMATED" == "1" ]]; then 14 | tools/amalgamate.sh 15 | cd .build/amalgamation 16 | fi 17 | pwd 18 | 19 | # Run the "more" variant of unit tests 20 | tools/unit.sh more 21 | -------------------------------------------------------------------------------- /tools/getversion.sh: -------------------------------------------------------------------------------- 1 | # This gets the MPack version string out of the source code. It 2 | # is meant to be sourced by other scripts. 3 | 4 | MAJOR=`grep '^#define MPACK_VERSION_MAJOR' src/mpack/mpack-common.h|sed 's/.* \([0-9][0-9]*\) .*/\1/'` 5 | MINOR=`grep '^#define MPACK_VERSION_MINOR' src/mpack/mpack-common.h|sed 's/.* \([0-9][0-9]*\) .*/\1/'` 6 | PATCH=`grep '^#define MPACK_VERSION_PATCH' src/mpack/mpack-common.h|sed 's/.* \([0-9][0-9]*\) .*/\1/'` 7 | 8 | VERSION="$MAJOR.$MINOR" 9 | if [[ "$PATCH" -gt 0 ]]; then 10 | VERSION="$VERSION.$PATCH" 11 | fi 12 | -------------------------------------------------------------------------------- /tools/gendocs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cd "$(dirname "$0")"/.. 3 | source tools/getversion.sh 4 | mkdir -p .build/docs 5 | 6 | # Write temporary README.md without "Build Status" section 7 | cat README.md | \ 8 | sed '/^## Build Status/,/^##/{//!d}' | \ 9 | sed '/^## Build Status/d' \ 10 | > .build/docs/README.temp.md 11 | 12 | # Generate docs with correct version number 13 | ( 14 | cat docs/doxyfile 15 | echo "PROJECT_NUMBER = $VERSION" 16 | echo 17 | ) | doxygen - || exit 1 18 | 19 | echo 20 | echo "Docs generated: file://$(pwd)/.build/docs/html/index.html" 21 | -------------------------------------------------------------------------------- /tools/vs2017_x86.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | REM Sets paths for Visual Studio 2017 build tools for x86. Visual Studio 2019 4 | REM creates command prompt shortcuts for 2015 and 2019 but not 2017. For 2017, 5 | REM open a plain command prompt and run this. 6 | 7 | setlocal enabledelayedexpansion 8 | 9 | for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -find **\vcvarsall.bat`) do (SET "vcvarsall=%%i") 10 | where cl 1>NUL 2>NUL 11 | if %errorlevel% neq 0 call "%vcvarsall%" x86 -vcvars_ver=14.16 12 | if %errorlevel% neq 0 exit /b %errorlevel% 13 | -------------------------------------------------------------------------------- /tools/vs2017_x64.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | REM Sets paths for Visual Studio 2017 build tools for amd64. Visual Studio 2019 4 | REM creates command prompt shortcuts for 2015 and 2019 but not 2017. For 2017, 5 | REM open a plain command prompt and run this. 6 | 7 | setlocal enabledelayedexpansion 8 | 9 | for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -find **\vcvarsall.bat`) do (SET "vcvarsall=%%i") 10 | where cl 1>NUL 2>NUL 11 | if %errorlevel% neq 0 call "%vcvarsall%" amd64 -vcvars_ver=14.16 12 | if %errorlevel% neq 0 exit /b %errorlevel% 13 | -------------------------------------------------------------------------------- /tools/afl.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Runs mpack-fuzz under american fuzzy lop. 4 | 5 | cd "$(dirname "$0")"/.. 6 | 7 | export AFL_HARDEN=1 8 | make -f test/fuzz/Makefile || exit 1 9 | 10 | echo 11 | echo "This will run the first fuzzer as fuzzer01. To run on additional" 12 | echo "cores, run:" 13 | echo 14 | echo " afl-fuzz -i test/messagepack -o .build/fuzz/sync -S fuzzer## -- .build/fuzz/mpack-fuzz" 15 | echo 16 | echo "To watch the overall progress, run:" 17 | echo 18 | echo " watch afl-whatsup .build/fuzz/sync" 19 | echo 20 | echo "Press enter to start..." 21 | read 22 | 23 | afl-fuzz -i test/messagepack -o .build/fuzz/sync -M fuzzer01 -- .build/fuzz/mpack-fuzz 24 | -------------------------------------------------------------------------------- /tools/coverage.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | # run unit tests with coverage 5 | tools/unit.sh run-coverage 6 | 7 | # run gcov for traditional text-based coverage output 8 | rm -rf coverage 9 | mkdir -p coverage 10 | gcov --object-directory .build/unit/coverage/objs/src/mpack `find src -name '*.c'` || exit $? 11 | mv *.gcov coverage 12 | 13 | # run lcov 14 | lcov --capture --directory .build/unit/coverage/objs/src --output-file coverage/lcov.info 15 | 16 | # generate HTML coverage 17 | genhtml coverage/lcov.info --output-directory coverage/html 18 | 19 | echo 20 | echo "Done. Results written in coverage/" 21 | echo "View HTML results in: file://$(pwd)/coverage/html/index.html" 22 | -------------------------------------------------------------------------------- /test/pseudojson/test-file-noext.debug: -------------------------------------------------------------------------------- 1 | [ 2 | null, 3 | false, 4 | true, 5 | [ 6 | 0, 7 | 1, 8 | -1, 9 | -32768, 10 | 65535, 11 | -2147483648, 12 | 4294967295, 13 | 0.000000 14 | ], 15 | { 16 | "a": 1, 17 | "b": 2, 18 | "c": 3, 19 | "d": 4 20 | }, 21 | "\"\n\\", 22 | "The quick brown fox jumps over the lazy dog.", 23 | 24 | ] 25 | -------------------------------------------------------------------------------- /test/unit/ci-windows.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | REM This script is run by the continuous integration server to build and run 4 | REM the MPack unit test suite with MSVC on Windows. 5 | 6 | setlocal enabledelayedexpansion 7 | 8 | REM Find build tools 9 | for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -find **\vcvarsall.bat`) do (SET "vcvarsall=%%i") 10 | 11 | IF "%AMALGAMATED%"=="1" ( 12 | cd .build\amalgamation 13 | ) 14 | 15 | IF "%COMPILER%"=="cl-2019-x64" ( 16 | call "%vcvarsall%" amd64 17 | ) 18 | IF "%COMPILER%"=="cl-2015-x86" ( 19 | call "%vcvarsall%" x86 -vcvars_ver=14.0 20 | ) 21 | 22 | REM Run the "more" variant of unit tests 23 | call tools\unit.bat more -------------------------------------------------------------------------------- /test/pseudojson/test-file-ext.debug: -------------------------------------------------------------------------------- 1 | [ 2 | null, 3 | false, 4 | true, 5 | [ 6 | 0, 7 | 1, 8 | -1, 9 | -32768, 10 | 65535, 11 | -2147483648, 12 | 4294967295, 13 | 0.000000 14 | ], 15 | { 16 | "a": 1, 17 | "b": 2, 18 | "c": 3, 19 | "d": 4 20 | }, 21 | "\"\n\\", 22 | "The quick brown fox jumps over the lazy dog.", 23 | , 24 | 25 | ] 26 | -------------------------------------------------------------------------------- /tools/valgrind-suppressions: -------------------------------------------------------------------------------- 1 | 2 | # Note: match-leak-kinds is currently commented out because older 3 | # versions of valgrind don't support it 4 | 5 | # C++ GCC 5.1 reachable objects 6 | # https://bugs.kde.org/show_bug.cgi?id=345307 7 | # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65434 8 | { 9 | GCC-STL-pool-reachable 10 | Memcheck:Leak 11 | #match-leak-kinds: reachable 12 | fun:malloc 13 | fun:pool 14 | fun:__static_initialization_and_destruction_0 15 | fun:_GLOBAL__sub_I_eh_alloc.cc 16 | fun:call_init.part.0 17 | fun:_dl_init 18 | } 19 | 20 | # musl libc supressions 21 | { 22 | musl-free 23 | Memcheck:Free 24 | fun:free 25 | obj:/lib/ld-musl-x86_64.so.1 26 | } 27 | { 28 | musl-leak 29 | Memcheck:Leak 30 | match-leak-kinds: reachable 31 | fun:calloc 32 | fun:load_library 33 | fun:load_preload 34 | fun:__dls3 35 | fun:__dls2 36 | obj:/lib/ld-musl-x86_64.so.1 37 | } 38 | -------------------------------------------------------------------------------- /tools/package.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Packages MPack up for amalgamation release. You can run tools/amalgamate.sh 3 | # instead of this script if you just want to generate mpack.h and mpack.c. 4 | 5 | set -v 6 | set -e 7 | 8 | [[ -z $(git status --porcelain) ]] || { git status --porcelain; echo "Tree is not clean!" ; exit 1; } 9 | "`dirname $0`"/clean.sh 10 | 11 | # generate package contents 12 | . "`dirname $0`"/amalgamate.sh 13 | . "`dirname $0`"/gendocs.sh 14 | cp -ar .build/docs/html .build/amalgamation/docs 15 | sed -i '/#define MPACK_AMALGAMATED 1/a\ 16 | #define MPACK_RELEASE_VERSION 1' .build/amalgamation/src/mpack/mpack.h 17 | 18 | # create package 19 | NAME=mpack-amalgamation-$VERSION 20 | tar -C .build/amalgamation --transform "s@^@$NAME/@" -czf .build/$NAME.UNTESTED.tar.gz `ls .build/amalgamation` || exit $? 21 | 22 | # build and run all unit tests 23 | pushd .build/amalgamation 24 | tools/unit.sh all || exit $? 25 | popd 26 | 27 | # done! 28 | mv .build/$NAME.UNTESTED.tar.gz $NAME.tar.gz 29 | echo Created $NAME.tar.gz 30 | -------------------------------------------------------------------------------- /tools/unit.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | REM Builds and runs the unit test suite under the Visual Studio C compiler on 4 | REM Windows. 5 | REM 6 | REM Pass a configuration to run or pass "all" to run all configurations. 7 | REM 8 | REM You can run this in a normal command prompt or in a Visual Studio Build 9 | REM Tools command prompt. It will find the build tools automatically if needed 10 | REM but it will run a lot faster if they are already available. 11 | 12 | setlocal enabledelayedexpansion 13 | 14 | REM Find build tools 15 | for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -find **\vcvarsall.bat`) do (SET "vcvarsall=%%i") 16 | 17 | REM Enable build tools if needed 18 | where cl 1>NUL 2>NUL 19 | if %errorlevel% neq 0 call "%vcvarsall%" amd64 20 | if %errorlevel% neq 0 exit /b %errorlevel% 21 | 22 | REM Configure unit tests 23 | python test\unit\configure.py 24 | if %errorlevel% neq 0 exit /b %errorlevel% 25 | 26 | REM Run unit tests 27 | ninja -f .build\unit\build.ninja %* 28 | if %errorlevel% neq 0 exit /b %errorlevel% 29 | -------------------------------------------------------------------------------- /test/fuzz/Makefile: -------------------------------------------------------------------------------- 1 | # This Makefile currently just builds fuzz.c into a fuzzer for use with 2 | # american fuzzy lop. Don't use this directly; use tools/afl.sh to fuzz MPack. 3 | 4 | ifeq (Makefile, $(firstword $(MAKEFILE_LIST))) 5 | $(error The current directory should be the root of the repository. Try "cd ../.." and then "make -f test/fuzz/Makefile") 6 | endif 7 | 8 | CC=afl-gcc 9 | 10 | CPPFLAGS := $(CPPFLAGS) \ 11 | -include test/fuzz/fuzz-config.h \ 12 | -Isrc \ 13 | -O0 -DDEBUG -g \ 14 | -MMD -MP \ 15 | 16 | BUILD := .build/fuzz 17 | PROG := mpack-fuzz 18 | 19 | SRCS := \ 20 | $(shell find src/ -type f -name '*.c') \ 21 | test/fuzz/fuzz.c 22 | 23 | OBJS := $(patsubst %, $(BUILD)/%.o, $(SRCS)) 24 | 25 | GLOBAL_DEPENDENCIES := test/fuzz/Makefile 26 | 27 | .PHONY: all 28 | all: $(PROG) 29 | 30 | -include $(patsubst %, $(BUILD)/%.d, $(SRCS)) 31 | 32 | .PHONY: $(PROG) 33 | $(PROG): $(BUILD)/$(PROG) 34 | 35 | $(OBJS): $(BUILD)/%.o: % $(GLOBAL_DEPENDENCIES) 36 | @mkdir -p $(dir $@) 37 | $(CC) -c $(CPPFLAGS) $(CFLAGS) -o $@ $< 38 | 39 | $(BUILD)/$(PROG): $(OBJS) 40 | @mkdir -p $(dir $@) 41 | $(CC) $(CPPFLAGS) $(CFLAGS) -o $@ $^ $(LDFLAGS) 42 | -------------------------------------------------------------------------------- /examples/sax-example.h: -------------------------------------------------------------------------------- 1 | #include "mpack/mpack.h" 2 | 3 | typedef struct sax_callbacks_t { 4 | void (*nil_element)(void* context, int depth); 5 | void (*bool_element)(void* context, int depth, int64_t value); 6 | void (*int_element)(void* context, int depth, int64_t value); 7 | void (*uint_element)(void* context, int depth, uint64_t value); 8 | void (*string_element)(void* context, int depth, 9 | const char* data, uint32_t length); 10 | void (*bin_element)(void* context, int depth, 11 | const char* data, uint32_t length); 12 | 13 | void (*start_map)(void* context, int depth, uint32_t pair_count); 14 | void (*start_array)(void* context, int depth, uint32_t element_count); 15 | void (*finish_map)(void* context, int depth); 16 | void (*finish_array)(void* context, int depth); 17 | } sax_callbacks_t; 18 | 19 | /** 20 | * Parse a blob of MessagePack data, calling the appropriate callback for each 21 | * element encountered. 22 | * 23 | * @return true if successful, false if any error occurs. 24 | */ 25 | bool parse_messagepack(const char* data, size_t length, 26 | const sax_callbacks_t* callbacks, void* context); 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /AUTHORS.md: -------------------------------------------------------------------------------- 1 | | Author | GitHub Profile | 2 | | :------------------------------ | :----------------------------------------- | 3 | | Nicholas Fraser | https://github.com/ludocode | 4 | | Jerry Jacobs | https://github.com/xor-gate | 5 | | Rik van der Heijden | https://github.com/rikvdh | 6 | | Chris Heijdens | https://github.com/chris-heijdens | 7 | | Jean-Louis Fuchs | https://github.com/ganwell | 8 | | Christopher Field | https://github.com/volks73 | 9 | | 喜欢兰花山丘 | https://github.com/wangzhione | 10 | | Vasily Postnicov | https://github.com/shamazmazum | 11 | | Tim Gates | https://github.com/timgates42 | 12 | | Dirkson | https://github.com/dirkson | 13 | | Ethan Li | https://github.com/ethanjli | 14 | | Danny Povolotski | https://github.com/israelidanny | 15 | | Weston Schmidt | https://github.com/schmidtw | 16 | | Nikita Danilov | https://github.com/nikitadanilov | 17 | -------------------------------------------------------------------------------- /test/unit/src/test-common.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | * this software and associated documentation files (the "Software"), to deal in 6 | * the Software without restriction, including without limitation the rights to 7 | * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 8 | * the Software, and to permit persons to whom the Software is furnished to do so, 9 | * subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 16 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 17 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 18 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 19 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 20 | */ 21 | 22 | #ifndef MPACK_TEST_COMMON_H 23 | #define MPACK_TEST_COMMON_H 1 24 | 25 | #include "test.h" 26 | 27 | #ifdef __cplusplus 28 | extern "C" { 29 | #endif 30 | 31 | void test_common(void); 32 | 33 | #ifdef __cplusplus 34 | } 35 | #endif 36 | 37 | #endif 38 | 39 | 40 | -------------------------------------------------------------------------------- /test/unit/src/test-builder.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019-2021 Nicholas Fraser and the MPack authors 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | * this software and associated documentation files (the "Software"), to deal in 6 | * the Software without restriction, including without limitation the rights to 7 | * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 8 | * the Software, and to permit persons to whom the Software is furnished to do so, 9 | * subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 16 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 17 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 18 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 19 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 20 | */ 21 | 22 | #ifndef MPACK_TEST_BUILDER_H 23 | #define MPACK_TEST_BUILDER_H 1 24 | 25 | #include "test.h" 26 | 27 | #ifdef __cplusplus 28 | extern "C" { 29 | #endif 30 | 31 | #if MPACK_BUILDER 32 | void test_builder(void); 33 | #endif 34 | 35 | #ifdef __cplusplus 36 | } 37 | #endif 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /test/unit/src/test-expect.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | * this software and associated documentation files (the "Software"), to deal in 6 | * the Software without restriction, including without limitation the rights to 7 | * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 8 | * the Software, and to permit persons to whom the Software is furnished to do so, 9 | * subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 16 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 17 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 18 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 19 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 20 | */ 21 | 22 | #ifndef MPACK_TEST_EXPECT_H 23 | #define MPACK_TEST_EXPECT_H 1 24 | 25 | #include "test-reader.h" 26 | 27 | #ifdef __cplusplus 28 | extern "C" { 29 | #endif 30 | 31 | #if MPACK_EXPECT 32 | void test_expect(void); 33 | #endif 34 | 35 | #ifdef __cplusplus 36 | } 37 | #endif 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /test/unit/src/test-file.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | * this software and associated documentation files (the "Software"), to deal in 6 | * the Software without restriction, including without limitation the rights to 7 | * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 8 | * the Software, and to permit persons to whom the Software is furnished to do so, 9 | * subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 16 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 17 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 18 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 19 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 20 | */ 21 | 22 | /* 23 | * test-file.h 24 | */ 25 | 26 | #ifndef MPACK_TEST_FILE_H 27 | #define MPACK_TEST_FILE_H 1 28 | 29 | #include "test.h" 30 | 31 | #ifdef __cplusplus 32 | extern "C" { 33 | #endif 34 | 35 | #if MPACK_STDIO 36 | void test_file(void); 37 | #endif 38 | 39 | #ifdef __cplusplus 40 | } 41 | #endif 42 | 43 | #endif 44 | 45 | -------------------------------------------------------------------------------- /src/mpack/mpack.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | * this software and associated documentation files (the "Software"), to deal in 6 | * the Software without restriction, including without limitation the rights to 7 | * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 8 | * the Software, and to permit persons to whom the Software is furnished to do so, 9 | * subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 16 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 17 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 18 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 19 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 20 | */ 21 | 22 | /** 23 | * @file 24 | * 25 | * Includes the full MPack API. Only this file needs to be included 26 | * in application code. 27 | */ 28 | 29 | #ifndef MPACK_H 30 | #define MPACK_H 1 31 | 32 | #include "mpack-common.h" 33 | #include "mpack-writer.h" 34 | #include "mpack-reader.h" 35 | #include "mpack-expect.h" 36 | #include "mpack-node.h" 37 | 38 | #endif 39 | 40 | -------------------------------------------------------------------------------- /test/fuzz/fuzz-config.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018-2021 Nicholas Fraser and the MPack authors 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | * this software and associated documentation files (the "Software"), to deal in 6 | * the Software without restriction, including without limitation the rights to 7 | * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 8 | * the Software, and to permit persons to whom the Software is furnished to do so, 9 | * subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 16 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 17 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 18 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 19 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 20 | */ 21 | 22 | #ifndef MPACK_FUZZ_CONFIG_H 23 | #define MPACK_FUZZ_CONFIG_H 24 | 25 | #define MPACK_FUZZ 26 | 27 | // we use small buffer sizes to test flushing and growing 28 | #define MPACK_TRACKING_INITIAL_CAPACITY 3 29 | #define MPACK_STACK_SIZE 33 30 | #define MPACK_BUFFER_SIZE 33 31 | #define MPACK_NODE_PAGE_SIZE 113 32 | #define MPACK_NODE_INITIAL_DEPTH 3 33 | 34 | #endif 35 | -------------------------------------------------------------------------------- /test/unit/src/test-buffer.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | * this software and associated documentation files (the "Software"), to deal in 6 | * the Software without restriction, including without limitation the rights to 7 | * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 8 | * the Software, and to permit persons to whom the Software is furnished to do so, 9 | * subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 16 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 17 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 18 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 19 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 20 | */ 21 | 22 | /* 23 | * test-buffer.h 24 | * 25 | * Tests whether the buffer flush/fill functions are used correctly 26 | * with various buffer sizes. 27 | */ 28 | 29 | #ifndef MPACK_TEST_BUFFER_H 30 | #define MPACK_TEST_BUFFER_H 1 31 | 32 | #include "test.h" 33 | 34 | #ifdef __cplusplus 35 | extern "C" { 36 | #endif 37 | 38 | void test_buffers(void); 39 | 40 | #ifdef __cplusplus 41 | } 42 | #endif 43 | 44 | #endif 45 | 46 | 47 | -------------------------------------------------------------------------------- /test/avr/Makefile: -------------------------------------------------------------------------------- 1 | # This builds MPack and its test suite with avr-gcc for AVR (e.g. Arduino.) 2 | # It doesn't actually work yet; in fact it doesn't link because the resulting 3 | # code is way too big. But it does actually compile all the source files at 4 | # least. Eventually it would be nice to trim the unit test suite down into 5 | # something that could run on an Arduino. It would also be nice to set up some 6 | # wrapper scripts to make it run under simavr so we can run it on CI builds. 7 | 8 | # Requires avr-gcc and avr-libc (even though it builds with MPACK_STDLIB 9 | # disabled) since avr-libc has stdint.h and friends. 10 | 11 | ifeq (Makefile, $(firstword $(MAKEFILE_LIST))) 12 | $(error The current directory should be the root of the repository. Try "cd ../.." and then "make -f test/avr/Makefile") 13 | endif 14 | 15 | CC=avr-gcc 16 | BUILD := .build/avr 17 | PROG := mpack-avr 18 | 19 | CPPFLAGS := -Isrc -Itest/unit/src -DMPACK_HAS_CONFIG=1 20 | CFLAGS := \ 21 | -Os -DNDEBUG \ 22 | -Wall -Wextra -Wpedantic -Werror \ 23 | -MMD -MP 24 | 25 | SRCS := \ 26 | $(shell find src/ -type f -name '*.c') \ 27 | $(wildcard test/test*.c) 28 | 29 | OBJS := $(patsubst %, $(BUILD)/%.o, $(SRCS)) 30 | 31 | GLOBAL_DEPENDENCIES := test/avr/Makefile 32 | 33 | .PHONY: all 34 | all: $(PROG) 35 | 36 | .PHONY: clean 37 | clean: 38 | rm -rf $(BUILD) 39 | 40 | -include $(patsubst %, $(BUILD)/%.d, $(SRCS)) 41 | 42 | .PHONY: $(PROG) 43 | $(PROG): $(BUILD)/$(PROG) 44 | 45 | $(OBJS): $(BUILD)/%.o: % $(GLOBAL_DEPENDENCIES) 46 | @mkdir -p $(dir $@) 47 | $(CC) -c $(CPPFLAGS) $(CFLAGS) -o $@ $< 48 | 49 | $(BUILD)/$(PROG): $(OBJS) 50 | @mkdir -p $(dir $@) 51 | $(CC) $(CPPFLAGS) $(CFLAGS) -o $@ $^ $(LDFLAGS) 52 | -------------------------------------------------------------------------------- /docs/doxygen-layout.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /docs/doxyfile: -------------------------------------------------------------------------------- 1 | # Doxyfile for MPack 2 | # 3 | # The documentation must be generated with tools/gendocs.sh 4 | 5 | PROJECT_NAME = "MPack" 6 | PROJECT_BRIEF = "A C encoding/decoding library for the MessagePack serialization format." 7 | 8 | PROJECT_NUMBER = develop 9 | 10 | INPUT = \ 11 | .build/docs/README.temp.md \ 12 | docs/reader.md \ 13 | docs/expect.md \ 14 | docs/node.md \ 15 | docs/features.md \ 16 | docs/protocol.md \ 17 | src/mpack/mpack-platform.h \ 18 | src/mpack/mpack-common.h \ 19 | src/mpack/mpack-writer.h \ 20 | src/mpack/mpack-reader.h \ 21 | src/mpack/mpack-expect.h \ 22 | src/mpack/mpack-node.h \ 23 | src/mpack/mpack.h \ 24 | 25 | LAYOUT_FILE = docs/doxygen-layout.xml 26 | USE_MDFILE_AS_MAINPAGE = README.temp.md 27 | HTML_OUTPUT = .build/docs/html 28 | GENERATE_LATEX = no 29 | STRIP_FROM_PATH = . src .build/docs 30 | HTML_EXTRA_STYLESHEET = docs/doxygen-mpack-css.css 31 | 32 | PREDEFINED = \ 33 | DEBUG=1 \ 34 | \ 35 | inline= \ 36 | MPACK_INLINE= \ 37 | MPACK_SILENCE_WARNINGS_BEGIN= \ 38 | MPACK_SILENCE_WARNINGS_END= \ 39 | MPACK_EXTERN_C_BEGIN= \ 40 | MPACK_EXTERN_C_END= \ 41 | \ 42 | MPACK_HAS_GENERIC=1 \ 43 | \ 44 | MPACK_COMPATIBILITY=1 \ 45 | MPACK_EXTENSIONS=1 \ 46 | \ 47 | MPACK_DOXYGEN=1 \ 48 | 49 | MARKDOWN_SUPPORT = YES 50 | JAVADOC_AUTOBRIEF = YES 51 | ALWAYS_DETAILED_SEC = YES 52 | SORT_BRIEF_DOCS = YES 53 | 54 | INCLUDE_PATH = .build/docs src 55 | 56 | SEARCH_INCLUDES = YES 57 | MACRO_EXPANSION = YES 58 | OPTIMIZE_OUTPUT_FOR_C = YES 59 | INLINE_SIMPLE_STRUCTS = YES 60 | TYPEDEF_HIDES_STRUCT = YES 61 | 62 | SHOW_FILES = NO 63 | VERBATIM_HEADERS = NO 64 | ALPHABETICAL_INDEX = NO 65 | 66 | # warn about anything undocumented 67 | HIDE_UNDOC_MEMBERS = NO 68 | WARNINGS = YES 69 | WARN_IF_UNDOCUMENTED = YES 70 | WARN_IF_DOC_ERROR = YES 71 | #WARN_NO_PARAMDOC = YES 72 | 73 | QUIET = YES 74 | -------------------------------------------------------------------------------- /tools/amalgamate.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This script amalgamates the MPack code into a single pair of 3 | # source files, mpack.h and mpack.c. The resulting amalgamation 4 | # is in .build/amalgamation/ (without documentation.) 5 | 6 | . "`dirname $0`"/getversion.sh 7 | 8 | HEADERS="\ 9 | mpack/mpack-platform.h \ 10 | mpack/mpack-common.h \ 11 | mpack/mpack-writer.h \ 12 | mpack/mpack-reader.h \ 13 | mpack/mpack-expect.h \ 14 | mpack/mpack-node.h \ 15 | " 16 | 17 | SOURCES="\ 18 | mpack/mpack-platform.c \ 19 | mpack/mpack-common.c \ 20 | mpack/mpack-writer.c \ 21 | mpack/mpack-reader.c \ 22 | mpack/mpack-expect.c \ 23 | mpack/mpack-node.c \ 24 | " 25 | 26 | TOOLS="\ 27 | tools/afl.sh \ 28 | tools/clean.sh \ 29 | tools/coverage.sh \ 30 | tools/scan-build.sh \ 31 | tools/unit.bat \ 32 | tools/unit.sh \ 33 | tools/valgrind-suppressions \ 34 | " 35 | 36 | FILES="\ 37 | test \ 38 | LICENSE \ 39 | AUTHORS.md \ 40 | README.md \ 41 | CHANGELOG.md \ 42 | " 43 | 44 | # add top license and comment 45 | rm -rf .build/amalgamation 46 | mkdir -p .build/amalgamation/src/mpack 47 | HEADER=.build/amalgamation/src/mpack/mpack.h 48 | SOURCE=.build/amalgamation/src/mpack/mpack.c 49 | echo '/**' > $HEADER 50 | sed 's/^/ * /' LICENSE >> $HEADER 51 | cat - >> $HEADER <> $HEADER 65 | echo -e "#define MPACK_AMALGAMATED 1\n" >> $HEADER 66 | echo -e "#if defined(MPACK_HAS_CONFIG) && MPACK_HAS_CONFIG" >> $HEADER 67 | echo -e "#include \"mpack-config.h\"" >> $HEADER 68 | echo -e "#endif\n" >> $HEADER 69 | for f in $HEADERS; do 70 | echo -e "\n/* $f.h */" >> $HEADER 71 | sed -e 's@^#include ".*@/* & */@' -e '0,/^ \*\/$/d' src/$f >> $HEADER 72 | done 73 | echo -e "#endif\n" >> $HEADER 74 | 75 | # assemble source 76 | echo -e "#define MPACK_INTERNAL 1" >> $SOURCE 77 | echo -e "#define MPACK_EMIT_INLINE_DEFS 1\n" >> $SOURCE 78 | echo -e "#include \"mpack.h\"\n" >> $SOURCE 79 | for f in $SOURCES; do 80 | echo -e "\n/* $f.c */" >> $SOURCE 81 | sed -e 's@^#include ".*@/* & */@' -e '0,/^ \*\/$/d' src/$f >> $SOURCE 82 | done 83 | 84 | # assemble package contents 85 | cp -a $FILES .build/amalgamation 86 | mkdir -p .build/amalgamation/tools 87 | cp -a $TOOLS .build/amalgamation/tools 88 | 89 | # done! 90 | echo "Done. MPack amalgamation is in .build/amalgamation/" 91 | -------------------------------------------------------------------------------- /test/unit/src/test-system.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | * this software and associated documentation files (the "Software"), to deal in 6 | * the Software without restriction, including without limitation the rights to 7 | * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 8 | * the Software, and to permit persons to whom the Software is furnished to do so, 9 | * subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 16 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 17 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 18 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 19 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 20 | */ 21 | 22 | #ifndef MPACK_TEST_SYSTEM_H 23 | #define MPACK_TEST_SYSTEM_H 1 24 | 25 | #include 26 | #include 27 | 28 | #ifdef __cplusplus 29 | extern "C" { 30 | #endif 31 | 32 | 33 | // Causes the next `count` system calls to succeed, and the 34 | // following call to fail. If all is true, all subsequent 35 | // calls will fail until the system is reset; otherwise the 36 | // system is reset immediately. 37 | void test_system_fail_after(size_t count, bool all); 38 | 39 | // Resets the system call failure simulation, allowing all 40 | // system calls to succeed 41 | void test_system_fail_reset(void); 42 | 43 | // Runs the given test repeatedly. On each iteration n, the test 44 | // is run failing only the nth system call, and again failing the 45 | // nth and all subsequent system calls. Repeats until both tests 46 | // return true. 47 | void test_system_fail_until_ok(bool (*test)(void)); 48 | 49 | 50 | // runs system tests 51 | void test_system(void); 52 | 53 | 54 | #ifdef MPACK_MALLOC 55 | void* test_malloc(size_t size); 56 | void* test_realloc(void* p, size_t size); 57 | void test_free(void* p); 58 | 59 | // Returns the number of mallocs that have not yet been freed. 60 | size_t test_malloc_active_count(void); 61 | 62 | // Returns the total number of mallocs or non-zero reallocs ever made. 63 | size_t test_malloc_total_count(void); 64 | #endif 65 | 66 | 67 | #if defined(MPACK_STDLIB) && MPACK_STDLIB 68 | // Causes the next call to strlen() to return the given value. 69 | void test_system_mock_strlen(size_t len); 70 | 71 | size_t test_strlen(const char* s); 72 | #endif 73 | 74 | 75 | #if defined(MPACK_STDIO) && MPACK_STDIO 76 | FILE* test_fopen(const char* path, const char* mode); 77 | int test_fclose(FILE* stream); 78 | size_t test_fread(void* ptr, size_t size, size_t nmemb, FILE* stream); 79 | size_t test_fwrite(const void* ptr, size_t size, size_t nmemb, FILE* stream); 80 | int test_fseek(FILE* stream, long offset, int whence); 81 | long test_ftell(FILE* stream); 82 | int test_ferror(FILE* stream); 83 | 84 | // Returns the number of files that have not yet been closed. 85 | size_t test_files_count(void); 86 | #endif 87 | 88 | 89 | #ifdef __cplusplus 90 | } 91 | #endif 92 | 93 | #endif 94 | 95 | -------------------------------------------------------------------------------- /docs/node.md: -------------------------------------------------------------------------------- 1 | # Using the Node API 2 | 3 | The Node API is used to parse MessagePack data into a tree in memory, providing DOM-style random access to its contents. 4 | 5 | The Node API can parse from a chunk of data in memory, or it can pull data from a file or stream. When the Node API uses a file or stream, it collects the data for a single message into one contiguous buffer in memory as it parses. The nodes for strings and binary data simply point to the data in the buffer, minimizing copies of the data. In this sense, a tree is just an index over a contiguous MessagePack message in memory. 6 | 7 | ## The Basics 8 | 9 | A tree is initialized with one of the `init()` functions, and optionally configured (e.g. with `mpack_set_limits()`.) A complete message is then parsed with `mpack_tree_parse()`. 10 | 11 | The data can then be accessed in random order. For example: 12 | 13 | ```C 14 | bool parse_messagepack(const char* data, size_t count) { 15 | mpack_tree_t tree; 16 | mpack_tree_init_data(&tree, data, count); 17 | mpack_tree_parse(&tree); 18 | 19 | mpack_node_t root = mpack_tree_root(&tree); 20 | do_something_with_node(root); 21 | 22 | return mpack_tree_destroy(&tree) == mpack_ok; 23 | } 24 | ``` 25 | 26 | As with the Expect API, the Node API contains helper functions for extracting data of expected types from the tree, and for stepping into maps and arrays. These can be nested together to quickly unpack a message. 27 | 28 | For example, to parse the data on the [msgpack.org](https://msgpack.org) homepage from a node: 29 | 30 | ```C 31 | void parse_msgpack_homepage(mpack_node_t node, bool* compact, int* schema) { 32 | *compact = mpack_node_bool(mpack_node_map_cstr(root, "compact")); 33 | *schema = mpack_node_int(mpack_node_map_cstr(root, "schema")); 34 | } 35 | ``` 36 | 37 | If any error occurs, these functions always return safe values and flag an error on the tree. You do not have to check the tree for errors in between each step; you only need an error check before using the data. 38 | 39 | ## Continuous Streams 40 | 41 | The Node API can parse messages indefinitely from a continuous stream. This can be used for inter-process or network communications. See [msgpack-rpc](https://github.com/msgpack-rpc/msgpack-rpc) for an example networking protocol. 42 | 43 | Here's a minimal example that wraps a tree parser around a BSD socket. We've defined a `stream_t` object to contain our file descriptor and other stream state, and we use it as the tree's context. 44 | 45 | ```C 46 | #define MAX_SIZE (1024*1024) 47 | #define MAX_NODES 1024 48 | 49 | typedef struct stream_t { 50 | int fd; 51 | } stream_t; 52 | 53 | static size_t read_stream(mpack_tree_t* tree, char* buffer, size_t count) { 54 | stream_t* stream = mpack_tree_context(tree); 55 | ssize_t step = read(stream->fd, buffer, count); 56 | if (step <= 0) 57 | mpack_tree_flag_error(tree, mpack_error_io); 58 | } 59 | 60 | void parse_stream(stream_t* stream) { 61 | mpack_tree_t tree; 62 | mpack_tree_init_stream(&tree, &read_stream, stream, MAX_SIZE, MAX_NODES); 63 | 64 | while (true) { 65 | mpack_tree_parse(&tree); 66 | if (mpack_tree_error(&tree) != mpack_ok) 67 | break; 68 | 69 | received_message(mpack_tree_root(&tree)); 70 | } 71 | } 72 | ``` 73 | 74 | The function `received_message()` will be called with each new message received from the peer. 75 | 76 | The Node API contains many more features, including non-blocking parsing, optional map lookups and more. This document is a work in progress. See the Node API reference for more information. 77 | -------------------------------------------------------------------------------- /test/README.md: -------------------------------------------------------------------------------- 1 | The MPack build process does not build MPack into a library; it is used to build and run various tests. You do not need to build MPack or the unit testing suite to use MPack. 2 | 3 | # Unit Tests 4 | 5 | ## Dependencies 6 | 7 | The unit test suite (on Linux and Windows) requires Python 3 (at least 3.5) and Ninja (at least 1.3). On Windows, it also requires Visual Studio with a version of native build tools. Ninja is included in the Visual Studio 2019 build tools but not in earlier versions. 8 | 9 | The unit test suite can also make use of 32-bit cross-compiling support if on a 32-bit Linux system. To do this, install the appropriate package: 10 | 11 | - On Arch, this is `gcc-multilib` or `lib32-clang` 12 | - On Ubuntu, this is `gcc-multilib` and `g++-multilib` 13 | 14 | In addition, if Valgrind is installed and the compiler supports 32-bit cross-compiling, you will need to install Valgrind's 32-bit support, e.g. `valgrind-multilib` and/or `libc6-dbg:i386`. 15 | 16 | ## Running the unit tests 17 | 18 | Run the build script: 19 | 20 | ```sh 21 | tools/unit.sh 22 | ``` 23 | 24 | or on Windows: 25 | 26 | ``` 27 | tools\unit.bat 28 | ``` 29 | 30 | This will run a Python script which generates a Ninja file, then invoke Ninja on it in the default configuration. 31 | 32 | ## Running more configurations 33 | 34 | You can run additional tests by passing specific targets on the command-line: 35 | 36 | - The "help" target lists all targets; 37 | - The "more" target runs additional configurations (those run by the CI); 38 | - The "all" target runs all tests (those run on various platforms and compilers before each release); 39 | - Most targets take the form `run-{name}-{mode}` where `{name}` is a configuration name and `{mode}` is either `debug` or `release`. 40 | 41 | For example, to run all tests with the default compiler: 42 | 43 | ```sh 44 | tools/unit.sh all 45 | ``` 46 | 47 | To run only a specific configuration, say, the debug `noio` configuration where `MPACK_STDIO` is disabled: 48 | 49 | ```sh 50 | tools/unit.sh run-noio-debug 51 | ``` 52 | 53 | To list all targets: 54 | 55 | ```sh 56 | tools/unit.sh help 57 | ``` 58 | 59 | The Windows script `tools\unit.bat` takes the same arguments. 60 | 61 | ### Choosing a compiler 62 | 63 | On Linux or macOS, you can choose the compiler by passing a different `CC` to the build script. For example, to build with [TinyCC](https://bellard.org/tcc/tcc-doc.html): 64 | 65 | ```sh 66 | CC=tcc tools/unit.sh 67 | ``` 68 | 69 | On Windows, you can run it in a build tools command prompt or load the build tools yourself to use a specific toolset. If no build tools are loaded it will load the latest Visual Studio Native Build Tools automatically. 70 | 71 | # Fuzz Testing 72 | 73 | MPack supports fuzzing with american fuzzy lop. Run `tools/afl.sh` to fuzz MPack. 74 | 75 | Fuzzing paths are limited right now. The fuzzer: 76 | 77 | - decodes stdin with the dynamic Reader API; 78 | - encodes the data to a growable buffer with the Write API; 79 | - parses the resulting buffer with the Node API; 80 | - and finally, prints a debug dump of the node tree to stdout. 81 | 82 | It thus passes all data through three major components of MPack. Not tested currently are the Expect API (and its many functions like range helpers) nor the Builder API. 83 | 84 | # AVR / Arduino 85 | 86 | MPack contains a Makefile for building the unit test suite for AVR. You'll need `avr-gcc` and `avr-libc` installed. 87 | 88 | Build it like this: 89 | 90 | ```sh 91 | make -f test/avr/Makefile 92 | ``` 93 | 94 | It doesn't actually link yet since the tests are too big. I don't have an Arduino to test it with anyway. But the object files compile at least. 95 | 96 | # Linux Kernel 97 | 98 | The [mpack-linux-kernel](https://github.com/ludocode/mpack-linux-kernel) project contains a KBuild and shims to build the MPack unit test suite as a Linux kernel module. This is kept separate from MPack to avoid having to dual-license MPack under the GPL. 99 | -------------------------------------------------------------------------------- /test/unit/src/test.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | * this software and associated documentation files (the "Software"), to deal in 6 | * the Software without restriction, including without limitation the rights to 7 | * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 8 | * the Software, and to permit persons to whom the Software is furnished to do so, 9 | * subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 16 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 17 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 18 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 19 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 20 | */ 21 | 22 | #include "test.h" 23 | 24 | #include "test-reader.h" 25 | #include "test-expect.h" 26 | #include "test-write.h" 27 | #include "test-builder.h" 28 | #include "test-buffer.h" 29 | #include "test-common.h" 30 | #include "test-node.h" 31 | #include "test-file.h" 32 | 33 | mpack_tag_t (*fn_mpack_tag_nil)(void) = &mpack_tag_nil; 34 | 35 | int passes; 36 | int tests; 37 | 38 | const char* lipsum = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed nec justo purus. Nunc finibus dolor id lorem sagittis, euismod efficitur arcu aliquam. Nullam a ante eget mi porttitor dignissim vitae at libero. Maecenas in justo massa. Mauris ultricies leo nisl, at ullamcorper erat maximus sit amet. Quisque pharetra sed ligula nec tristique. Mauris consectetur sapien lacus, et pharetra turpis rhoncus a. Sed in eleifend eros. Donec in libero lacus. Sed et finibus ipsum. Etiam eros leo, mollis eget molestie quis, rhoncus ac magna. Donec dolor risus, bibendum et scelerisque at, faucibus in mi. Interdum et malesuada fames ac ante ipsum primis in faucibus. Vestibulum convallis accumsan mollis."; 39 | 40 | #if MPACK_CUSTOM_ASSERT 41 | bool test_jmp_set = false; 42 | jmp_buf test_jmp_buf; 43 | bool test_break_set = false; 44 | bool test_break_hit; 45 | 46 | void mpack_assert_fail(const char* message) { 47 | if (!test_jmp_set) { 48 | TEST_TRUE(false, "assertion hit! %s", message); 49 | abort(); 50 | } 51 | longjmp(test_jmp_buf, 1); 52 | } 53 | 54 | void mpack_break_hit(const char* message) { 55 | if (!test_break_set) { 56 | TEST_TRUE(false, "break hit! %s", message); 57 | abort(); 58 | } 59 | test_break_hit = true; 60 | } 61 | #endif 62 | 63 | void test_true_impl(bool result, const char* file, int line, const char* format, ...) { 64 | ++tests; 65 | if (result) { 66 | ++passes; 67 | } else { 68 | printf("TEST FAILED AT %s:%i --", file, line); 69 | 70 | va_list args; 71 | va_start(args, format); 72 | vprintf(format, args); 73 | va_end(args); 74 | 75 | printf("\n"); 76 | if (TEST_EARLY_EXIT) 77 | abort(); 78 | } 79 | } 80 | 81 | int main(void) { 82 | printf("\n\n"); 83 | 84 | test_system(); 85 | test_common(); 86 | 87 | #if MPACK_READER 88 | test_reader(); 89 | #endif 90 | #if MPACK_EXPECT 91 | test_expect(); 92 | #endif 93 | #if MPACK_WRITER 94 | test_writes(); 95 | #endif 96 | #if MPACK_BUILDER 97 | test_builder(); 98 | #endif 99 | #if MPACK_NODE 100 | test_node(); 101 | #endif 102 | #if MPACK_STDIO 103 | test_file(); 104 | #endif 105 | 106 | test_buffers(); 107 | 108 | printf("\n\nUnit testing complete. %i failures in %i checks.\n\n\n", tests - passes, tests); 109 | return (passes == tests) ? EXIT_SUCCESS : EXIT_FAILURE; 110 | } 111 | 112 | -------------------------------------------------------------------------------- /.github/workflows/unit.yml: -------------------------------------------------------------------------------- 1 | name: Unit Tests 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | 8 | macos-clang-amalgamated: 9 | name: "macOS Clang (amalgamated)" 10 | runs-on: macos-latest 11 | env: 12 | AMALGAMATED: 1 13 | steps: 14 | - uses: actions/checkout@v2 15 | 16 | - name: Install Dependencies 17 | run: brew install ninja 18 | 19 | - name: Run ci-unix 20 | run: test/unit/ci-unix.sh 21 | 22 | 23 | 24 | ubuntu-scan-build: 25 | name: "Clang Static Analysis (source)" 26 | runs-on: ubuntu-latest 27 | steps: 28 | - uses: actions/checkout@v2 29 | 30 | - name: Install Dependencies 31 | run: | 32 | sudo apt-get update 33 | sudo apt-get install -y ninja-build clang-tools 34 | 35 | - name: Run scan-build 36 | run: tools/scan-build.sh 37 | 38 | 39 | 40 | ubuntu-tcc: 41 | name: "TinyCC (source)" 42 | runs-on: ubuntu-latest 43 | env: 44 | CC: tcc 45 | CXX: /bin/false 46 | steps: 47 | - uses: actions/checkout@v2 48 | 49 | - name: Install Dependencies 50 | run: | 51 | sudo apt-get update 52 | sudo apt-get install -y ninja-build tcc 53 | 54 | - name: Run ci-unix 55 | run: test/unit/ci-unix.sh 56 | 57 | 58 | 59 | ubuntu-gcc-9: 60 | name: "Ubuntu 20.04 GCC 9 (source)" 61 | runs-on: ubuntu-20.04 62 | steps: 63 | - uses: actions/checkout@v2 64 | 65 | - name: Install Ninja 66 | run: | 67 | sudo apt-get update 68 | sudo apt-get install -y ninja-build 69 | 70 | - name: Install lcov 71 | run: | 72 | curl -LO https://github.com/linux-test-project/lcov/releases/download/v1.15/lcov-1.15.tar.gz 73 | tar xzf lcov-1.15.tar.gz 74 | cd lcov-1.15 75 | sudo make install 76 | 77 | - name: Run ci-unix 78 | run: test/unit/ci-unix.sh 79 | 80 | - name: Collect coverage 81 | run: tools/coverage.sh 82 | 83 | - name: Submit coverage to Coveralls 84 | id: coverage 85 | uses: coverallsapp/github-action@1.1.3 86 | with: 87 | github-token: ${{ secrets.GITHUB_TOKEN }} 88 | 89 | - name: Print Coveralls submission result 90 | run: echo ${{ steps.coverage.outputs['coveralls-api-result'] }} 91 | 92 | 93 | 94 | ubuntu-gcc-7: 95 | name: "Ubuntu 20.04 GCC 7 (amalgamated)" 96 | runs-on: ubuntu-20.04 97 | env: 98 | CC: gcc-7 99 | CXX: g++-7 100 | AMALGAMATED: 1 101 | steps: 102 | - uses: actions/checkout@v2 103 | 104 | - name: Install Dependencies 105 | run: | 106 | sudo apt-get update 107 | sudo apt-get install -y ninja-build gcc-7 108 | 109 | - name: Run ci-unix 110 | run: test/unit/ci-unix.sh 111 | 112 | 113 | 114 | ubuntu-gcc-5: 115 | name: "Ubuntu 18.04 GCC 5 (amalgamated)" 116 | runs-on: ubuntu-18.04 117 | env: 118 | CC: gcc-5 119 | CXX: g++-5 120 | AMALGAMATED: 1 121 | steps: 122 | - uses: actions/checkout@v2 123 | 124 | - name: Install Dependencies 125 | # https://askubuntu.com/a/1087368 126 | # note for when 18.04 is EOL: https://askubuntu.com/a/1313032 127 | run: | 128 | sudo apt-get update 129 | sudo apt-get install -y ninja-build gcc-5 130 | 131 | - name: Run ci-unix 132 | run: test/unit/ci-unix.sh 133 | 134 | 135 | 136 | windows-cl-2019-x64: 137 | name: "Windows VS2019 amd64 (source)" 138 | runs-on: windows-latest 139 | env: 140 | COMPILER: cl-2019-x64 141 | steps: 142 | - uses: actions/checkout@v2 143 | 144 | - name: Run ci-windows (Windows) 145 | shell: cmd 146 | run: test\\unit\\ci-windows.bat 147 | 148 | 149 | 150 | windows-cl-2015-x86: 151 | name: "Windows VS2015 x86 (amalgamated)" 152 | runs-on: windows-latest 153 | env: 154 | COMPILER: cl-2015-x86 155 | AMALGAMATED: 1 156 | steps: 157 | - uses: actions/checkout@v2 158 | 159 | - name: Amalgamate 160 | shell: bash 161 | run: tools\\amalgamate.sh 162 | 163 | - name: Run ci-windows (Windows) 164 | shell: cmd 165 | run: test\\unit\\ci-windows.bat 166 | -------------------------------------------------------------------------------- /test/unit/src/test-write.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | * this software and associated documentation files (the "Software"), to deal in 6 | * the Software without restriction, including without limitation the rights to 7 | * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 8 | * the Software, and to permit persons to whom the Software is furnished to do so, 9 | * subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 16 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 17 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 18 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 19 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 20 | */ 21 | 22 | #ifndef MPACK_TEST_WRITE_H 23 | #define MPACK_TEST_WRITE_H 1 24 | 25 | #include "test.h" 26 | 27 | #ifdef __cplusplus 28 | extern "C" { 29 | #endif 30 | 31 | #if MPACK_WRITER 32 | 33 | extern mpack_error_t test_write_error; 34 | void test_write_error_handler(mpack_writer_t* writer, mpack_error_t error); 35 | 36 | 37 | // these setup and destroy test writers and check them for errors. 38 | // they are generally macros so that the asserts are on the line of the test. 39 | 40 | 41 | // tears down a writer, ensuring it didn't fail 42 | #define TEST_WRITER_DESTROY_NOERROR(writer) do {\ 43 | mpack_error_t error = mpack_writer_destroy(writer); \ 44 | TEST_TRUE(error == mpack_ok, "writer is in error state %i (%s)", \ 45 | (int)error, mpack_error_to_string(error)); \ 46 | } while (0) 47 | 48 | // tears down a writer, ensuring the given error occurred 49 | #define TEST_WRITER_DESTROY_ERROR(writer, error) do { \ 50 | mpack_error_t expected = (error); \ 51 | mpack_error_t actual = mpack_writer_destroy(writer); \ 52 | TEST_TRUE(actual == expected, "writer is in error state %i (%s) instead of %i (%s)", \ 53 | (int)actual, mpack_error_to_string(actual), \ 54 | (int)expected, mpack_error_to_string(expected)); \ 55 | } while (0) 56 | 57 | // performs an operation on a writer, ensuring no error occurs 58 | #define TEST_WRITE_NOERROR(writer, write_expr) do { \ 59 | (write_expr); \ 60 | mpack_error_t error = mpack_writer_error(writer); \ 61 | TEST_TRUE(error == mpack_ok, "writer is in error state %i (%s)", \ 62 | (int)error, mpack_error_to_string(error)); \ 63 | } while (0) 64 | 65 | #define TEST_DESTROY_MATCH_IMPL(buf, expect) do { \ 66 | static const char data[] = expect; \ 67 | size_t used = mpack_writer_buffer_used(&writer); \ 68 | TEST_WRITER_DESTROY_NOERROR(&writer); \ 69 | TEST_TRUE(sizeof(data)-1 == used && memcmp(data, buf, used) == 0, \ 70 | "written data (of length %i) does not match expected (of length %i)", \ 71 | (int)used, (int)(sizeof(data)-1)); \ 72 | } while (0) 73 | 74 | #define TEST_DESTROY_MATCH(buf, expect) do { \ 75 | TEST_DESTROY_MATCH_IMPL(buf, expect); \ 76 | if (buf) MPACK_FREE(buf); \ 77 | } while (0) 78 | 79 | // runs a simple writer test, ensuring it matches the given data 80 | #define TEST_SIMPLE_WRITE(expect, write_op) do { \ 81 | mpack_writer_init(&writer, buf, sizeof(buf)); \ 82 | mpack_writer_set_error_handler(&writer, test_write_error_handler); \ 83 | write_op; \ 84 | TEST_DESTROY_MATCH_IMPL(buf, expect); \ 85 | TEST_TRUE(test_write_error == mpack_ok); \ 86 | test_write_error = mpack_ok; \ 87 | } while (0) 88 | 89 | // runs a simple writer test, ensuring it does not cause an error and ignoring the result 90 | #define TEST_SIMPLE_WRITE_NOERROR(write_op) do { \ 91 | mpack_writer_init(&writer, buf, sizeof(buf)); \ 92 | mpack_writer_set_error_handler(&writer, test_write_error_handler); \ 93 | (write_op); \ 94 | TEST_WRITER_DESTROY_NOERROR(&writer); \ 95 | TEST_TRUE(test_write_error == mpack_ok); \ 96 | test_write_error = mpack_ok; \ 97 | } while (0) 98 | 99 | // runs a simple writer test, ensuring it does causes the given error 100 | #define TEST_SIMPLE_WRITE_ERROR(write_op, error) do { \ 101 | mpack_error_t expected2 = (error); \ 102 | mpack_writer_init(&writer, buf, sizeof(buf)); \ 103 | mpack_writer_set_error_handler(&writer, test_write_error_handler); \ 104 | (write_op); \ 105 | TEST_WRITER_DESTROY_ERROR(&writer, expected2); \ 106 | TEST_TRUE(test_write_error == expected2); \ 107 | test_write_error = mpack_ok; \ 108 | } while (0) 109 | 110 | void test_writes(void); 111 | 112 | #endif 113 | 114 | #ifdef __cplusplus 115 | } 116 | #endif 117 | 118 | #endif 119 | 120 | -------------------------------------------------------------------------------- /test/unit/src/mpack-config.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | * this software and associated documentation files (the "Software"), to deal in 6 | * the Software without restriction, including without limitation the rights to 7 | * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 8 | * the Software, and to permit persons to whom the Software is furnished to do so, 9 | * subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 16 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 17 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 18 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 19 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 20 | */ 21 | 22 | #ifndef MPACK_CONFIG_H 23 | #define MPACK_CONFIG_H 1 24 | 25 | // This is the configuration for the MPack test harness. 26 | 27 | #define MPACK_UNIT_TESTS 1 28 | 29 | #if defined(DEBUG) || defined(_DEBUG) 30 | #define MPACK_DEBUG 1 31 | #endif 32 | 33 | #ifdef MPACK_VARIANT_BUILDS 34 | // Most options such as featureset and platform configuration 35 | // are specified by the buildsystem. Any options that are 36 | // unset on the command line are considered disabled. 37 | #ifndef MPACK_READER 38 | #define MPACK_READER 0 39 | #endif 40 | #ifndef MPACK_EXPECT 41 | #define MPACK_EXPECT 0 42 | #endif 43 | #ifndef MPACK_NODE 44 | #define MPACK_NODE 0 45 | #endif 46 | #ifndef MPACK_WRITER 47 | #define MPACK_WRITER 0 48 | #endif 49 | #ifndef MPACK_STDLIB 50 | #define MPACK_STDLIB 0 51 | #endif 52 | #ifndef MPACK_STDIO 53 | #define MPACK_STDIO 0 54 | #endif 55 | 56 | #elif defined(__AVR__) 57 | #define MPACK_STDLIB 0 58 | #define MPACK_STDIO 0 59 | 60 | #else 61 | // For other platforms, we currently only test in a single configuration, 62 | // so we enable everything and otherwise use the default for most settings. 63 | #define MPACK_COMPATIBILITY 1 64 | #define MPACK_EXTENSIONS 1 65 | 66 | // We define our own allocators to test allocations. 67 | #define MPACK_MALLOC test_malloc 68 | #define MPACK_FREE test_free 69 | 70 | // We need MPACK_STDLIB and MPACK_STDIO defined before test-system.h to 71 | // override their functions. 72 | #define MPACK_STDLIB 1 73 | #define MPACK_STDIO 1 74 | 75 | #endif 76 | 77 | // We've disabled the unit test for single inline under tcc. 78 | #ifdef __TINYC__ 79 | #define MPACK_DISABLE_TINYC_INLINE_WARNING 80 | #endif 81 | 82 | // We replace the file i/o functions to simulate failures 83 | #if MPACK_STDIO 84 | #include 85 | #undef fopen 86 | #undef fclose 87 | #undef fread 88 | #undef fwrite 89 | #undef fseek 90 | #undef ftell 91 | #undef ferror 92 | #define fopen test_fopen 93 | #define fclose test_fclose 94 | #define fread test_fread 95 | #define fwrite test_fwrite 96 | #define fseek test_fseek 97 | #define ftell test_ftell 98 | #define ferror test_ferror 99 | #endif 100 | 101 | // We replace strlen to simulate extremely large c-strings (only on stdlib 102 | // builds, so that non-stdlib builds test the mpack implementations) 103 | #if MPACK_STDLIB 104 | #define MPACK_STRLEN test_strlen 105 | #endif 106 | 107 | // Tracking matches the default config, except the test suite 108 | // also supports MPACK_NO_TRACKING to disable it. 109 | #if defined(MPACK_MALLOC) && !defined(MPACK_NO_TRACKING) 110 | #if defined(MPACK_DEBUG) && MPACK_DEBUG && defined(MPACK_READER) && MPACK_READER 111 | #define MPACK_READ_TRACKING 1 112 | #endif 113 | #if defined(MPACK_DEBUG) && MPACK_DEBUG && defined(MPACK_WRITER) && MPACK_WRITER 114 | #define MPACK_WRITE_TRACKING 1 115 | #endif 116 | #endif 117 | 118 | // We use a custom assert function which longjmps, allowing 119 | // us to test assertions in debug mode. 120 | #ifdef MPACK_DEBUG 121 | #define MPACK_CUSTOM_ASSERT 1 122 | #define MPACK_CUSTOM_BREAK 1 123 | #endif 124 | 125 | #include "test-system.h" 126 | 127 | // we use small buffer sizes to test flushing, growing, and malloc failures 128 | #define MPACK_TRACKING_INITIAL_CAPACITY 3 129 | #define MPACK_STACK_SIZE 33 130 | #define MPACK_BUFFER_SIZE 33 131 | #define MPACK_NODE_PAGE_SIZE 113 132 | #define MPACK_BUILDER_INTERNAL_STORAGE_SIZE (sizeof(mpack_builder_page_t) + \ 133 | sizeof(mpack_build_t) + MPACK_WRITER_MINIMUM_BUFFER_SIZE + 33) 134 | #define MPACK_BUILDER_PAGE_SIZE (sizeof(mpack_builder_page_t) + \ 135 | sizeof(mpack_build_t) + MPACK_WRITER_MINIMUM_BUFFER_SIZE + 77) 136 | 137 | /* 138 | #undef MPACK_BUILDER_INTERNAL_STORAGE_SIZE 139 | #define MPACK_BUILDER_INTERNAL_STORAGE_SIZE 0 140 | #undef MPACK_BUILDER_PAGE_SIZE 141 | #define MPACK_BUILDER_PAGE_SIZE 80 142 | */ 143 | 144 | #ifdef MPACK_MALLOC 145 | #define MPACK_NODE_INITIAL_DEPTH 3 146 | #else 147 | #define MPACK_NODE_MAX_DEPTH_WITHOUT_MALLOC 32 148 | #endif 149 | 150 | #endif 151 | -------------------------------------------------------------------------------- /test/unit/src/test-node.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | * this software and associated documentation files (the "Software"), to deal in 6 | * the Software without restriction, including without limitation the rights to 7 | * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 8 | * the Software, and to permit persons to whom the Software is furnished to do so, 9 | * subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 16 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 17 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 18 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 19 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 20 | */ 21 | 22 | #ifndef MPACK_TEST_NODE_H 23 | #define MPACK_TEST_NODE_H 1 24 | 25 | #include "test.h" 26 | 27 | #ifdef __cplusplus 28 | extern "C" { 29 | #endif 30 | 31 | #if MPACK_NODE 32 | 33 | extern mpack_error_t test_tree_error; 34 | void test_tree_error_handler(mpack_tree_t* tree, mpack_error_t error); 35 | 36 | 37 | #define TEST_TREE_DESTROY_NOERROR(tree) do { \ 38 | mpack_error_t error = mpack_tree_destroy(tree); \ 39 | TEST_TRUE(error == mpack_ok, "tree is in error state %i (%s)", \ 40 | (int)error, mpack_error_to_string(error)); \ 41 | } while (0) 42 | 43 | #define TEST_TREE_DESTROY_ERROR(tree, error) do { \ 44 | mpack_error_t expected = (error); \ 45 | mpack_error_t actual = mpack_tree_destroy(tree); \ 46 | TEST_TRUE(actual == expected, "tree is in error state %i (%s) instead of %i (%s)", \ 47 | (int)actual, mpack_error_to_string(actual), \ 48 | (int)expected, mpack_error_to_string(expected)); \ 49 | } while (0) 50 | 51 | #define TEST_SIMPLE_TREE_READ(data, read_expr) do { \ 52 | mpack_tree_init_pool(&tree, data, sizeof(data) - 1, pool, sizeof(pool) / sizeof(*pool)); \ 53 | mpack_tree_set_error_handler(&tree, test_tree_error_handler); \ 54 | mpack_tree_parse(&tree); \ 55 | mpack_node_t node = mpack_tree_root(&tree); \ 56 | TEST_TRUE((read_expr), "simple tree test did not pass: " #read_expr); \ 57 | TEST_TREE_DESTROY_NOERROR(&tree); \ 58 | TEST_TRUE(test_tree_error == mpack_ok); \ 59 | test_tree_error = mpack_ok; \ 60 | } while (0) 61 | 62 | #ifdef MPACK_MALLOC 63 | #define TEST_TREE_INIT mpack_tree_init 64 | #else 65 | #define TEST_TREE_INIT(tree, data, data_size) \ 66 | mpack_tree_init_pool((tree), (data), (data_size), pool, sizeof(pool) / sizeof(*pool)); 67 | #endif 68 | 69 | // the error handler is only called if the tree is not already in an 70 | // error state, so we call it ourselves if the tree init failed. 71 | #define TEST_SIMPLE_TREE_READ_ERROR(data, read_expr, error) do { \ 72 | mpack_tree_init_pool(&tree, data, sizeof(data) - 1, pool, sizeof(pool) / sizeof(*pool)); \ 73 | if (mpack_tree_error(&tree) != mpack_ok) \ 74 | test_tree_error_handler(&tree, error); \ 75 | mpack_tree_set_error_handler(&tree, test_tree_error_handler); \ 76 | mpack_tree_parse(&tree); \ 77 | mpack_node_t node = mpack_tree_root(&tree); \ 78 | TEST_TRUE((read_expr), "simple read error test did not pass: " #read_expr); \ 79 | TEST_TREE_DESTROY_ERROR(&tree, (error)); \ 80 | TEST_TRUE(test_tree_error == (error)); \ 81 | test_tree_error = mpack_ok; \ 82 | } while (0) 83 | 84 | 85 | 86 | // bug tests 87 | 88 | #if MPACK_DEBUG 89 | 90 | // runs a simple tree test, ensuring it causes an assert. 91 | // we flag mpack_error_data to cancel out of any tracking. 92 | // (note about volatile, see TEST_ASSERT()) 93 | #define TEST_SIMPLE_TREE_READ_ASSERT(data, read_expr) do { \ 94 | volatile mpack_tree_t v_tree; \ 95 | TEST_MPACK_SILENCE_SHADOW_BEGIN \ 96 | mpack_tree_t* tree = (mpack_tree_t*)(uintptr_t)&v_tree; \ 97 | TEST_MPACK_SILENCE_SHADOW_END \ 98 | mpack_tree_init_pool(tree, data, sizeof(data) - 1, pool, sizeof(pool) / sizeof(*pool)); \ 99 | mpack_tree_parse(tree); \ 100 | mpack_node_t node = mpack_tree_root(tree); \ 101 | TEST_ASSERT(read_expr); \ 102 | mpack_tree_destroy(tree); \ 103 | } while (0) 104 | 105 | #else 106 | 107 | // we cannot test asserts in release mode because they are 108 | // compiled away; code would continue to run and cause 109 | // undefined behavior. 110 | #define TEST_SIMPLE_TREE_READ_ASSERT(data, read_expr) ((void)0) 111 | 112 | #endif 113 | 114 | // runs a simple tree test, ensuring it causes a break in 115 | // debug mode and flags mpack_error_bug in both debug and release. 116 | #define TEST_SIMPLE_TREE_READ_BREAK(data, read_expr) do { \ 117 | mpack_tree_init_pool(&tree, data, sizeof(data) - 1, pool, sizeof(pool) / sizeof(*pool)); \ 118 | mpack_tree_parse(&tree); \ 119 | mpack_node_t node = mpack_tree_root(&tree); \ 120 | TEST_BREAK(read_expr); \ 121 | TEST_TREE_DESTROY_ERROR(&tree, mpack_error_bug); \ 122 | } while (0) 123 | 124 | 125 | 126 | void test_node(void); 127 | 128 | #endif 129 | 130 | #ifdef __cplusplus 131 | } 132 | #endif 133 | 134 | #endif 135 | 136 | 137 | -------------------------------------------------------------------------------- /docs/features.md: -------------------------------------------------------------------------------- 1 | # Feature Comparisons 2 | 3 | This document compares the features of various C libraries for encoding and decoding MessagePack. (It attempts to be neutral, but you may find it biased towards MPack since it is part of this project.) 4 | 5 | Feedback is welcome! Please let me know if any entries in the table are incorrect, or if there are any killer features from other parsers that are missing in this list. 6 | 7 | ## Feature Matrix 8 | 9 | [mpack]: https://github.com/ludocode/mpack 10 | [msgpack-c]: https://github.com/msgpack/msgpack-c 11 | [cmp]: https://github.com/camgunz/cmp 12 | [cwpack]: https://github.com/clwi/CWPack 13 | 14 | | | [MPack][mpack]
(v1.1) | [msgpack-c][msgpack-c]
(v3.3.0) | [CMP][cmp]
(v19) | [CWPack][cwpack]
(v1.3.1) | 15 | |:------------------------------------|:---:|:---:|:---:|:---:| 16 | | No libc requirement | ✓ | | ✓ | ✓ | 17 | | No allocator requirement | ✓ | | ✓ | ✓ | 18 | | Growable memory writer | ✓ | ✓ | | ✓ | 19 | | File I/O helpers | ✓ | ✓ | | ✓ | 20 | | Tree parser | ✓ | ✓ | | | 21 | | Stateful error handling | ✓ | | ✓ | ✓ | 22 | | Descriptive error information | | | | | 23 | | Compound size tracking | ✓ | | | | 24 | | Automatic compound size | ✓ | | | | 25 | | Incremental parser | ✓ | | ✓ | ✓ | 26 | | Typed read helpers | ✓ | | ✓ | | 27 | | Range/match read helpers | ✓ | | | | 28 | | Asynchronous incremental parser | | | | | 29 | | Peek next element | ✓ | | | | 30 | | Tree stream parser | ✓ | ✓ | | | 31 | | Asynchronous tree stream parser | ✓ | ✓ | | | 32 | | Support for new (2.0) spec | ✓ | ✓ | ✓ | ✓ | 33 | | Compatible with older (1.0) spec | ✓ | ✓ | ✓ | ✓ | 34 | | UTF-8 verification | ✓ | | | | 35 | | Type-generic write helpers | ✓ | ✓ | | | 36 | | Timestamps | ✓ | ✓ | | | 37 | 38 | Most of the features above are optional when supported and can be configured in all libraries. In particular, UTF-8 verification is optional with MPack; compound size tracking is optional and disabled in release by default with MPack; and 1.0 (v4) spec compatibility is optional in all libraries (v5/2.0 is the recommended and default usage.) 39 | 40 | ## Glossary 41 | 42 | *Tree parsing* means parsing a MessagePack object into a DOM-style tree of dynamically-typed elements supporting random access. 43 | 44 | *Incremental parsing* means being able to parse one basic MessagePack element at a time (either imperatively or with a SAX-style callback) with no per-element memory usage. 45 | 46 | *Stateful error handling* means a parse error or type error on one element places the whole parser, encoder or tree in an error state. This means you can check for errors only at certain key points rather than at every interaction with the library, and you get a final error state indicating whether any error occurred at any point during parsing or encoding. 47 | 48 | *Descriptive error information* means being able to get additional information when an error occurs, such as the tree position and byte position in the message where the error occurred. 49 | 50 | *Compound size tracking* means verifying that the same number of child elements or bytes were written or read as was declared at the start of a compound element. 51 | 52 | *Automatic compound size* means not having to specify the number of elements or bytes in an element up-front, instead determining it automatically when the element is closed. 53 | 54 | *Typed read helpers* means helper functions for a parser that can check the expected type of an element and return its value in that type. For example `cmp_read_int()` in CMP or `mpack_expect_u32()` in MPack. 55 | 56 | *Range/match read helpers* means helper functions for a parser that can check not only the expected type of an element, but also enforce an allowed range or exact match on a given value. 57 | 58 | *Peeking the next element* means being able to view the next element during incremental parsing without popping it from the data buffer. 59 | 60 | *Tree stream parsing* means the ability to parse a continuous stream of MessagePack with no external delimitation, emitting complete objects with a DOM-style tree API as they are found. The parser must be able to pause if the data is incomplete and continue parsing later when more data is available. This is necessary for implementing RPC over a socket with a tree parser (without needing to delimit messages by size.) 61 | 62 | *Asynchronous* means the parser is cooperative and re-entrant; when not enough data is available, it will return out of the parser with a "continue later" result rather than failing or blocking the read. There are two asynchronous entries in the above table, one for incremental parsing and one for tree parsing. 63 | 64 | *Compatible with older (1.0) spec* means the ability to produce messages compatible with parsers that only understand the old v4/1.0 version of MessagePack. A backwards-compatible encoder must at a minimum support writing an old-style raw without "str8", since there was no "raw8" type in old MessagePack. 65 | 66 | *Type-generic write helpers* means a generic write function or macro that can serialize based on the static type of the argument, in at least one of C11 or C++. (The reference [msgpack-c][msgpack-c] currently supports this only in C++ mode. MPack supports this both in C++ with templates and in C11 with `_Generic`.) 67 | -------------------------------------------------------------------------------- /test/unit/src/test-reader.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | * this software and associated documentation files (the "Software"), to deal in 6 | * the Software without restriction, including without limitation the rights to 7 | * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 8 | * the Software, and to permit persons to whom the Software is furnished to do so, 9 | * subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 16 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 17 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 18 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 19 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 20 | */ 21 | 22 | #ifndef MPACK_TEST_READER_H 23 | #define MPACK_TEST_READER_H 1 24 | 25 | #include "test.h" 26 | 27 | #ifdef __cplusplus 28 | extern "C" { 29 | #endif 30 | 31 | #if MPACK_READER 32 | 33 | extern mpack_error_t test_read_error; 34 | void test_read_error_handler(mpack_reader_t* reader, mpack_error_t error); 35 | 36 | 37 | // these setup and destroy test readers and check them for errors. 38 | // they are generally macros so that the checks are on the line of the test. 39 | 40 | 41 | // destroy wrappers expecting error or no error 42 | 43 | // tears down a reader, ensuring it has no errors and no extra data 44 | #define TEST_READER_DESTROY_NOERROR(reader) do { \ 45 | size_t remaining = mpack_reader_remaining(reader, NULL); \ 46 | mpack_error_t error = mpack_reader_destroy(reader); \ 47 | TEST_TRUE(error == mpack_ok, "reader is in error state %i (%s)", \ 48 | (int)error, mpack_error_to_string(error)); \ 49 | TEST_TRUE(remaining == 0, \ 50 | "reader has %i extra bytes", (int)remaining); \ 51 | } while (0) 52 | 53 | // tears down a reader, ensuring it is in the given error state 54 | #define TEST_READER_DESTROY_ERROR(reader, error) do { \ 55 | mpack_error_t expected = (error); \ 56 | mpack_error_t actual = mpack_reader_destroy(reader); \ 57 | TEST_TRUE(actual == expected, "reader is in error state %i (%s) instead of %i (%s)", \ 58 | (int)actual, mpack_error_to_string(actual), \ 59 | (int)expected, mpack_error_to_string(expected)); \ 60 | } while (0) 61 | 62 | 63 | 64 | // reader helpers 65 | 66 | // performs an operation on a reader, ensuring no error occurs 67 | #define TEST_READ_NOERROR(reader, read_expr) do { \ 68 | TEST_TRUE((read_expr), "read did not pass: " #read_expr); \ 69 | TEST_TRUE(mpack_reader_error(reader) == mpack_ok, \ 70 | "reader flagged error %i", (int)mpack_reader_error(reader)); \ 71 | } while (0) 72 | 73 | 74 | 75 | // simple read tests 76 | 77 | // initializes a reader from a literal string 78 | #define TEST_READER_INIT_STR(reader, data) \ 79 | mpack_reader_init_data(reader, data, sizeof(data) - 1) 80 | 81 | // runs a simple reader test, ensuring the expression is true and no errors occur 82 | #define TEST_SIMPLE_READ(data, read_expr) do { \ 83 | TEST_READER_INIT_STR(&reader, data); \ 84 | mpack_reader_set_error_handler(&reader, test_read_error_handler); \ 85 | TEST_TRUE((read_expr), "simple read test did not pass: " #read_expr); \ 86 | TEST_READER_DESTROY_NOERROR(&reader); \ 87 | TEST_TRUE(test_read_error == mpack_ok); \ 88 | test_read_error = mpack_ok; \ 89 | } while (0) 90 | 91 | // runs a simple reader test, ensuring the expression is true and no errors occur, cancelling to ignore tracking info 92 | #define TEST_SIMPLE_READ_CANCEL(data, read_expr) do { \ 93 | TEST_READER_INIT_STR(&reader, data); \ 94 | TEST_TRUE((read_expr), "simple read test did not pass: " #read_expr); \ 95 | mpack_reader_flag_error(&reader, mpack_error_data); \ 96 | TEST_READER_DESTROY_ERROR(&reader, mpack_error_data); \ 97 | } while (0) 98 | 99 | // runs a simple reader test, ensuring the expression is true and the given error is raised 100 | #define TEST_SIMPLE_READ_ERROR(data, read_expr, error) do { \ 101 | TEST_READER_INIT_STR(&reader, data); \ 102 | mpack_reader_set_error_handler(&reader, test_read_error_handler); \ 103 | TEST_TRUE((read_expr), "simple read error test did not pass: " #read_expr); \ 104 | TEST_READER_DESTROY_ERROR(&reader, (error)); \ 105 | TEST_TRUE(test_read_error == (error)); \ 106 | test_read_error = mpack_ok; \ 107 | } while (0) 108 | 109 | 110 | 111 | // simple read bug tests 112 | 113 | #if MPACK_DEBUG 114 | 115 | // runs a simple reader test, ensuring it causes an assert. 116 | // we flag mpack_error_data to cancel out of any tracking. 117 | // (note about volatile, see TEST_ASSERT()) 118 | #define TEST_SIMPLE_READ_ASSERT(data, read_expr) do { \ 119 | volatile mpack_reader_t v_reader; \ 120 | TEST_MPACK_SILENCE_SHADOW_BEGIN \ 121 | mpack_reader_t* reader = (mpack_reader_t*)(uintptr_t)&v_reader; \ 122 | TEST_MPACK_SILENCE_SHADOW_END \ 123 | mpack_reader_init_data(reader, data, sizeof(data) - 1); \ 124 | TEST_ASSERT(read_expr); \ 125 | mpack_reader_flag_error(reader, mpack_error_data); \ 126 | mpack_reader_destroy(reader); \ 127 | } while (0) 128 | 129 | #else 130 | 131 | // we cannot test asserts in release mode because they are 132 | // compiled away; code would continue to run and cause 133 | // undefined behavior. 134 | #define TEST_SIMPLE_READ_ASSERT(data, read_expr) ((void)0) 135 | 136 | #endif 137 | 138 | // runs a simple reader test, ensuring it causes a break in 139 | // debug mode and flags mpack_error_bug in both debug and release. 140 | #define TEST_SIMPLE_READ_BREAK(data, read_expr) do { \ 141 | mpack_reader_init_data(&reader, data, sizeof(data) - 1); \ 142 | TEST_BREAK(read_expr); \ 143 | TEST_READER_DESTROY_ERROR(&reader, mpack_error_bug); \ 144 | } while (0) 145 | 146 | 147 | 148 | void test_reader(void); 149 | 150 | #endif 151 | 152 | #ifdef __cplusplus 153 | } 154 | #endif 155 | 156 | #endif 157 | 158 | -------------------------------------------------------------------------------- /examples/sax-example.c: -------------------------------------------------------------------------------- 1 | #include "sax-example.h" 2 | 3 | static void parse_element(mpack_reader_t* reader, int depth, 4 | const sax_callbacks_t* callbacks, void* context); 5 | 6 | bool parse_messagepack(const char* data, size_t length, 7 | const sax_callbacks_t* callbacks, void* context) 8 | { 9 | mpack_reader_t reader; 10 | mpack_reader_init_data(&reader, data, length); 11 | parse_element(&reader, 0, callbacks, context); 12 | return mpack_ok == mpack_reader_destroy(&reader); 13 | } 14 | 15 | static void parse_element(mpack_reader_t* reader, int depth, 16 | const sax_callbacks_t* callbacks, void* context) 17 | { 18 | if (depth >= 32) { // critical check! 19 | mpack_reader_flag_error(reader, mpack_error_too_big); 20 | return; 21 | } 22 | 23 | mpack_tag_t tag = mpack_read_tag(reader); 24 | if (mpack_reader_error(reader) != mpack_ok) 25 | return; 26 | 27 | switch (mpack_tag_type(&tag)) { 28 | case mpack_type_nil: 29 | callbacks->nil_element(context, depth); 30 | break; 31 | case mpack_type_bool: 32 | callbacks->bool_element(context, depth, mpack_tag_bool_value(&tag)); 33 | break; 34 | case mpack_type_int: 35 | callbacks->int_element(context, depth, mpack_tag_int_value(&tag)); 36 | break; 37 | case mpack_type_uint: 38 | callbacks->uint_element(context, depth, mpack_tag_uint_value(&tag)); 39 | break; 40 | 41 | case mpack_type_str: { 42 | uint32_t length = mpack_tag_str_length(&tag); 43 | const char* data = mpack_read_bytes_inplace(reader, length); 44 | callbacks->string_element(context, depth, data, length); 45 | mpack_done_str(reader); 46 | break; 47 | } 48 | 49 | case mpack_type_bin: { 50 | uint32_t length = mpack_tag_bin_length(&tag); 51 | const char* data = mpack_read_bytes_inplace(reader, length); 52 | callbacks->bin_element(context, depth, data, length); 53 | mpack_done_bin(reader); 54 | break; 55 | } 56 | 57 | case mpack_type_array: { 58 | uint32_t count = mpack_tag_array_count(&tag); 59 | callbacks->start_array(context, depth, count); 60 | while (count-- > 0) { 61 | parse_element(reader, depth + 1, callbacks, context); 62 | if (mpack_reader_error(reader) != mpack_ok) // critical check! 63 | break; 64 | } 65 | callbacks->finish_array(context, depth); 66 | mpack_done_array(reader); 67 | break; 68 | } 69 | 70 | case mpack_type_map: { 71 | uint32_t count = mpack_tag_map_count(&tag); 72 | callbacks->start_map(context, depth, count); 73 | while (count-- > 0) { 74 | parse_element(reader, depth + 1, callbacks, context); 75 | parse_element(reader, depth + 1, callbacks, context); 76 | if (mpack_reader_error(reader) != mpack_ok) // critical check! 77 | break; 78 | } 79 | callbacks->finish_map(context, depth); 80 | mpack_done_map(reader); 81 | break; 82 | } 83 | 84 | default: 85 | fprintf(stderr, "Error: type %s not implemented by this example SAX parser.\n", 86 | mpack_type_to_string(mpack_tag_type(&tag))); 87 | exit(1); 88 | } 89 | } 90 | 91 | #define SAX_EXAMPLE_TEST 1 92 | #if SAX_EXAMPLE_TEST 93 | #include 94 | #include 95 | #include 96 | #include 97 | #include 98 | #include 99 | 100 | static void indent(int depth) { 101 | const char ruler[] = " "; 102 | printf("%*.*s", depth * 4, depth * 4, ruler); 103 | } 104 | 105 | static void nil_element(void* context, int depth) { 106 | indent(depth); 107 | printf("nil\n"); 108 | } 109 | 110 | static void bool_element(void* context, int depth, int64_t value) { 111 | indent(depth); 112 | printf("bool: %s\n", value ? "true" : "false"); 113 | } 114 | 115 | static void int_element(void* context, int depth, int64_t value) { 116 | indent(depth); 117 | printf("int: %" PRIi64 "\n", value); 118 | } 119 | 120 | static void uint_element(void* context, int depth, uint64_t value) { 121 | indent(depth); 122 | printf("uint: %" PRIu64 "\n", value); 123 | } 124 | 125 | static void string_element(void* context, int depth, 126 | const char* data, uint32_t length) { 127 | indent(depth); 128 | printf("string: \""); 129 | fwrite(data, 1, length, stdout); 130 | printf("\"\n"); 131 | } 132 | 133 | static void bin_element(void* context, int depth, 134 | const char* data, uint32_t length) { 135 | uint32_t i; 136 | indent(depth); 137 | printf("bin: \""); 138 | for (i = 0; i < length; ++i) { 139 | printf("%2.2x", data[i] & 0xff); 140 | } 141 | printf("\"\n"); 142 | } 143 | 144 | static void start_map(void* context, int depth, uint32_t pair_count) { 145 | indent(depth); 146 | printf("starting map of %u key-value pairs\n", pair_count); 147 | } 148 | 149 | static void start_array(void* context, int depth, uint32_t element_count) { 150 | indent(depth); 151 | printf("starting array of %u key-value pairs\n", element_count); 152 | } 153 | 154 | static void finish_map(void* context, int depth) { 155 | indent(depth); 156 | printf("finishing map\n"); 157 | } 158 | 159 | static void finish_array(void* context, int depth) { 160 | indent(depth); 161 | printf("finishing array\n"); 162 | } 163 | 164 | static sax_callbacks_t callbacks = { 165 | nil_element, 166 | bool_element, 167 | int_element, 168 | uint_element, 169 | string_element, 170 | bin_element, 171 | start_map, 172 | start_array, 173 | finish_map, 174 | finish_array, 175 | }; 176 | 177 | 178 | int main(int argc, char** argv) { 179 | if (argc != 2) { 180 | fprintf(stderr, "First argument must be path to MessagePack file."); 181 | exit(1); 182 | } 183 | struct stat stat; 184 | int fd = open(argv[1], O_RDONLY); 185 | fstat(fd, &stat); 186 | const char* p = (const char*) mmap(0, stat.st_size, PROT_READ, MAP_SHARED, fd, 0); 187 | bool ok = parse_messagepack(p, stat.st_size, &callbacks, NULL); 188 | if (!ok) 189 | fprintf(stderr, "Parse failed!\n"); 190 | return ok ? EXIT_SUCCESS : EXIT_FAILURE; 191 | } 192 | #endif 193 | -------------------------------------------------------------------------------- /test/fuzz/fuzz.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018-2021 Nicholas Fraser and the MPack authors 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | * this software and associated documentation files (the "Software"), to deal in 6 | * the Software without restriction, including without limitation the rights to 7 | * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 8 | * the Software, and to permit persons to whom the Software is furnished to do so, 9 | * subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 16 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 17 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 18 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 19 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 20 | */ 21 | 22 | #ifdef MPACK_FUZZ 23 | 24 | /* 25 | * fuzz.c is a test program to assist with fuzzing MPack. It: 26 | * 27 | * - decodes stdin with the dynamic Reader API; 28 | * - encodes the data to a growable buffer with the Write API; 29 | * - parses the resulting buffer with the Node API; 30 | * - and finally, prints a debug dump of the node tree to stdout. 31 | * 32 | * It thus passes all data through three major components of MPack (but not 33 | * the Expect API.) 34 | */ 35 | 36 | #include "mpack/mpack.h" 37 | 38 | #ifndef MPACK_FUZZ_CONFIG_H 39 | #error "This should be built with fuzz-config.h as a prefix header." 40 | #endif 41 | 42 | static void print_callback(void* context, const char* data, size_t count) { 43 | fwrite(data, 1, count, stdout); 44 | } 45 | 46 | static void transfer_bytes(mpack_reader_t* reader, mpack_writer_t* writer, uint32_t count) { 47 | if (mpack_should_read_bytes_inplace(reader, count)) { 48 | const char* data = mpack_read_bytes_inplace(reader, count); 49 | if (mpack_reader_error(reader) == mpack_ok) 50 | mpack_write_bytes(writer, data, count); 51 | return; 52 | } 53 | 54 | while (count > 0) { 55 | char buffer[79]; 56 | uint32_t step = (count < sizeof(buffer)) ? count : sizeof(buffer); 57 | mpack_read_bytes(reader, buffer, step); 58 | if (mpack_reader_error(reader) != mpack_ok) 59 | return; 60 | mpack_write_bytes(writer, buffer, step); 61 | count -= step; 62 | } 63 | } 64 | 65 | static void transfer_element(mpack_reader_t* reader, mpack_writer_t* writer, int depth) { 66 | 67 | // We apply a depth limit manually right now to avoid a stack overflow. A 68 | // depth limit should probably be added to the reader and tree at some 69 | // point because even though the reader and tree can themselves handle 70 | // arbitrary depths, any dynamic use that doesn't account for this is 71 | // likely to be vulnerable to such stack overflows. 72 | if (depth >= 1024) { 73 | fprintf(stderr, "hit depth limit!\n"); 74 | mpack_reader_flag_error(reader, mpack_error_too_big); 75 | return; 76 | } 77 | ++depth; 78 | 79 | mpack_tag_t tag = mpack_read_tag(reader); 80 | if (mpack_reader_error(reader) != mpack_ok) { 81 | fprintf(stderr, "error reading tag!\n"); 82 | return; 83 | } 84 | 85 | /* 86 | static char describe_buffer[64]; 87 | mpack_tag_debug_describe(tag, describe_buffer, sizeof(describe_buffer)); 88 | printf("%s\n", describe_buffer); 89 | */ 90 | 91 | mpack_write_tag(writer, tag); 92 | 93 | switch (tag.type) { 94 | #if MPACK_EXTENSIONS 95 | case mpack_type_ext: // fallthrough 96 | #endif 97 | case mpack_type_str: // fallthrough 98 | case mpack_type_bin: 99 | transfer_bytes(reader, writer, mpack_tag_bytes(&tag)); 100 | if (mpack_reader_error(reader) != mpack_ok) 101 | return; 102 | mpack_done_type(reader, tag.type); 103 | mpack_finish_type(writer, tag.type); 104 | break; 105 | 106 | case mpack_type_map: 107 | for (uint32_t i = 0; i < mpack_tag_map_count(&tag); ++i) { 108 | transfer_element(reader, writer, depth); 109 | if (mpack_reader_error(reader) != mpack_ok) 110 | return; 111 | transfer_element(reader, writer, depth); 112 | if (mpack_reader_error(reader) != mpack_ok) 113 | return; 114 | } 115 | mpack_done_map(reader); 116 | mpack_finish_map(writer); 117 | break; 118 | 119 | case mpack_type_array: 120 | for (uint32_t i = 0; i < mpack_tag_array_count(&tag); ++i) { 121 | transfer_element(reader, writer, depth); 122 | if (mpack_reader_error(reader) != mpack_ok) 123 | return; 124 | } 125 | mpack_done_array(reader); 126 | mpack_finish_array(writer); 127 | break; 128 | 129 | default: 130 | break; 131 | } 132 | } 133 | 134 | int main(int argc, char** argv) { 135 | char* data; 136 | size_t size; 137 | mpack_writer_t writer; 138 | mpack_writer_init_growable(&writer, &data, &size); 139 | 140 | mpack_reader_t reader; 141 | mpack_reader_init_stdfile(&reader, stdin, false); 142 | 143 | transfer_element(&reader, &writer, 0); 144 | 145 | if (mpack_reader_destroy(&reader) != mpack_ok || mpack_writer_destroy(&writer) != mpack_ok) { 146 | fprintf(stderr, "error in reader or writer!\n"); 147 | return EXIT_FAILURE; 148 | } 149 | 150 | mpack_tree_t tree; 151 | mpack_tree_init_stdfile(&tree, stdin, 0, false); 152 | mpack_tree_parse(&tree); 153 | if (mpack_tree_error(&tree) != mpack_ok) { 154 | fprintf(stderr, "error parsing tree!\n"); 155 | return EXIT_FAILURE; 156 | } 157 | 158 | mpack_node_print_to_callback(mpack_tree_root(&tree), print_callback, NULL); 159 | 160 | if (mpack_tree_destroy(&tree) != mpack_ok) { 161 | fprintf(stderr, "error printing or destroying tree!\n"); 162 | return EXIT_FAILURE; 163 | } 164 | 165 | return EXIT_SUCCESS; 166 | } 167 | 168 | #else 169 | typedef int mpack_pedantic_allow_empty_translation_unit; 170 | #endif 171 | -------------------------------------------------------------------------------- /src/mpack/mpack-platform.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | * this software and associated documentation files (the "Software"), to deal in 6 | * the Software without restriction, including without limitation the rights to 7 | * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 8 | * the Software, and to permit persons to whom the Software is furnished to do so, 9 | * subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 16 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 17 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 18 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 19 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 20 | */ 21 | 22 | 23 | // We define MPACK_EMIT_INLINE_DEFS and include mpack.h to emit 24 | // standalone definitions of all (non-static) inline functions in MPack. 25 | 26 | #define MPACK_INTERNAL 1 27 | #define MPACK_EMIT_INLINE_DEFS 1 28 | 29 | #include "mpack-platform.h" 30 | #include "mpack.h" 31 | 32 | MPACK_SILENCE_WARNINGS_BEGIN 33 | 34 | #if MPACK_DEBUG 35 | 36 | #if MPACK_STDIO 37 | void mpack_assert_fail_format(const char* format, ...) { 38 | char buffer[512]; 39 | va_list args; 40 | va_start(args, format); 41 | vsnprintf(buffer, sizeof(buffer), format, args); 42 | va_end(args); 43 | buffer[sizeof(buffer) - 1] = 0; 44 | mpack_assert_fail_wrapper(buffer); 45 | } 46 | 47 | void mpack_break_hit_format(const char* format, ...) { 48 | char buffer[512]; 49 | va_list args; 50 | va_start(args, format); 51 | vsnprintf(buffer, sizeof(buffer), format, args); 52 | va_end(args); 53 | buffer[sizeof(buffer) - 1] = 0; 54 | mpack_break_hit(buffer); 55 | } 56 | #endif 57 | 58 | #if !MPACK_CUSTOM_ASSERT 59 | void mpack_assert_fail(const char* message) { 60 | MPACK_UNUSED(message); 61 | 62 | #if MPACK_STDIO 63 | fprintf(stderr, "%s\n", message); 64 | #endif 65 | } 66 | #endif 67 | 68 | // We split the assert failure from the wrapper so that a 69 | // custom assert function can return. 70 | void mpack_assert_fail_wrapper(const char* message) { 71 | 72 | #ifdef MPACK_GCOV 73 | // gcov marks even __builtin_unreachable() as an uncovered line. this 74 | // silences it. 75 | (mpack_assert_fail(message), __builtin_unreachable()); 76 | 77 | #else 78 | mpack_assert_fail(message); 79 | 80 | // mpack_assert_fail() is not supposed to return. in case it does, we 81 | // abort. 82 | 83 | #if !MPACK_NO_BUILTINS 84 | #if defined(__GNUC__) || defined(__clang__) 85 | __builtin_trap(); 86 | #elif defined(WIN32) 87 | __debugbreak(); 88 | #endif 89 | #endif 90 | 91 | #if (defined(__GNUC__) || defined(__clang__)) && !MPACK_NO_BUILTINS 92 | __builtin_abort(); 93 | #elif MPACK_STDLIB 94 | abort(); 95 | #endif 96 | 97 | MPACK_UNREACHABLE; 98 | #endif 99 | } 100 | 101 | #if !MPACK_CUSTOM_BREAK 102 | 103 | // If we have a custom assert handler, break wraps it by default. 104 | // This allows users of MPack to only implement mpack_assert_fail() without 105 | // having to worry about the difference between assert and break. 106 | // 107 | // MPACK_CUSTOM_BREAK is available to define a separate break handler 108 | // (which is needed by the unit test suite), but this is not offered in 109 | // mpack-config.h for simplicity. 110 | 111 | #if MPACK_CUSTOM_ASSERT 112 | void mpack_break_hit(const char* message) { 113 | mpack_assert_fail_wrapper(message); 114 | } 115 | #else 116 | void mpack_break_hit(const char* message) { 117 | MPACK_UNUSED(message); 118 | 119 | #if MPACK_STDIO 120 | fprintf(stderr, "%s\n", message); 121 | #endif 122 | 123 | #if defined(__GNUC__) || defined(__clang__) && !MPACK_NO_BUILTINS 124 | __builtin_trap(); 125 | #elif defined(WIN32) && !MPACK_NO_BUILTINS 126 | __debugbreak(); 127 | #elif MPACK_STDLIB 128 | abort(); 129 | #endif 130 | } 131 | #endif 132 | 133 | #endif 134 | 135 | #endif 136 | 137 | 138 | 139 | // The below are adapted from the C wikibook: 140 | // https://en.wikibooks.org/wiki/C_Programming/Strings 141 | 142 | #ifndef mpack_memcmp 143 | int mpack_memcmp(const void* s1, const void* s2, size_t n) { 144 | const unsigned char *us1 = (const unsigned char *) s1; 145 | const unsigned char *us2 = (const unsigned char *) s2; 146 | while (n-- != 0) { 147 | if (*us1 != *us2) 148 | return (*us1 < *us2) ? -1 : +1; 149 | us1++; 150 | us2++; 151 | } 152 | return 0; 153 | } 154 | #endif 155 | 156 | #ifndef mpack_memcpy 157 | void* mpack_memcpy(void* MPACK_RESTRICT s1, const void* MPACK_RESTRICT s2, size_t n) { 158 | char* MPACK_RESTRICT dst = (char *)s1; 159 | const char* MPACK_RESTRICT src = (const char *)s2; 160 | while (n-- != 0) 161 | *dst++ = *src++; 162 | return s1; 163 | } 164 | #endif 165 | 166 | #ifndef mpack_memmove 167 | void* mpack_memmove(void* s1, const void* s2, size_t n) { 168 | char *p1 = (char *)s1; 169 | const char *p2 = (const char *)s2; 170 | if (p2 < p1 && p1 < p2 + n) { 171 | p2 += n; 172 | p1 += n; 173 | while (n-- != 0) 174 | *--p1 = *--p2; 175 | } else 176 | while (n-- != 0) 177 | *p1++ = *p2++; 178 | return s1; 179 | } 180 | #endif 181 | 182 | #ifndef mpack_memset 183 | void* mpack_memset(void* s, int c, size_t n) { 184 | unsigned char *us = (unsigned char *)s; 185 | unsigned char uc = (unsigned char)c; 186 | while (n-- != 0) 187 | *us++ = uc; 188 | return s; 189 | } 190 | #endif 191 | 192 | #ifndef mpack_strlen 193 | size_t mpack_strlen(const char* s) { 194 | const char* p = s; 195 | while (*p != '\0') 196 | p++; 197 | return (size_t)(p - s); 198 | } 199 | #endif 200 | 201 | 202 | 203 | #if defined(MPACK_MALLOC) && !defined(MPACK_REALLOC) 204 | void* mpack_realloc(void* old_ptr, size_t used_size, size_t new_size) { 205 | if (new_size == 0) { 206 | if (old_ptr) 207 | MPACK_FREE(old_ptr); 208 | return NULL; 209 | } 210 | 211 | void* new_ptr = MPACK_MALLOC(new_size); 212 | if (new_ptr == NULL) 213 | return NULL; 214 | 215 | mpack_memcpy(new_ptr, old_ptr, used_size); 216 | MPACK_FREE(old_ptr); 217 | return new_ptr; 218 | } 219 | #endif 220 | 221 | MPACK_SILENCE_WARNINGS_END 222 | -------------------------------------------------------------------------------- /test/unit/src/test-reader.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | * this software and associated documentation files (the "Software"), to deal in 6 | * the Software without restriction, including without limitation the rights to 7 | * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 8 | * the Software, and to permit persons to whom the Software is furnished to do so, 9 | * subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 16 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 17 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 18 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 19 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 20 | */ 21 | 22 | #include "test-expect.h" 23 | 24 | #if MPACK_READER 25 | 26 | mpack_error_t test_read_error = mpack_ok; 27 | 28 | void test_read_error_handler(mpack_reader_t* reader, mpack_error_t error) { 29 | TEST_TRUE(test_read_error == mpack_ok, "error handler was called multiple times"); 30 | TEST_TRUE(error != mpack_ok, "error handler was called with mpack_ok"); 31 | TEST_TRUE(mpack_reader_error(reader) == error, "reader error does not match given error"); 32 | test_read_error = error; 33 | } 34 | 35 | // almost all reader functions are tested by the expect tests. 36 | // minor miscellaneous read tests are added here. 37 | 38 | static void test_reader_should_inplace(void) { 39 | static char buf[4096]; 40 | mpack_reader_t reader; 41 | mpack_reader_init(&reader, buf, sizeof(buf), 0); 42 | 43 | TEST_TRUE(true == mpack_should_read_bytes_inplace(&reader, 0)); 44 | TEST_TRUE(true == mpack_should_read_bytes_inplace(&reader, 1)); 45 | TEST_TRUE(true == mpack_should_read_bytes_inplace(&reader, 20)); 46 | TEST_TRUE(false == mpack_should_read_bytes_inplace(&reader, 500)); 47 | TEST_TRUE(false == mpack_should_read_bytes_inplace(&reader, 10000)); 48 | 49 | mpack_reader_destroy(&reader); 50 | } 51 | 52 | static void test_reader_miscellaneous(void) { 53 | mpack_reader_t reader; 54 | 55 | // 0xc1 is reserved; it should always raise mpack_error_invalid 56 | TEST_SIMPLE_READ_ERROR("\xc1", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_invalid); 57 | 58 | #if !MPACK_EXTENSIONS 59 | // ext types are unsupported without MPACK_EXTENSIONS 60 | TEST_SIMPLE_READ_ERROR("\xc7", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_unsupported); 61 | TEST_SIMPLE_READ_ERROR("\xc8", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_unsupported); 62 | TEST_SIMPLE_READ_ERROR("\xc9", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_unsupported); 63 | TEST_SIMPLE_READ_ERROR("\xd4", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_unsupported); 64 | TEST_SIMPLE_READ_ERROR("\xd5", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_unsupported); 65 | TEST_SIMPLE_READ_ERROR("\xd6", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_unsupported); 66 | TEST_SIMPLE_READ_ERROR("\xd7", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_unsupported); 67 | TEST_SIMPLE_READ_ERROR("\xd8", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_unsupported); 68 | #endif 69 | 70 | // simple truncated tags (testing discard of additional 71 | // temporary data in mpack_parse_tag()) 72 | TEST_SIMPLE_READ_ERROR("\xcc", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_invalid); 73 | TEST_SIMPLE_READ_ERROR("\xcd", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_invalid); 74 | TEST_SIMPLE_READ_ERROR("\xce", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_invalid); 75 | TEST_SIMPLE_READ_ERROR("\xcf", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_invalid); 76 | TEST_SIMPLE_READ_ERROR("\xd0", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_invalid); 77 | TEST_SIMPLE_READ_ERROR("\xd1", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_invalid); 78 | TEST_SIMPLE_READ_ERROR("\xd2", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_invalid); 79 | TEST_SIMPLE_READ_ERROR("\xd3", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_invalid); 80 | 81 | // truncated discard errors 82 | TEST_SIMPLE_READ_ERROR("\x91", (mpack_discard(&reader), true), mpack_error_invalid); // array 83 | TEST_SIMPLE_READ_ERROR("\x81", (mpack_discard(&reader), true), mpack_error_invalid); // map 84 | } 85 | 86 | #if MPACK_DEBUG && MPACK_STDIO 87 | static void test_print_buffer(void) { 88 | static const char test[] = "\x82\xA7""compact\xC3\xA6""schema\x00"; 89 | 90 | char buffer[1024]; 91 | mpack_print_data_to_buffer(test, sizeof(test) - 1, buffer, sizeof(buffer)); 92 | 93 | static const char* expected = 94 | "{\n" 95 | " \"compact\": true,\n" 96 | " \"schema\": 0\n" 97 | "}"; 98 | TEST_TRUE(buffer[strlen(expected)] == 0); 99 | TEST_TRUE(0 == strcmp(buffer, expected)); 100 | } 101 | 102 | static void test_print_buffer_bounds(void) { 103 | static const char test[] = "\x82\xA7""compact\xC3\xA6""schema\x00"; 104 | 105 | char buffer[10]; 106 | mpack_print_data_to_buffer(test, sizeof(test) - 1, buffer, sizeof(buffer)); 107 | 108 | // string should be truncated with a null-terminator 109 | static const char* expected = "{\n \"co"; 110 | TEST_TRUE(buffer[strlen(expected)] == 0); 111 | TEST_TRUE(0 == strcmp(buffer, expected)); 112 | } 113 | 114 | static void test_print_buffer_hexdump(void) { 115 | static const char test[] = "\xc4\x03""abc"; 116 | 117 | char buffer[64]; 118 | mpack_print_data_to_buffer(test, sizeof(test) - 1, buffer, sizeof(buffer)); 119 | 120 | // string should be truncated with a null-terminator 121 | static const char* expected = ""; 122 | TEST_TRUE(buffer[strlen(expected)] == 0); 123 | TEST_TRUE(0 == strcmp(buffer, expected)); 124 | } 125 | 126 | static void test_print_buffer_no_hexdump(void) { 127 | static const char test[] = "\xc4\x00"; 128 | 129 | char buffer[64]; 130 | mpack_print_data_to_buffer(test, sizeof(test) - 1, buffer, sizeof(buffer)); 131 | 132 | // string should be truncated with a null-terminator 133 | static const char* expected = ""; 134 | TEST_TRUE(buffer[strlen(expected)] == 0); 135 | TEST_TRUE(0 == strcmp(buffer, expected)); 136 | } 137 | #endif 138 | 139 | static bool count_messages(const void* buffer, size_t byte_count, size_t* message_count) { 140 | mpack_reader_t reader; 141 | mpack_reader_init_data(&reader, (const char*)buffer, byte_count); 142 | 143 | *message_count = 0; 144 | while (mpack_reader_remaining(&reader, NULL) > 0) { 145 | ++*message_count; 146 | mpack_discard(&reader); 147 | } 148 | 149 | return mpack_ok == mpack_reader_destroy(&reader); 150 | } 151 | 152 | static void test_count_messages(void) { 153 | static const char test[] = "\x80\x81\xA3""key\xA5""value\x92\xc2\xc3\x90"; 154 | size_t message_count; 155 | TEST_TRUE(count_messages(test, sizeof(test)-1, &message_count)); 156 | TEST_TRUE(message_count == 4); 157 | 158 | static const char test2[] = "\x92\xc0"; // truncated buffer 159 | TEST_TRUE(!count_messages(test2, sizeof(test2)-1, &message_count)); 160 | } 161 | 162 | void test_reader() { 163 | #if MPACK_DEBUG && MPACK_STDIO 164 | test_print_buffer(); 165 | test_print_buffer_bounds(); 166 | test_print_buffer_hexdump(); 167 | test_print_buffer_no_hexdump(); 168 | #endif 169 | test_reader_should_inplace(); 170 | test_reader_miscellaneous(); 171 | test_count_messages(); 172 | } 173 | 174 | #endif 175 | 176 | -------------------------------------------------------------------------------- /test/unit/src/test-system.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | * this software and associated documentation files (the "Software"), to deal in 6 | * the Software without restriction, including without limitation the rights to 7 | * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 8 | * the Software, and to permit persons to whom the Software is furnished to do so, 9 | * subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 16 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 17 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 18 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 19 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 20 | */ 21 | 22 | #ifdef _WIN32 23 | #define _CRT_SECURE_NO_WARNINGS 24 | #endif 25 | 26 | // We need to include test.h here instead of test-system.h because 27 | // test-system.h is included within the mpack-config.h. 28 | #include "test.h" 29 | 30 | #if MPACK_STDIO 31 | #include 32 | #endif 33 | 34 | static bool test_system_fail = false; 35 | static bool test_system_fail_all = false; 36 | static size_t test_system_left = 0; 37 | static const int test_system_fail_until_max = 500; 38 | 39 | void test_system_fail_after(size_t count, bool all) { 40 | test_system_fail = true; 41 | test_system_left = count; 42 | test_system_fail_all = all; 43 | } 44 | 45 | void test_system_fail_reset(void) { 46 | test_system_fail = false; 47 | } 48 | 49 | #if defined(MPACK_MALLOC) || MPACK_STDIO 50 | static bool test_system_should_fail(void) { 51 | if (!test_system_fail) 52 | return false; 53 | if (test_system_left == 0) { 54 | if (!test_system_fail_all) 55 | test_system_fail = false; 56 | return true; 57 | } 58 | --test_system_left; 59 | return false; 60 | } 61 | #endif 62 | 63 | void test_system_fail_until_ok(bool (*test)(void)) { 64 | #ifdef MPACK_MALLOC 65 | TEST_TRUE(test_malloc_active_count() == 0, "allocations exist before starting failure test"); 66 | #endif 67 | #if MPACK_STDIO 68 | TEST_TRUE(test_files_count() == 0, "files are open before starting failure test"); 69 | #endif 70 | 71 | int i; 72 | for (i = 0; i < test_system_fail_until_max; ++i) { 73 | test_system_fail_after((size_t)i, false); 74 | bool ok = test(); 75 | 76 | #ifdef MPACK_MALLOC 77 | TEST_TRUE(test_malloc_active_count() == 0, "test single failure leaked memory on iteration %i!", i); 78 | #endif 79 | #if MPACK_STDIO 80 | TEST_TRUE(test_files_count() == 0, "test single failure leaked file on iteration %i!", i); 81 | #endif 82 | 83 | test_system_fail_after((size_t)i, true); 84 | ok &= test(); 85 | 86 | #ifdef MPACK_MALLOC 87 | TEST_TRUE(test_malloc_active_count() == 0, "test all failures leaked memory on iteration %i!", i); 88 | #endif 89 | #if MPACK_STDIO 90 | TEST_TRUE(test_files_count() == 0, "test all failures leaked file on iteration %i!", i); 91 | #endif 92 | 93 | if (ok) { 94 | test_system_fail_reset(); 95 | return; 96 | } 97 | } 98 | 99 | TEST_TRUE(false, "hit maximum number of system calls in a system fail test"); 100 | test_system_fail_reset(); 101 | } 102 | 103 | void test_system(void) { 104 | #ifdef MPACK_MALLOC 105 | TEST_TRUE(test_malloc_active_count() == 0); 106 | TEST_TRUE(NULL == mpack_realloc(NULL, 0, 0)); 107 | void* p = MPACK_MALLOC(1); 108 | TEST_TRUE(NULL == mpack_realloc(p, 1, 0)); 109 | TEST_TRUE(test_malloc_active_count() == 0, "realloc leaked"); 110 | #endif 111 | } 112 | 113 | 114 | 115 | #ifdef MPACK_MALLOC 116 | static size_t test_malloc_active = 0; 117 | static size_t test_malloc_total = 0; 118 | 119 | size_t test_malloc_active_count(void) { 120 | return test_malloc_active; 121 | } 122 | 123 | size_t test_malloc_total_count(void) { 124 | return test_malloc_total; 125 | } 126 | 127 | void* test_malloc(size_t size) { 128 | TEST_TRUE(size != 0, "cannot allocate zero bytes!"); 129 | if (size == 0) 130 | return NULL; 131 | 132 | if (test_system_should_fail()) 133 | return NULL; 134 | 135 | ++test_malloc_total; 136 | ++test_malloc_active; 137 | return malloc(size); 138 | } 139 | 140 | void* test_realloc(void* p, size_t size) { 141 | if (size == 0) { 142 | if (p) { 143 | free(p); 144 | --test_malloc_active; 145 | } 146 | return NULL; 147 | } 148 | 149 | if (test_system_should_fail()) 150 | return NULL; 151 | 152 | ++test_malloc_total; 153 | if (!p) 154 | ++test_malloc_active; 155 | return realloc(p, size); 156 | } 157 | 158 | void test_free(void* p) { 159 | // while free() is supposed to allow NULL, not all custom allocators 160 | // may handle this, so we don't free NULL. 161 | TEST_TRUE(p != NULL, "attempting to free NULL"); 162 | 163 | if (p) 164 | --test_malloc_active; 165 | free(p); 166 | } 167 | 168 | #endif 169 | 170 | 171 | 172 | #if defined(MPACK_STDLIB) && MPACK_STDLIB 173 | static bool test_system_mock_strlen_enabled = false; 174 | static size_t test_system_mock_strlen_value; 175 | 176 | void test_system_mock_strlen(size_t len) { 177 | test_system_mock_strlen_enabled = true; 178 | test_system_mock_strlen_value = len; 179 | } 180 | 181 | size_t test_strlen(const char* s) { 182 | if (test_system_mock_strlen_enabled) { 183 | test_system_mock_strlen_enabled = false; 184 | return test_system_mock_strlen_value; 185 | } 186 | return strlen(s); 187 | } 188 | #endif 189 | 190 | 191 | 192 | #if MPACK_STDIO 193 | #undef fopen 194 | #undef fclose 195 | #undef fread 196 | #undef fwrite 197 | #undef fseek 198 | #undef ftell 199 | #undef ferror 200 | 201 | static size_t test_files_active = 0; 202 | 203 | size_t test_files_count(void) { 204 | return test_files_active; 205 | } 206 | 207 | FILE* test_fopen(const char* path, const char* mode) { 208 | if (test_system_should_fail()) { 209 | errno = EACCES; 210 | return NULL; 211 | } 212 | 213 | FILE* file = fopen(path, mode); 214 | if (file) 215 | ++test_files_active; 216 | return file; 217 | } 218 | 219 | int test_fclose(FILE* stream) { 220 | TEST_TRUE(stream != NULL); 221 | 222 | --test_files_active; 223 | 224 | // if we're simulating failure, we still close the file 225 | // anyway to avoid leaking any files 226 | int ret = fclose(stream); 227 | 228 | if (test_system_should_fail()) { 229 | errno = EACCES; 230 | return EOF; 231 | } 232 | 233 | return ret; 234 | } 235 | 236 | size_t test_fread(void* ptr, size_t size, size_t nmemb, FILE* stream) { 237 | TEST_TRUE(stream != NULL); 238 | 239 | if (test_system_should_fail()) { 240 | errno = EACCES; 241 | return 0; 242 | } 243 | 244 | return fread(ptr, size, nmemb, stream); 245 | } 246 | 247 | size_t test_fwrite(const void* ptr, size_t size, size_t nmemb, FILE* stream) { 248 | TEST_TRUE(stream != NULL); 249 | 250 | if (test_system_should_fail()) { 251 | errno = EACCES; 252 | return 0; 253 | } 254 | 255 | return fwrite(ptr, size, nmemb, stream); 256 | } 257 | 258 | int test_fseek(FILE* stream, long offset, int whence) { 259 | TEST_TRUE(stream != NULL); 260 | 261 | if (test_system_should_fail()) { 262 | errno = EACCES; 263 | return -1; 264 | } 265 | 266 | return fseek(stream, offset, whence); 267 | } 268 | 269 | long test_ftell(FILE* stream) { 270 | TEST_TRUE(stream != NULL); 271 | 272 | if (test_system_should_fail()) { 273 | errno = EACCES; 274 | return -1; 275 | } 276 | 277 | return ftell(stream); 278 | } 279 | 280 | int test_ferror(FILE * stream) { 281 | TEST_TRUE(stream != NULL); 282 | 283 | if (test_system_should_fail()) { 284 | errno = EACCES; 285 | return -1; 286 | } 287 | 288 | return ferror(stream); 289 | } 290 | #endif 291 | 292 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Introduction 2 | 3 | MPack is a C implementation of an encoder and decoder for the [MessagePack](http://msgpack.org/) serialization format. It is: 4 | 5 | * Simple and easy to use 6 | * Secure against untrusted data 7 | * Lightweight, suitable for embedded 8 | * [Extensively documented](http://ludocode.github.io/mpack/) 9 | * [Extremely fast](https://github.com/ludocode/schemaless-benchmarks#speed---desktop-pc) 10 | 11 | The core of MPack contains a buffered reader and writer, and a tree-style parser that decodes into a tree of dynamically typed nodes. Helper functions can be enabled to read values of expected type, to work with files, to grow buffers or allocate strings automatically, to check UTF-8 encoding, and more. 12 | 13 | The MPack code is small enough to be embedded directly into your codebase. Simply download the [amalgamation package](https://github.com/ludocode/mpack/releases) and add `mpack.h` and `mpack.c` to your project. 14 | 15 | MPack supports all modern compilers, all desktop and smartphone OSes, WebAssembly, [inside the Linux kernel](https://github.com/ludocode/mpack-linux-kernel), and even 8-bit microcontrollers such as Arduino. The MPack featureset can be customized at compile-time to set which features, components and debug checks are compiled, and what dependencies are available. 16 | 17 | ## Build Status 18 | 19 | [![Unit Tests](https://github.com/ludocode/mpack/workflows/Unit%20Tests/badge.svg)](https://github.com/ludocode/mpack/actions?query=workflow%3A%22Unit+Tests%22) 20 | [![Coverage](https://coveralls.io/repos/ludocode/mpack/badge.svg?branch=develop&service=github)](https://coveralls.io/github/ludocode/mpack?branch=develop) 21 | 22 | ## The Node API 23 | 24 | The Node API parses a chunk of MessagePack data into an immutable tree of dynamically-typed nodes. A series of helper functions can be used to extract data of specific types from each node. 25 | 26 | ```C 27 | // parse a file into a node tree 28 | mpack_tree_t tree; 29 | mpack_tree_init_filename(&tree, "homepage-example.mp", 0); 30 | mpack_tree_parse(&tree); 31 | mpack_node_t root = mpack_tree_root(&tree); 32 | 33 | // extract the example data on the msgpack homepage 34 | bool compact = mpack_node_bool(mpack_node_map_cstr(root, "compact")); 35 | int schema = mpack_node_i32(mpack_node_map_cstr(root, "schema")); 36 | 37 | // clean up and check for errors 38 | if (mpack_tree_destroy(&tree) != mpack_ok) { 39 | fprintf(stderr, "An error occurred decoding the data!\n"); 40 | return; 41 | } 42 | ``` 43 | 44 | Note that no additional error handling is needed in the above code. If the file is missing or corrupt, if map keys are missing or if nodes are not in the expected types, special "nil" nodes and false/zero values are returned and the tree is placed in an error state. An error check is only needed before using the data. 45 | 46 | The above example allocates nodes automatically. A fixed node pool can be provided to the parser instead in memory-constrained environments. For maximum performance and minimal memory usage, the [Expect API](docs/expect.md) can be used to parse data of a predefined schema. 47 | 48 | ## The Write API 49 | 50 | The Write API encodes structured data to MessagePack. 51 | 52 | ```C 53 | // encode to memory buffer 54 | char* data; 55 | size_t size; 56 | mpack_writer_t writer; 57 | mpack_writer_init_growable(&writer, &data, &size); 58 | 59 | // write the example on the msgpack homepage 60 | mpack_build_map(&writer); 61 | mpack_write_cstr(&writer, "compact"); 62 | mpack_write_bool(&writer, true); 63 | mpack_write_cstr(&writer, "schema"); 64 | mpack_write_uint(&writer, 0); 65 | mpack_complete_map(&writer); 66 | 67 | // finish writing 68 | if (mpack_writer_destroy(&writer) != mpack_ok) { 69 | fprintf(stderr, "An error occurred encoding the data!\n"); 70 | return; 71 | } 72 | 73 | // use the data 74 | do_something_with_data(data, size); 75 | free(data); 76 | ``` 77 | 78 | In the above example, we encode to a growable memory buffer. The writer can instead write to a pre-allocated or stack-allocated buffer (with up-front sizes for compound types), avoiding the need for memory allocation. The writer can also be provided with a flush function (such as a file or socket write function) to call when the buffer is full or when writing is done. 79 | 80 | If any error occurs, the writer is placed in an error state. The writer will flag an error if too much data is written, if the wrong number of elements are written, if an allocation failure occurs, if the data could not be flushed, etc. No additional error handling is needed in the above code; any subsequent writes are ignored when the writer is in an error state, so you don't need to check every write for errors. 81 | 82 | The above example uses `mpack_build_map()` to automatically determine the number of key-value pairs contained. If you know up-front the number of elements needed, you can pass it to `mpack_start_map()` instead. In that case the corresponding `mpack_finish_map()` will assert in debug mode that the expected number of elements were actually written, which is something that other MessagePack C/C++ libraries may not do. 83 | 84 | ## Comparison With Other Parsers 85 | 86 | MPack is rich in features while maintaining very high performance and a small code footprint. Here's a short feature table comparing it to other C parsers: 87 | 88 | [mpack]: https://github.com/ludocode/mpack 89 | [msgpack-c]: https://github.com/msgpack/msgpack-c 90 | [cmp]: https://github.com/camgunz/cmp 91 | [cwpack]: https://github.com/clwi/CWPack 92 | 93 | | | [MPack][mpack]
(v1.1) | [msgpack-c][msgpack-c]
(v3.3.0) | [CMP][cmp]
(v19) | [CWPack][cwpack]
(v1.3.1) | 94 | |:------------------------------------|:---:|:---:|:---:|:---:| 95 | | No libc requirement | ✓ | | ✓ | ✓ | 96 | | Growable memory writer | ✓ | ✓ | | ✓\* | 97 | | File I/O helpers | ✓ | ✓ | | ✓\* | 98 | | Stateful error handling | ✓ | | ✓ | | 99 | | Incremental parser | ✓ | | ✓ | ✓ | 100 | | Tree stream parser | ✓ | ✓ | | | 101 | | Compound size tracking | ✓ | | | | 102 | | Automatic compound size | ✓ | | | | 103 | 104 | A larger feature comparison table is available [here](docs/features.md) which includes descriptions of the various entries in the table. 105 | 106 | [This benchmarking suite](https://github.com/ludocode/schemaless-benchmarks) compares the performance of MPack to other implementations of schemaless serialization formats. MPack outperforms all JSON and MessagePack libraries (except [CWPack][cwpack]), and in some tests MPack is several times faster than [RapidJSON](https://github.com/miloyip/rapidjson) for equivalent data. 107 | 108 | ## Why Not Just Use JSON? 109 | 110 | Conceptually, MessagePack stores data similarly to JSON: they are both composed of simple values such as numbers and strings, stored hierarchically in maps and arrays. So why not just use JSON instead? The main reason is that JSON is designed to be human-readable, so it is not as efficient as a binary serialization format: 111 | 112 | - Compound types such as strings, maps and arrays are delimited, so appropriate storage cannot be allocated upfront. The whole object must be parsed to determine its size. 113 | 114 | - Strings are not stored in their native encoding. Special characters such as quotes and backslashes must be escaped when written and converted back when read. 115 | 116 | - Numbers are particularly inefficient (especially when parsing back floats), making JSON inappropriate as a base format for structured data that contains lots of numbers. 117 | 118 | - Binary data is not supported by JSON at all. Small binary blobs such as icons and thumbnails need to be Base64 encoded or passed out-of-band. 119 | 120 | The above issues greatly increase the complexity of the decoder. Full-featured JSON decoders are quite large, and minimal decoders tend to leave out such features as string unescaping and float parsing, instead leaving these up to the user or platform. This can lead to hard-to-find platform-specific and locale-specific bugs, as well as a greater potential for security vulnerabilites. This also significantly decreases performance, making JSON unattractive for use in applications such as mobile games. 121 | 122 | While the space inefficiencies of JSON can be partially mitigated through minification and compression, the performance inefficiencies cannot. More importantly, if you are minifying and compressing the data, then why use a human-readable format in the first place? 123 | 124 | ## Testing MPack 125 | 126 | The MPack build process does not build MPack into a library; it is used to build and run the unit tests. You do not need to build MPack or the unit testing suite to use MPack. 127 | 128 | See [test/README.md](test/README.md) for information on how to test MPack. 129 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | MPack v1.1.1 2 | ------------ 3 | 4 | Bug Fixes: 5 | 6 | - Fixed a crash that could occur when a builder element was aligned exactly at the end of a page. (See #94) 7 | 8 | - Fixed a crash when encountering an I/O error during builder resolution without an error handler callback. (See #98) 9 | 10 | - Fixed an error destroying a writer while a builder is open. (See #88) 11 | 12 | - Fixed an issue with incorrect NULL checks for 0-length buffers. (See #97) 13 | 14 | - Fixed a string formatting issue on platforms where `int` is not 32 bits. (See #103) 15 | 16 | - Fixed some documentation errors. (See #93, #102) 17 | 18 | - Cleaned up some references to old unit test buildsystem. (See #95) 19 | 20 | MPack v1.1 21 | ---------- 22 | 23 | New Features: 24 | 25 | - Maps and arrays can now be built dynamically without specifying their size up front. See `mpack_build_map()` and `mpack_build_array()`. 26 | 27 | New Platforms: 28 | 29 | - Compiling as gnu89 is now supported. (See #68, #69) 30 | 31 | - Compiling in the Linux kernel is now possible using a [standalone configuration file](https://github.com/ludocode/mpack-linux-kernel). (See #80) 32 | 33 | - Compiling for AVR (e.g. Arduino) and other microcontrollers is now supported. MPack now compiles cleanly on platforms with 16-bit `int` and `size_t`. (See #74, #79) 34 | 35 | - `float` and/or `double` can now be disabled individually on platforms with limited floating point support. If `float` is supported but not `double`, MessagePack doubles can be converted to `float`. (See #74, #79) 36 | 37 | - MPack now builds cleanly under /W4 with Visual Studio 2015, 2017 and 2019 build tools. 38 | 39 | Bug Fixes and Other Changes: 40 | 41 | - An `mpack-defaults.h` sample configuration is no longer provided. 42 | 43 | - Replaced SCons unit test buildsystem and XCode/VS projects with Python+Ninja. 44 | 45 | - Fixed an issue where write overloads could be erroneously defined in C++ without `MPACK_WRITER` (#66). 46 | 47 | - Fixed some minor portability issues. 48 | 49 | MPack v1.0 50 | ---------- 51 | 52 | A number of breaking API changes have been made for the 1.0 release. Please take note of these changes when upgrading. 53 | 54 | Breaking Changes: 55 | 56 | - The Node API now separates tree initialization from parsing. After calling one of the `mpack_tree_init()` functions, you must explicitly call `mpack_tree_parse()` before accessing any nodes. 57 | 58 | - The configuration file `mpack-config.h` is now optional, and requires `MPACK_HAS_CONFIG` in order to be included. This means you must define `MPACK_HAS_CONFIG` when upgrading or your config file will be ignored! 59 | 60 | - Extension types are now disabled by default. You must define `MPACK_EXTENSIONS` to use them. 61 | 62 | - `mpack_tag_t` is now considered an opaque type to prevent future breakage when changing its layout. Compatibility is maintained for this release, but this may change in future releases. 63 | 64 | New Features: 65 | 66 | - The Node API can now parse multiple messages from a data source. `mpack_tree_parse()` can be called repeatedly to parse each message. 67 | 68 | - The Node API can now parse messages indefinitely from a continuous stream. A tree can be initialized with `mpack_tree_init_stream()` to receive a callback for more data. 69 | 70 | - The Node API can now parse messages incrementally from a non-blocking stream. Call `mpack_tree_try_parse()` with a non-blocking read function to start and resume parsing. It will return true when a complete message has become available. 71 | 72 | - The stdio helpers now allow reading from a `FILE*`. `_init_file()` functions have been renamed to `_init_filename()`. (The old names will continue to work for a few more versions.) 73 | 74 | - The Node API now returns a node of "missing" type instead of "nil" type for optional map lookups. This allows the caller to tell the difference between a key having value nil and a missing key. 75 | 76 | - The writer now supports a v4 compatibility mode. Call `mpack_writer_set_version(writer, mpack_version_v4);` to encode without using the `raw8`, `bin` and `ext` types. (This requires `MPACK_COMPATIBILITY`.) 77 | 78 | - The timestamp type has been implemented. A timestamp is a signed number of nanoseconds since the Unix epoch (1970-01-01T00:00:00Z). (This requires `MPACK_EXTENSIONS`.) 79 | 80 | Bug Fixes and Other Changes: 81 | 82 | - Fixed an allocation bug when closing a growable writer without having written anything (#58). 83 | 84 | - The reader's skip function is no longer ignored under `MPACK_OPTIMIZE_FOR_SIZE`. 85 | 86 | MPack v0.8.2 87 | ------------ 88 | 89 | Changes: 90 | 91 | - Fixed incorrect element tracking in `mpack_write_tag()` 92 | - Added type-generic writer functions `mpack_write()` and `mpack_write_kv()` 93 | - Added `mpack_write_object_bytes()` to insert pre-encoded MessagePack into a larger message 94 | - Enabled strings in all builds by default 95 | - Fixed unit test errors under `-ffast-math` 96 | - Fixed some compiler warnings 97 | 98 | MPack v0.8.1 99 | ------------ 100 | 101 | Changes: 102 | 103 | - Fixed some compiler warnings 104 | - Added various performance improvements 105 | - Improved documentation 106 | 107 | MPack v0.8 108 | ---------- 109 | 110 | Changes: 111 | 112 | - Added `mpack_peek_tag()` 113 | - Added reader helper functions to [expect re-ordered map keys](http://ludocode.github.io/mpack/md_docs_expect.html) 114 | - [Improved documentation](http://ludocode.github.io/mpack/) and added [Pages](http://ludocode.github.io/mpack/pages.html) 115 | - Made node key lookups check for duplicate keys 116 | - Added various UTF-8 checking functions for reader and nodes 117 | - Added support for compiling as C in recent versions of Visual Studio 118 | - Removed `mpack_expect_str_alloc()` and `mpack_expect_utf8_alloc()` 119 | - Fixed miscellaneous bugs and improved performance 120 | 121 | MPack v0.7.1 122 | ------------ 123 | 124 | Changes: 125 | 126 | - Removed `mpack_reader_destroy_cancel()` and `mpack_writer_destroy_cancel()`. You must now flag an error (such as `mpack_error_data`) in order to cancel reading. 127 | - Added many code size optimizations. `MPACK_OPTIMIZE_FOR_SIZE` is no longer experimental. 128 | - Improved and reorganized [Writer documentation](http://ludocode.github.io/mpack/group__writer.html) 129 | - Made writer flag `mpack_error_too_big` instead of `mpack_error_io` if writing too much data without a flush callback 130 | - Added optional `skip` callback and optimized `mpack_discard()` 131 | - Fixed various compiler and code analysis warnings 132 | - Optimized speed and memory usage 133 | 134 | MPack v0.7 135 | ---------- 136 | 137 | Changes: 138 | 139 | - Fixed various bugs in UTF-8 checking, error handler callbacks, out-of-memory and I/O errors, debug print functions and more 140 | - Added many missing Tag and Expect functions such as `mpack_tag_ext()`, `mpack_expect_int_range()` and `mpack_expect_utf8()` 141 | - Added extensive unit tests 142 | 143 | MPack v0.6 144 | ---------- 145 | 146 | Changes: 147 | 148 | - `setjmp`/`longjmp` support has been replaced by error callbacks. You can safely `longjmp` or throw C++ exceptions out of error callbacks. Be aware of local variable invalidation rules regarding `setjmp` if you use it. See the [documentation for `mpack_reader_error_t`](http://ludocode.github.io/mpack/mpack-reader_8h.html) and issue #19 for more details. 149 | - All `inline` functions in the MPack API are no longer `static`. A single non-`inline` definition of each `inline` function is emitted, so they behave like normal functions with external linkage. 150 | - Configuration options can now be pre-defined before including `mpack-config.h`, so you can customize MPack by defining these in your build system rather than editing the configuration file. 151 | 152 | MPack v0.5.1 153 | ------------ 154 | 155 | Changes: 156 | 157 | - Fixed compile errors in debug print function 158 | - Fixed C++11 warnings 159 | 160 | MPack v0.5 161 | ---------- 162 | 163 | Changes: 164 | 165 | - `mpack_node_t` is now a handle, so it should be passed by value, not by pointer. Porting to the new version should be as simple as replacing `mpack_node_t*` with `mpack_node_t` in your code. 166 | - Various other minor API changes have been made. 167 | - Major performance improvements were made across all aspects of MPack. 168 | 169 | MPack v0.4 170 | ---------- 171 | 172 | Changes 173 | 174 | - Added `mpack_writer_init_growable()` to write to a growable buffer 175 | - Converted tree parser to support node pool and pages. The Node API no longer requires an allocator. 176 | - Added Xcode unit test project, included projects in release package 177 | - Fixed various bugs 178 | 179 | MPack v0.3 180 | ---------- 181 | 182 | Changes: 183 | 184 | - Changed default config and test suite to use `DEBUG` and `_DEBUG` (instead of `NDEBUG`) 185 | - Added Visual Studio project for running unit tests 186 | - Fixed various bugs 187 | 188 | MPack v0.2 189 | ---------- 190 | 191 | Changes: 192 | 193 | - Added teardown callbacks to reader, writer and tree 194 | - Simplified API for working with files (`mpack_file_tree_t` is now internal) 195 | 196 | MPack v0.1 197 | ---------- 198 | 199 | Initial release. 200 | -------------------------------------------------------------------------------- /docs/expect.md: -------------------------------------------------------------------------------- 1 | # Using the Expect API 2 | 3 | The Expect API is used to imperatively parse data of a fixed (hardcoded) schema. It is most useful when parsing very large MessagePack files, parsing in memory-constrained environments, or generating parsing code from a schema. The API is similar to [CMP](https://github.com/camgunz/cmp), but has many helper functions especially for map keys and expected value ranges. Some of these will be covered below. 4 | 5 | Check out the [Reader API](docs/reader.md) guide first for information on setting up a reader and reading strings. 6 | 7 | If you are not writing code for an embedded device or generating parsing code from a schema, you should not follow this guide. You should most likely be using the [Node API](docs/node.md) instead. 8 | 9 | ## A simple example 10 | 11 | Suppose we have data that we know will have the following schema: 12 | 13 | ``` 14 | an array containing three elements 15 | a UTF-8 string of at most 127 characters 16 | a UTF-8 string of at most 127 characters 17 | an array containing up to ten elements 18 | where all elements are ints 19 | ``` 20 | 21 | For example, we could have the following bytes in a MessagePack file called `example.mp`: 22 | 23 | ```Shell 24 | 93 # an array containing three elements 25 | a5 68 65 6c 6c 6f # "hello" 26 | a6 77 6f 72 6c 64 21 # "world!" 27 | 94 # an array containing four elements 28 | 01 # 1 29 | 02 # 2 30 | 03 # 3 31 | 04 # 4 32 | ``` 33 | 34 | In JSON this would look like this: 35 | 36 | ```JSON 37 | [ 38 | "hello", 39 | "world!", 40 | [ 41 | 1, 42 | 2, 43 | 3, 44 | 4 45 | ] 46 | ] 47 | ``` 48 | 49 | You can use [msgpack-tools](https://github.com/ludocode/msgpack-tools) with the above JSON to generate `example.mp`. The below code demonstrates reading this data from a file using the Expect API: 50 | 51 | ```C 52 | #include "mpack.h" 53 | 54 | int main(void) { 55 | 56 | // Initialize a reader from a file 57 | mpack_reader_t reader; 58 | mpack_reader_init_file(&reader, "example.mp"); 59 | 60 | // The top-level array must have exactly three elements 61 | mpack_expect_array_match(&reader, 3); 62 | 63 | // The first two elements are short strings 64 | char first[128]; 65 | char second[128]; 66 | mpack_expect_utf8_cstr(&reader, first, sizeof(first)); 67 | mpack_expect_utf8_cstr(&reader, second, sizeof(second)); 68 | 69 | // Next we have an array of up to ten ints 70 | int32_t numbers[10]; 71 | size_t count = mpack_expect_array_max(&reader, sizeof(numbers) / sizeof(numbers[0])); 72 | for (size_t i = 0; i < count; ++i) 73 | numbers[i] = mpack_expect_i32(&reader); 74 | mpack_done_array(&reader); 75 | 76 | // Done reading the top-level array 77 | mpack_done_array(&reader); 78 | 79 | // Clean up and handle errors 80 | mpack_error_t error = mpack_reader_destroy(&reader); 81 | if (error != mpack_ok) { 82 | fprintf(stderr, "Error %i occurred reading data!\n", (int)error); 83 | return EXIT_FAILURE; 84 | } 85 | 86 | // We now know the data was parsed correctly and can safely 87 | // be used. The strings are null-terminated and valid UTF-8, 88 | // the array contained at most ten elements, and the numbers 89 | // are all within the range of an int32_t. 90 | printf("%s\n", first); 91 | printf("%s\n", second); 92 | for (size_t i = 0; i < count; ++i) 93 | printf("%i ", numbers[i]); 94 | printf("\n"); 95 | 96 | return EXIT_SUCCESS; 97 | } 98 | ``` 99 | 100 | With the file given above, this example will print: 101 | 102 | ``` 103 | hello 104 | world! 105 | 1 2 3 4 106 | ``` 107 | 108 | Note that there is only a single error check in this example. In fact each call to the reader is checking for errors and storing any error in the reader. These could be errors from reading data from the file, from invalid or corrupt MessagePack, or from not matching our expected types or ranges. On any call to the reader, if the reader was already in error or an error occurs during the call, a safe value is returned. 109 | 110 | For example the `mpack_expect_array_max()` call above will return zero if the element is not an array, if it has more than ten elements, if the MessagePack data is corrupt, or even if the file does not exist. The `mpack_expect_utf8_cstr()` calls will also place a null-terminator at the start of the given buffer if any error occurs just in case the data is used without an error check. The error check can be performed later at a more convenient time. 111 | 112 | ## Maps 113 | 114 | Maps can be more complicated to read because you usually want to safely handle keys being re-ordered. MessagePack itself does not specify whether maps can be re-ordered, so if you are sticking only to MessagePack implementations that preserve ordering, it may not be strictly necessary to handle this. (MPack always preserves map key ordering.) However many MessagePack implementations will ignore the order of map keys in the original data, especially in scripting languages where the data will be parsed into or encoded from an unordered map or dict. If you plan to interoperate with them, you will need to allow keys to be re-ordered. 115 | 116 | Suppose we expect to receive a map containing two key/value pairs: a key called "compact" with a boolean value, and a key called "schema" with an int value. The example on the [MessagePack homepage](http://msgpack.org/) fits this schema, which looks like this in JSON: 117 | 118 | ```JSON 119 | { 120 | "compact": true, 121 | "schema": 0 122 | } 123 | ``` 124 | 125 | If we also expect the key called "compact" to always come first, then parsing this is straightforward: 126 | 127 | ```C 128 | mpack_expect_map_match(&reader, 2); 129 | mpack_expect_cstr_match(&reader, "compact"); 130 | bool compact = mpack_expect_bool(&reader); 131 | mpack_expect_cstr_match(&reader, "schema"); 132 | int schema = mpack_expect_int(&reader); 133 | mpack_done_map(&reader); 134 | ``` 135 | 136 | If we expect the "schema" key to be optional, but always after "compact", then parsing this is longer but still straightforward: 137 | 138 | ```C 139 | size_t count = mpack_expect_map_max(&reader, 2); 140 | 141 | mpack_expect_cstr_match(&reader, "compact"); 142 | bool compact = mpack_expect_bool(&reader); 143 | 144 | bool has_schema = false; 145 | int schema = -1; 146 | if (count == 0) { 147 | mpack_expect_cstr_match(&reader, "schema"); 148 | schema = mpack_expect_int(&reader); 149 | } 150 | 151 | mpack_done_map(&reader); 152 | ``` 153 | 154 | If however we want to allow keys to be re-ordered, then parsing this can become a lot more verbose. You need to switch on the key, but you also need to track whether each key has been used to prevent duplicate keys and ensure that required keys were found. Using the `mpack_expect_cstr()` directly for keys, this would look like this: 155 | 156 | ```C 157 | bool has_compact = false; 158 | bool compact = false; 159 | bool has_schema = false; 160 | int schema = -1; 161 | 162 | for (size_t i = mpack_expect_map_max(&reader, 100); i > 0 && mpack_reader_error(&reader) == mpack_ok; --i) { 163 | char key[20]; 164 | mpack_expect_cstr(&reader, key, sizeof(key)); 165 | 166 | if (strcmp(key, "compact") == 0) { 167 | if (has_compact) 168 | mpack_flag_error(&reader, mpack_error_data); // duplicate key 169 | has_compact = true; 170 | compact = mpack_expect_bool(&reader); 171 | 172 | } else if (strcmp(key, "schema") == 0) { 173 | if (has_schema) 174 | mpack_flag_error(&reader, mpack_error_data); // duplicate key 175 | has_schema = true; 176 | schema = mpack_expect_int(&reader); 177 | 178 | } else { 179 | mpack_discard(&reader); 180 | } 181 | 182 | } 183 | mpack_done_map(&reader); 184 | 185 | // compact is not optional 186 | if (!has_compact) 187 | mpack_reader_flag_error(&reader, mpack_error_data); 188 | ``` 189 | 190 | This is obviously way too verbose. In order to simplify this code, MPack includes an Expect function called `mpack_expect_key_cstr()` to switch on string keys. This function should be passed an array of key strings and an array of bool flags storing whether each key was found. It will find the key in the given string array, check for duplicate keys, and return the index of the found key (or the key count if it is unrecognized or if an error occurs.) You would use it with an `enum` and a `switch`, like this: 191 | 192 | ```C 193 | enum key_names {KEY_COMPACT, KEY_SCHEMA, KEY_COUNT}; 194 | const char* keys[] = {"compact" , "schema" }; 195 | 196 | bool found[KEY_COUNT] = {0}; 197 | bool compact = false; 198 | int schema = -1; 199 | 200 | size_t i = mpack_expect_map_max(&reader, 100); // critical check! 201 | for (; i > 0 && mpack_reader_error(&reader) == mpack_ok; --i) { // critical check! 202 | switch (mpack_expect_key_cstr(&reader, keys, found, KEY_COUNT)) { 203 | case KEY_COMPACT: compact = mpack_expect_bool(&reader); break; 204 | case KEY_SCHEMA: schema = mpack_expect_int(&reader); break; 205 | default: mpack_discard(&reader); break; 206 | } 207 | } 208 | 209 | // compact is not optional 210 | if (!found[KEY_COMPACT]) 211 | mpack_reader_flag_error(&reader, mpack_error_data); 212 | ``` 213 | 214 | In the above examples, the call to `mpack_discard(&reader);` skips over the value for unrecognized keys, allowing the format to be extensible and providing forwards-compatibility. If you want to forbid unrecognized keys, you can flag an error (e.g. `mpack_reader_flag_error(&reader, mpack_error_data);`) instead of discarding the value. 215 | 216 | WARNING: See above the importance of using a reasonable limit on `mpack_expect_map_max()`, and of checking for errors in each iteration of the loop. If we were to leave these out, an attacker could craft a message declaring an array of a billion elements, forcing this code into a very long loop. We specify a size of 100 here as an arbitrary limit that leaves enough space for the schema to grow in the future. If you forbid unrecognized keys, you could specify the key count as the limit. 217 | 218 | Unlike JSON, MessagePack supports any type as a map key, so the enum integer values can themselves be used as keys. This reduces message size at some expense of debuggability (losing some of the value of a schemaless format.) There is a simpler function `mpack_expect_key_uint()` which can be used to switch on small non-negative enum values directly. 219 | 220 | On the surface this doesn't appear much shorter than the previous code, but it becomes much nicer when you have many possible keys in a map. Of course if at all possible you should consider using the [Node API](docs/node.md) which is much less error-prone and will handle all of this for you. 221 | -------------------------------------------------------------------------------- /test/unit/src/test.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | * this software and associated documentation files (the "Software"), to deal in 6 | * the Software without restriction, including without limitation the rights to 7 | * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 8 | * the Software, and to permit persons to whom the Software is furnished to do so, 9 | * subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 16 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 17 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 18 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 19 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 20 | */ 21 | 22 | #ifndef MPACK_TEST_H 23 | #define MPACK_TEST_H 1 24 | 25 | #define _DEFAULT_SOURCE 1 26 | #define _BSD_SOURCE 1 27 | 28 | #ifdef WIN32 29 | #define _CRT_SECURE_NO_WARNINGS 1 30 | #endif 31 | 32 | // mpack poisons float and double when disabled so we give ourselves macros to 33 | // use them manually in tests 34 | #define TEST_FLOAT float 35 | #define TEST_DOUBLE double 36 | 37 | // In the special case where we have float but not double, we have to include 38 | // first since MPack poisons double in the unit test suite to make 39 | // sure it isn't used. 40 | #if defined(MPACK_DOUBLE) && !defined(MPACK_FLOAT) 41 | #if !MPACK_DOUBLE 42 | #include 43 | #endif 44 | #endif 45 | 46 | #include "mpack/mpack.h" 47 | 48 | #if MPACK_CONFORMING 49 | #include 50 | #include 51 | #include 52 | #include 53 | #include 54 | #if MPACK_FLOAT 55 | #include 56 | #endif 57 | #endif 58 | 59 | #if MPACK_STDIO 60 | #ifdef WIN32 61 | #include 62 | #define mkdir(path, mode) ((void)(mode), _mkdir(path)) 63 | #define rmdir _rmdir 64 | #else 65 | #include 66 | #include 67 | #endif 68 | #endif 69 | 70 | // We silence the same warnings as MPack across the entire unit test suite. 71 | // There's no MPACK_SILENCE_WARNINGS_END to match this. 72 | MPACK_SILENCE_WARNINGS_BEGIN 73 | 74 | // We also silence warnings specifically for unit tests 75 | #ifdef _MSC_VER 76 | #pragma warning(disable:4611) // interaction between '_setjmp' and C++ object destruction is non-portable 77 | #pragma warning(disable:4204) // nonstandard extension used: non-constant aggregate initializer 78 | #pragma warning(disable:4221) // nonstandard extension used: cannot be initialized using address of automatic variable 79 | #endif 80 | 81 | // We silence shadow warnings around variable declarations in some macros 82 | #if defined(MPACK_SILENCE_WARNINGS_PUSH) 83 | #ifdef __GNUC__ 84 | #define TEST_MPACK_SILENCE_SHADOW_BEGIN \ 85 | MPACK_SILENCE_WARNINGS_PUSH \ 86 | _Pragma ("GCC diagnostic ignored \"-Wshadow\"") 87 | #define TEST_MPACK_SILENCE_SHADOW_END \ 88 | MPACK_SILENCE_WARNINGS_POP 89 | #elif defined(_MSC_VER) 90 | #define TEST_MPACK_SILENCE_SHADOW_BEGIN \ 91 | MPACK_SILENCE_WARNINGS_PUSH \ 92 | __pragma(warning(disable:4456)) \ 93 | __pragma(warning(disable:4459)) 94 | #define TEST_MPACK_SILENCE_SHADOW_END \ 95 | MPACK_SILENCE_WARNINGS_POP 96 | #endif 97 | #endif 98 | #ifndef TEST_MPACK_SILENCE_SHADOW_BEGIN 99 | #define TEST_MPACK_SILENCE_SHADOW_BEGIN /*nothing*/ 100 | #define TEST_MPACK_SILENCE_SHADOW_END /*nothing*/ 101 | #endif 102 | 103 | // For some older versions of GCC, we silence shadow warnings globally since 104 | // they don't properly handle push/pop around variable declarations. 105 | #ifdef __GNUC__ 106 | #if __GNUC__ < 9 107 | #pragma GCC diagnostic ignored "-Wshadow" 108 | #endif 109 | #endif 110 | 111 | 112 | 113 | /** 114 | * Floating point infinities 115 | * 116 | * MSVC is ridiculous with warnings when it comes to infinity. All of this 117 | * wraps various infinities to avoid constant arithmetic overflow warnings. 118 | */ 119 | 120 | #if MPACK_FLOAT 121 | 122 | #ifdef __cplusplus 123 | #include 124 | #define MPACK_FLOAT_POSITIVE_INFINITY (std::numeric_limits::infinity()) 125 | #define MPACK_DOUBLE_POSITIVE_INFINITY (std::numeric_limits::infinity()) 126 | #endif 127 | 128 | #ifdef _MSC_VER 129 | #pragma warning(push) 130 | #pragma warning(disable:4056) // overflow in floating-point constant arithmetic 131 | #pragma warning(disable:4756) // overflow in constant arithmetic 132 | 133 | #ifndef MPACK_FLOAT_POSITIVE_INFINITY 134 | MPACK_STATIC_INLINE float mpack_float_positive_infinity() { 135 | return (float)(INFINITY); 136 | } 137 | #define MPACK_FLOAT_POSITIVE_INFINITY (mpack_float_positive_infinity()) 138 | #endif 139 | 140 | #ifndef MPACK_DOUBLE_POSITIVE_INFINITY 141 | MPACK_STATIC_INLINE double mpack_double_positive_infinity() { 142 | return (double)(INFINITY); 143 | } 144 | #define MPACK_DOUBLE_POSITIVE_INFINITY (mpack_double_positive_infinity()) 145 | #endif 146 | 147 | MPACK_STATIC_INLINE float mpack_float_negative_infinity() { 148 | return -MPACK_FLOAT_POSITIVE_INFINITY; 149 | } 150 | #define MPACK_FLOAT_NEGATIVE_INFINITY (mpack_float_negative_infinity()) 151 | 152 | MPACK_STATIC_INLINE double mpack_double_negative_infinity() { 153 | return -MPACK_DOUBLE_POSITIVE_INFINITY; 154 | } 155 | #define MPACK_DOUBLE_NEGATIVE_INFINITY (mpack_double_negative_infinity()) 156 | 157 | #pragma warning(pop) 158 | #endif 159 | 160 | #if MPACK_FLOAT 161 | #ifndef MPACK_FLOAT_POSITIVE_INFINITY 162 | #define MPACK_FLOAT_POSITIVE_INFINITY ((float)(INFINITY)) 163 | #endif 164 | #ifndef MPACK_FLOAT_NEGATIVE_INFINITY 165 | #define MPACK_FLOAT_NEGATIVE_INFINITY (-MPACK_FLOAT_POSITIVE_INFINITY) 166 | #endif 167 | #endif 168 | #if MPACK_DOUBLE 169 | #ifndef MPACK_DOUBLE_POSITIVE_INFINITY 170 | #define MPACK_DOUBLE_POSITIVE_INFINITY ((double)(INFINITY)) 171 | #endif 172 | #ifndef MPACK_DOUBLE_NEGATIVE_INFINITY 173 | #define MPACK_DOUBLE_NEGATIVE_INFINITY (-MPACK_DOUBLE_POSITIVE_INFINITY) 174 | #endif 175 | #endif 176 | 177 | #endif 178 | 179 | 180 | 181 | #if !MPACK_FINITE_MATH 182 | #if defined(WIN32) 183 | #include 184 | #define isnanf _isnan 185 | #elif defined(__APPLE__) 186 | #define isnanf isnan 187 | #elif defined(__GNUC__) || defined(__clang__) 188 | // On some versions of GCC/Ubuntu (e.g. 4.8.4 on Ubuntu 14.4), 189 | // isnan() incorrectly causes double->float warnings even when 190 | // called on a double 191 | // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61501 192 | #undef isnan 193 | #define isnan(x) __builtin_isnan(x) 194 | #endif 195 | #if !defined(isnanf) && !defined(MPACK_ISNANF_IS_FUNC) 196 | #define isnanf isnan 197 | #endif 198 | #endif 199 | 200 | extern mpack_tag_t (*fn_mpack_tag_nil)(void); 201 | 202 | extern const char* lipsum; 203 | 204 | #ifdef __cplusplus 205 | extern "C" { 206 | #endif 207 | 208 | /* 209 | * This is basically the whole unit testing framework for mpack. 210 | * The reported number of "tests" is the total number of test asserts, 211 | * where each actual test case has several asserts. 212 | */ 213 | 214 | // enable this to exit at the first error 215 | #define TEST_EARLY_EXIT 1 216 | 217 | // runs the given expression, causing a unit test failure with the 218 | // given printf format string if the expression is not true. 219 | #define TEST_TRUE(...) \ 220 | MPACK_EXPAND(TEST_TRUE_IMPL((MPACK_EXTRACT_ARG0(__VA_ARGS__)), __FILE__, __LINE__, __VA_ARGS__ , "" , NULL)) 221 | 222 | #define TEST_TRUE_IMPL(result, file, line, ignored, ...) \ 223 | MPACK_EXPAND(test_true_impl(result, file, line, __VA_ARGS__)) 224 | 225 | void test_true_impl(bool result, const char* file, int line, const char* format, ...); 226 | 227 | extern int tests; 228 | extern int passes; 229 | 230 | #if MPACK_DEBUG 231 | extern bool test_jmp_set; 232 | extern jmp_buf test_jmp_buf; 233 | extern bool test_break_set; 234 | extern bool test_break_hit; 235 | 236 | // calls setjmp to expect an assert from a unit test. an assertion 237 | // will cause a longjmp to here with a value of 1. 238 | #define TEST_TRUE_SETJMP() \ 239 | (TEST_TRUE(!test_jmp_set, "an assert jmp is already set!"), \ 240 | test_jmp_set = true, \ 241 | setjmp(test_jmp_buf)) 242 | 243 | // clears the expectation of an assert. a subsequent assert will 244 | // cause the unit test suite to abort with error. 245 | #define TEST_TRUE_CLEARJMP() \ 246 | (TEST_TRUE(test_jmp_set, "an assert jmp is not set!"), \ 247 | test_jmp_set = false) 248 | 249 | // runs the given expression, causing a unit test failure if an assertion 250 | // is not triggered. (note that stack variables may need to be marked volatile 251 | // since non-volatile stack variables that are written to after setjmp are 252 | // undefined after longjmp.) 253 | #define TEST_ASSERT(expr) do { \ 254 | volatile bool jumped = false; \ 255 | if (TEST_TRUE_SETJMP()) { \ 256 | jumped = true; \ 257 | } else { \ 258 | (expr); \ 259 | } \ 260 | TEST_TRUE_CLEARJMP(); \ 261 | TEST_TRUE(jumped, "expression should assert, but didn't: " #expr); \ 262 | } while (0) 263 | 264 | #define TEST_BREAK(expr) do { \ 265 | test_break_set = true; \ 266 | test_break_hit = false; \ 267 | TEST_TRUE(expr, "expression is not true: " # expr); \ 268 | TEST_TRUE(test_break_hit, "expression should break, but didn't: " # expr); \ 269 | test_break_set = false; \ 270 | } while (0); 271 | 272 | #else 273 | 274 | // in release mode there are no asserts or break functions, so 275 | // TEST_BREAK() just runs the expr. it is usually used to test 276 | // that something flags mpack_error_bug. 277 | #define TEST_BREAK(expr) do { TEST_TRUE(expr); } while (0) 278 | 279 | // TEST_ASSERT() is not defined because code that causes an assert 280 | // cannot continue to run; it would cause undefined behavior (and 281 | // crash the unit test framework.) it cannot be defined to nothing 282 | // because the tests around it wouldn't make sense (and would cause 283 | // unused warnings); the tests that use it must be disabled entirely. 284 | 285 | #endif 286 | 287 | #ifdef __cplusplus 288 | } 289 | #endif 290 | 291 | #endif 292 | 293 | -------------------------------------------------------------------------------- /docs/protocol.md: -------------------------------------------------------------------------------- 1 | # Protocol Clarifications 2 | 3 | The MessagePack specification contains overlap between different types, allowing the same data to be encoded in many different representations. For example there are overlong sequences, signed/unsigned overlap for non-negative integers, different floating-point widths, raw/str/bin/ext types, and more. 4 | 5 | MessagePack also does not specify how types should be interpreted, such as whether maps are ordered, whether strings can be treated as binary data, whether integers can be treated as real numbers, and so on. Some of these are explicitly left up to the implementation, such as whether unrecognized extensions should be rejected or treated as opaque data. 6 | 7 | MPack currently implements the [v5/2.0 MessagePack specification](https://github.com/msgpack/msgpack/blob/0b8f5ac67cdd130f4d4d4fe6afb839b989fdb86a/spec.md?). This document describes MPack's implementation. 8 | 9 | ## Overlong Sequences 10 | 11 | MessagePack provides several different widths for many types. For example fixint, int8, int16, int32, int64 for integers; fixstr, str8, str16, str32 for strings; and so on. The number -20 could be encoded as `EC`, `D0 EC`, `D1 FF EC`, and more. 12 | 13 | UTF-8 has similar overlap between codepoint widths. In UTF-8, inefficient representations of codepoints are called "overlong sequences", and decoders are required to treat them as errors. MessagePack on the other hand does not have any restrictions on inefficient representations. 14 | 15 | - When encoding any value (other than floating point numbers), MPack always chooses the shortest representation. This is the case for integers and all compound types. 16 | 17 | - When encoding a string, MPack always uses the str8 type if possible (which is not mapped to a "raw" from the old version of the MessagePack spec, since there was no "raw8" type.) 18 | 19 | - When encoding an ext type, MPack always chooses a fixext type if available. This means if the ext size is 1, 2, 4, 8 or 16, the ext start tag will be encoded in two bytes (fixext and the ext type.) Otherwise MPack chooses the shortest representation. 20 | 21 | - When decoding any value, MPack allows any length representation. Inefficiently encoded sequences are not an error. So `EC`, `D0 EC` and `D1 FF EC` would all be decoded to the same value, `mpack_type_int` of value -20. 22 | 23 | As of this writing, all C and C++ libraries seem to write data in the shortest representation, and none forbid overlong sequences (including the reference implementation.) 24 | 25 | ## Integer Signedness 26 | 27 | MessagePack allows non-negative integers to be encoded both as signed and unsigned, and the specification does not specify how a library should serialize them. For example the number 100 in the shortest representation could be encoded as either `CC 64` (unsigned) or `D0 64` (signed). 28 | 29 | - When encoding, MPack writes all non-negative integers in the shortest unsigned integer type, regardless of the signedness of the input type. (The signedness of the input type is discarded.) 30 | 31 | - When decoding as a dynamic tag or node, MPack returns the signedness of the serialized type. (This means you always need to handle both `mpack_type_int` and `mpack_type_uint`, regardless of whether you want a signed or unsigned integer, and regardless of whether you want a negative or non-negative integer.) 32 | 33 | - When retrieving an integer with the Expect or Node APIs, MPack will automatically convert between signedness without loss of data. (For example if you call `mpack_expect_uint()` or `mpack_node_uint()`, MPack will allow both signed and unsigned data, and will flag an error if the type is signed with a negative value. Likewise if you call `mpack_expect_i8()` or `mpack_node_i8()`, MPack will allow both signed and unsigned data, and will flag an error for values below `INT8_MIN` or above `INT8_MAX`.) The expect and node integer functions allow an integer of any size or signedness, and are only checking that it falls within the range of the requested type. 34 | 35 | A library could technically preserve the signedness of variables by writing any signed variable as int8/int16/int32/int64 or a negative fixint even if the value is non-negative. This does not seem to be the intent of the specification. For example there are no positive signed fixint values, so encoding the signed int with value 1 would take two bytes (`D0 01`) to preserve signedness. This is why MPack discards signedness. 36 | 37 | As of this writing, all C and C++ libraries supporting the modern MessagePack specification appear to discard signedness and write all non-negative ints as unsigned (including the reference implementation.) 38 | 39 | ## Floating Point Numbers 40 | 41 | In addition to the integer types, the MessagePack specification includes "float 32" and "float 64" types for real numbers. 42 | 43 | - When encoding, MPack writes real numbers as the original width of the data (so `mpack_write_float()` writes a "float 32", and `mpack_write_double()` writes a "float 64".) 44 | 45 | - When decoding as a dynamic tag or node, MPack returns the width of the serialized type. (It is recommended to handle both `mpack_type_float` and `mpack_type_double` (or neither) since other libraries may write real numbers in any width.) 46 | 47 | - When expecting a real number with the Expect API, or when getting a float or double from a node in the Node API, MPack includes two different sets of functions: 48 | 49 | - The lax versions are the default. These will allow any integer or real type and convert it to the expected type, which may involve loss of precision. These include `mpack_expect_float()`, `mpack_expect_double()`, `mpack_node_float()` and `mpack_node_double()`. 50 | 51 | - The strict versions, suffixed by `_strict`, will allow only real numbers, and only of a width of at least that of the expected type. So `mpack_node_float_strict()` or `mpack_expect_float_strict()` allow only "float 32", while `mpack_node_double_strict()` or `mpack_expect_double_strict()` allow both "float 32" and "float 64". 52 | 53 | - If you want to allow only a "float 64", you would have to read a tag or check the node type and make sure it contains `mpack_type_double`. 54 | 55 | - If you want a `float` version that allows either "float 32" or "float 64" but not integers, you could use `(float)mpack_node_double_strict()` or `(float)mpack_expect_double_strict()`. But if you are using `float` you probably don't care much about precision anyway so you should just use `mpack_node_float()` or `mpack_expect_float()`. 56 | 57 | MessagePack libraries in dynamic languages may support an option to generate floats instead of doubles for space efficiency. If you're converting data from JSON, you could use [json2msgpack -f](https://github.com/ludocode/msgpack-tools) to convert to floats instead of doubles. 58 | 59 | ## Map Ordering 60 | 61 | MessagePack does not specify the ordering of map key/value pairs. Key/value pairs have a well-defined order when serialized, but the specification does not specify whether implementations should observe it when encoding or preserve it when decoding, and does not specify whether it should be adapted to an ordered associative array when de-serialized. 62 | 63 | MPack always preserves map ordering. Key/value pairs are written in the given order in the write API, read in the serialized order in the read and expect APIs, and provided in their original serialized order in the Node API. In particular this means `mpack_node_map_key_at()` and `mpack_node_map_value_at()` are always ordered as stored in the original serialized data. An application using only MPack can always assume a fixed map order. 64 | 65 | However, MPack strongly recommends writing code that allows for map re-ordering. This is for two reasons: 66 | 67 | - MessagePack is often used to interface with languages that do not preserve map ordering. For example the msgpack-python library in Python unpacks a map to a `dict`, not an `OrderedDict`. Many languages use hashtables to store keys, so MessagePack encoded by these languages will have map key/value pairs in a random order. The order may be different between compiler or interpreter versions even for identical map content. 68 | 69 | - MessagePack is designed to be at least partly compatible with JSON. It is sometimes converted from JSON, and is sometimes recommended as an efficient replacement for JSON. Unlike MessagePack, JSON explicitly allows map re-ordering. Two JSON documents that have re-ordered key/value pairs but are otherwise the same are considered equivalent. 70 | 71 | MPack contains functions to make it easy to parse messages with re-ordered map pairs. For the Node API, lookup functions such as `mpack_node_map_cstr()` and `mpack_node_map_int()` will find the value for a given key regardless of ordering. For the expect API, the functions `mpack_expect_key_cstr()` and `mpack_expect_key_uint()` can be used to switch on a key in a read loop, which allows parsing map pairs in any order. 72 | 73 | ## Duplicate Map Keys 74 | 75 | MessagePack has no restrictions against duplicate keys in a map, so MPack allows duplicate keys in maps. Iterating over a map in the Node or Reader will provide key/value pairs in serialized order and will not flag any errors for duplicates. However, helper functions that compare keys (such as the "lookup" or "match" functions) do check for duplicates. 76 | 77 | In the Node API, the MPack lookup functions that search for a given key to find its value always check for duplicates. They are meant to provide a unique value for a given key. For example `mpack_node_map_cstr()` and `mpack_node_map_int()` will always check the whole map and will flag an error if a duplicate key is found. If you want to find multiple values for a given key, you will need to iterate over them manually with `mpack_node_map_key_at()` and `mpack_node_map_value_at()`. 78 | 79 | In the Expect API, the key match functions (such as `mpack_expect_key_cstr()` and `mpack_expect_key_uint()`) check for duplicate keys, and will flag an error when a duplicate is found. If you want to use the match functions with duplicates, you can toggle off the `bool` flag corresponding to a found key to allow it to be matched again. This allows implicit checking of duplicate keys, with an opt-in to safely handle duplicates in order. 80 | 81 | Despite the allowance for duplicate keys, MPack recommends against providing multiple values for the same key in order to more safely interface with other languages and formats (as with the Map Ordering recommendations above.) A safer and more explicit way to accomplish this is to simply use an array containing the desired values as the single value for a map key. 82 | 83 | ## v4 Compatibility 84 | 85 | The MessagePack [v4/1.0 spec](https://github.com/msgpack/msgpack/blob/acbcdf6b2a5a62666987c041124a10c69124be0d/spec-old.md?) did not distinguish between strings and binary data. It only provided the "raw" type in widths of fixraw, raw16 and raw32, which was used for both. The [v5/2.0 spec](https://github.com/msgpack/msgpack/blob/0b8f5ac67cdd130f4d4d4fe6afb839b989fdb86a/spec.md?) on the other hand renames the raw type to str, adds the bin type to represent binary data, and adds an 8-bit width for strings called str8. This means that even when binary data is not used, the new specification is not backwards compatible with the old one that expects raw to contain strings, because a modern encoder will use the str8 type. The new specification also adds an ext type to distinguish between arbitrary binary blobs and MessagePack extensions. 86 | 87 | - MPack by default always encodes with the str8 type for strings where possible. This means that MessagePack encoded with MPack is by default not backwards compatible with decoders that only understand the raw types from the old specification. This matches the behaviour of other C/C++ libraries that support the modern spec, including the reference implementation. 88 | 89 | - Since MPack allows overlong sequences, it does not require that the str8 type be used, so data encoded with an old-style encoder will be parsed correctly by MPack (with raw types parsed as strings.) 90 | 91 | MPack supports a compatibility mode if interoperability is required with applications or data that do not support the new (v5) MessagePack spec. To use it, you must define `MPACK_COMPATIBILITY`, and then call `mpack_writer_set_version()` with the value `mpack_version_v4`. A writer in this mode will never use the str8 type, and will output the old fixraw, raw16 and raw32 types when writing both strings and bins. 92 | 93 | Note that there is no mode to configure readers to interpret the old style raws as either bins or strings. They will only be interpreted as strings. 94 | 95 | ## Extension Types 96 | 97 | The MessagePack specification defines support for extension types. These are like bin types, but with an additional marker to define the semantic type, giving a clue to parsers about how to interpret the contained bits. 98 | 99 | This author considers them redundant with the bin type. Adding semantic information about binary data does nothing to improve interoperability. It instead only increases the complexity of decoders and fragments support for MessagePack. In my opinion the MessagePack spec should be forever frozen at v5 without extension types, similar (in theory) to JSON. See the discussion in [msgpack/msgpack!206](https://github.com/msgpack/msgpack/issues/206#issuecomment-386066548) for more. 100 | 101 | MPack discourages the use of extension types. In its default configuration, MPack will flag `mpack_error_unsupported` when encountering them, and functions to encode them are preproc'd out. Support for extension types can be enabled by defining `MPACK_EXTENSIONS`. 102 | -------------------------------------------------------------------------------- /test/unit/src/test-builder.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019-2021 Nicholas Fraser and the MPack authors 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | * this software and associated documentation files (the "Software"), to deal in 6 | * the Software without restriction, including without limitation the rights to 7 | * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 8 | * the Software, and to permit persons to whom the Software is furnished to do so, 9 | * subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 16 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 17 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 18 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 19 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 20 | */ 21 | 22 | #include "test-builder.h" 23 | #include "test-write.h" 24 | 25 | #if MPACK_BUILDER 26 | static void test_builder_basic(void) { 27 | static char buf[4096]; 28 | mpack_writer_t writer; 29 | 30 | mpack_writer_init(&writer, buf, sizeof(buf)); 31 | mpack_build_array(&writer); 32 | mpack_complete_array(&writer); 33 | TEST_DESTROY_MATCH_IMPL(buf, "\x90"); 34 | 35 | mpack_writer_init(&writer, buf, sizeof(buf)); 36 | mpack_build_array(&writer); 37 | mpack_write_u8(&writer, 2); 38 | mpack_complete_array(&writer); 39 | TEST_DESTROY_MATCH_IMPL(buf, "\x91\x02"); 40 | 41 | mpack_writer_init(&writer, buf, sizeof(buf)); 42 | mpack_build_map(&writer); 43 | mpack_write_cstr(&writer, "hello"); 44 | mpack_write_cstr(&writer, "world"); 45 | mpack_complete_map(&writer); 46 | TEST_DESTROY_MATCH_IMPL(buf, "\x81\xa5hello\xa5world"); 47 | } 48 | 49 | static void test_builder_repeat(void) { 50 | static char buf[4096]; 51 | mpack_writer_t writer; 52 | 53 | mpack_writer_init(&writer, buf, sizeof(buf)); 54 | mpack_start_array(&writer, 4); 55 | mpack_build_array(&writer); 56 | mpack_complete_array(&writer); 57 | mpack_build_map(&writer); 58 | mpack_complete_map(&writer); 59 | mpack_build_array(&writer); 60 | mpack_write_u8(&writer, 2); 61 | mpack_complete_array(&writer); 62 | mpack_build_map(&writer); 63 | mpack_write_cstr(&writer, "hello"); 64 | mpack_write_cstr(&writer, "world"); 65 | mpack_complete_map(&writer); 66 | mpack_finish_array(&writer); 67 | 68 | TEST_DESTROY_MATCH_IMPL(buf, "\x94\x90\x80\x91\x02\x81\xa5hello\xa5world"); 69 | } 70 | 71 | static void test_builder_nested(void) { 72 | static char buf[4096]; 73 | mpack_writer_t writer; 74 | 75 | mpack_writer_init(&writer, buf, sizeof(buf)); 76 | mpack_build_map(&writer); 77 | mpack_write_cstr(&writer, "nums"); 78 | mpack_build_array(&writer); 79 | mpack_write_int(&writer, 1); 80 | mpack_write_int(&writer, 2); 81 | mpack_write_int(&writer, 3); 82 | mpack_complete_array(&writer); 83 | mpack_write_cstr(&writer, "nil"); 84 | mpack_write_nil(&writer); 85 | mpack_complete_map(&writer); 86 | TEST_DESTROY_MATCH_IMPL(buf, "\x82\xa4nums\x93\x01\x02\x03\xa3nil\xc0"); 87 | 88 | mpack_writer_init(&writer, buf, sizeof(buf)); 89 | mpack_build_array(&writer); 90 | mpack_build_array(&writer); 91 | mpack_build_array(&writer); 92 | mpack_write_int(&writer, 1); 93 | mpack_write_int(&writer, 2); 94 | mpack_write_int(&writer, 3); 95 | mpack_complete_array(&writer); 96 | mpack_complete_array(&writer); 97 | mpack_complete_array(&writer); 98 | TEST_DESTROY_MATCH_IMPL(buf, "\x91\x91\x93\x01\x02\x03"); 99 | } 100 | 101 | static void test_builder_deep(void) { 102 | static char buf[16*1024]; 103 | mpack_writer_t writer; 104 | mpack_writer_init(&writer, buf, sizeof(buf)); 105 | 106 | char expected[sizeof(buf)]; 107 | size_t pos = 0; 108 | int depth = 2;//50; 109 | 110 | int i; 111 | for (i = 0; i < depth; ++i) { 112 | //mpack_build_map(&writer); 113 | mpack_start_map(&writer, 2); 114 | expected[pos++] = '\x82'; 115 | mpack_write_cstr(&writer, "ab"); 116 | expected[pos++] = '\xa2'; 117 | expected[pos++] = 'a'; 118 | expected[pos++] = 'b'; 119 | mpack_build_array(&writer); 120 | expected[pos++] = '\x94'; 121 | mpack_write_int(&writer, 2); 122 | expected[pos++] = '\x02'; 123 | mpack_write_int(&writer, 3); 124 | expected[pos++] = '\x03'; 125 | mpack_write_int(&writer, 4); 126 | expected[pos++] = '\x04'; 127 | } 128 | 129 | mpack_write_bool(&writer, true); 130 | expected[pos++] = '\xc3'; 131 | 132 | for (i = 0; i < depth; ++i) { 133 | mpack_complete_array(&writer); 134 | mpack_write_int(&writer, 1); 135 | expected[pos++] = '\x01'; 136 | mpack_write_nil(&writer); 137 | expected[pos++] = '\xc0'; 138 | //mpack_complete_map(&writer); 139 | mpack_finish_map(&writer); 140 | } 141 | 142 | TEST_TRUE(pos <= sizeof(expected)); 143 | size_t used = mpack_writer_buffer_used(&writer); 144 | 145 | /* 146 | printf("actual %zi expected %zi\n", used, pos); 147 | for (size_t i = 0; i < used; ++i) { 148 | printf("%02hhx ", buf[i]); 149 | if (((i+1) % 16)==0) 150 | printf("\n"); 151 | } 152 | printf("\n"); 153 | printf("\n"); 154 | for (size_t i = 0; i < pos; ++i) { 155 | printf("%02hhx ", expected[i]); 156 | if (((i+1) % 16)==0) 157 | printf("\n"); 158 | } 159 | printf("\n"); 160 | */ 161 | 162 | TEST_WRITER_DESTROY_NOERROR(&writer); 163 | TEST_TRUE(used == pos); 164 | TEST_TRUE(0 == memcmp(buf, expected, used)); 165 | } 166 | 167 | static void test_builder_large(void) { 168 | static char buf[16*1024]; 169 | mpack_writer_t writer; 170 | mpack_writer_init(&writer, buf, sizeof(buf)); 171 | 172 | char expected[sizeof(buf)]; 173 | size_t pos = 0; 174 | int depth = 6; 175 | 176 | int i; 177 | for (i = 0; i < depth; ++i) { 178 | mpack_build_map(&writer); 179 | expected[pos++] = '\xde'; 180 | expected[pos++] = '\x00'; 181 | expected[pos++] = '\x32'; 182 | size_t j; 183 | for (j = 0; j < 99; ++j) { 184 | mpack_write_int(&writer, -1); 185 | expected[pos++] = '\xff'; 186 | } 187 | } 188 | 189 | mpack_write_int(&writer, -1); 190 | expected[pos++] = '\xff'; 191 | 192 | for (i = 0; i < depth; ++i) { 193 | mpack_complete_map(&writer); 194 | } 195 | 196 | TEST_TRUE(pos <= sizeof(expected)); 197 | size_t used = mpack_writer_buffer_used(&writer); 198 | TEST_WRITER_DESTROY_NOERROR(&writer); 199 | TEST_TRUE(used == pos); 200 | TEST_TRUE(0 == memcmp(buf, expected, used)); 201 | } 202 | 203 | static void test_builder_content(void) { 204 | static char buf[16*1024]; 205 | mpack_writer_t writer; 206 | mpack_writer_init(&writer, buf, sizeof(buf)); 207 | 208 | char expected[sizeof(buf)]; 209 | size_t pos = 0; 210 | 211 | mpack_build_map(&writer); 212 | //mpack_start_map(&writer, 3); 213 | expected[pos++] = '\x83'; 214 | 215 | mpack_write_cstr(&writer, "rid"); 216 | expected[pos++] = '\xa3'; 217 | expected[pos++] = 'r'; 218 | expected[pos++] = 'i'; 219 | expected[pos++] = 'd'; 220 | 221 | char rid[16] = {0}; 222 | mpack_write_bin(&writer, rid, sizeof(rid)); 223 | expected[pos++] = '\xc4'; 224 | expected[pos++] = '\x10'; 225 | size_t i; 226 | for (i = 0; i < 16; ++i) 227 | expected[pos++] = '\x00'; 228 | 229 | mpack_write_cstr(&writer, "type"); 230 | expected[pos++] = '\xa4'; 231 | expected[pos++] = 't'; 232 | expected[pos++] = 'y'; 233 | expected[pos++] = 'p'; 234 | expected[pos++] = 'e'; 235 | 236 | mpack_write_cstr(&writer, "inode"); 237 | expected[pos++] = '\xa5'; 238 | expected[pos++] = 'i'; 239 | expected[pos++] = 'n'; 240 | expected[pos++] = 'o'; 241 | expected[pos++] = 'd'; 242 | expected[pos++] = 'e'; 243 | 244 | mpack_write_cstr(&writer, "content"); 245 | expected[pos++] = '\xa7'; 246 | expected[pos++] = 'c'; 247 | expected[pos++] = 'o'; 248 | expected[pos++] = 'n'; 249 | expected[pos++] = 't'; 250 | expected[pos++] = 'e'; 251 | expected[pos++] = 'n'; 252 | expected[pos++] = 't'; 253 | 254 | mpack_start_map(&writer, 3); 255 | expected[pos++] = '\x83'; 256 | 257 | mpack_write_cstr(&writer, "path"); 258 | expected[pos++] = '\xa4'; 259 | expected[pos++] = 'p'; 260 | expected[pos++] = 'a'; 261 | expected[pos++] = 't'; 262 | expected[pos++] = 'h'; 263 | 264 | mpack_write_cstr(&writer, "IMG_2445.JPG"); 265 | expected[pos++] = '\xac'; 266 | expected[pos++] = 'I'; 267 | expected[pos++] = 'M'; 268 | expected[pos++] = 'G'; 269 | expected[pos++] = '_'; 270 | expected[pos++] = '2'; 271 | expected[pos++] = '4'; 272 | expected[pos++] = '4'; 273 | expected[pos++] = '5'; 274 | expected[pos++] = '.'; 275 | expected[pos++] = 'J'; 276 | expected[pos++] = 'P'; 277 | expected[pos++] = 'G'; 278 | 279 | mpack_write_cstr(&writer, "parent"); 280 | expected[pos++] = '\xa6'; 281 | expected[pos++] = 'p'; 282 | expected[pos++] = 'a'; 283 | expected[pos++] = 'r'; 284 | expected[pos++] = 'e'; 285 | expected[pos++] = 'n'; 286 | expected[pos++] = 't'; 287 | 288 | mpack_write_bin(&writer, rid, sizeof(rid)); 289 | expected[pos++] = '\xc4'; 290 | expected[pos++] = '\x10'; 291 | for (i = 0; i < 16; ++i) 292 | expected[pos++] = '\x00'; 293 | 294 | mpack_write_cstr(&writer, "pass"); 295 | expected[pos++] = '\xa4'; 296 | expected[pos++] = 'p'; 297 | expected[pos++] = 'a'; 298 | expected[pos++] = 's'; 299 | expected[pos++] = 's'; 300 | 301 | mpack_write_int(&writer, 0); 302 | expected[pos++] = '\x00'; 303 | 304 | mpack_finish_map(&writer); 305 | 306 | //mpack_finish_map(&writer); 307 | mpack_complete_map(&writer); 308 | 309 | TEST_TRUE(pos <= sizeof(expected)); 310 | size_t used = mpack_writer_buffer_used(&writer); 311 | 312 | /* 313 | printf("actual %zi expected %zi\n", used, pos); 314 | for (i = 0; i < used; ++i) { 315 | printf("%02hhx ", buf[i]); 316 | if (((i+1) % 16)==0) 317 | printf("\n"); 318 | } 319 | printf("\n"); 320 | printf("\n"); 321 | for (i = 0; i < pos; ++i) { 322 | printf("%02hhx ", expected[i]); 323 | if (((i+1) % 16)==0) 324 | printf("\n"); 325 | } 326 | printf("\n"); 327 | */ 328 | 329 | TEST_WRITER_DESTROY_NOERROR(&writer); 330 | TEST_TRUE(used == pos); 331 | TEST_TRUE(0 == memcmp(buf, expected, used)); 332 | } 333 | 334 | static void test_builder_add_expected_str(char* expected, size_t* pos, const char* str, uint32_t length) { 335 | if (length <= 31) { 336 | expected[(*pos)++] = (char)((uint32_t)'\xa0' + length); 337 | } else if (length <= MPACK_UINT8_MAX) { 338 | expected[(*pos)++] = '\xd9'; 339 | expected[(*pos)++] = (char)(uint8_t)length; 340 | } else { 341 | expected[(*pos)++] = '\xda'; 342 | expected[(*pos)++] = (char)(uint8_t)(length >> 8); 343 | expected[(*pos)++] = (char)(uint8_t)(length); 344 | } 345 | memcpy(expected + *pos, str, length); 346 | *pos += length; 347 | } 348 | 349 | static void test_builder_strings_length(uint32_t length) { 350 | static char buf[16*1024]; 351 | mpack_writer_t writer; 352 | mpack_writer_init(&writer, buf, sizeof(buf)); 353 | 354 | char expected[sizeof(buf)]; 355 | size_t pos = 0; 356 | 357 | static char str[1024]; 358 | TEST_TRUE(length <= sizeof(str)); 359 | memset(str, 'a', length); 360 | size_t depth = 2; 361 | 362 | size_t i; 363 | for (i = 0; i < depth; ++i) { 364 | mpack_build_array(&writer); 365 | expected[pos++] = '\x93'; 366 | mpack_write_str(&writer, str, length); 367 | test_builder_add_expected_str(expected, &pos, str, length); 368 | } 369 | 370 | mpack_write_str(&writer, str, length); 371 | test_builder_add_expected_str(expected, &pos, str, length); 372 | 373 | for (i = 0; i < depth; ++i) { 374 | mpack_write_str(&writer, str, length); 375 | test_builder_add_expected_str(expected, &pos, str, length); 376 | mpack_complete_array(&writer); 377 | } 378 | 379 | TEST_TRUE(pos <= sizeof(expected)); 380 | size_t used = mpack_writer_buffer_used(&writer); 381 | 382 | /* 383 | printf("actual %zi expected %zi\n", used, pos); 384 | for (i = 0; i < used; ++i) { 385 | printf("%02hhx ", buf[i]); 386 | if (((i+1) % 16)==0) 387 | printf("\n"); 388 | } 389 | printf("\n"); 390 | printf("\n"); 391 | for (i = 0; i < pos; ++i) { 392 | printf("%02hhx ", expected[i]); 393 | if (((i+1) % 16)==0) 394 | printf("\n"); 395 | } 396 | printf("\n"); 397 | */ 398 | 399 | TEST_WRITER_DESTROY_NOERROR(&writer); 400 | TEST_TRUE(used == pos); 401 | TEST_TRUE(0 == memcmp(buf, expected, used)); 402 | } 403 | 404 | static void test_builder_strings(void) { 405 | test_builder_strings_length(3); 406 | test_builder_strings_length(17); 407 | test_builder_strings_length(32); 408 | test_builder_strings_length(129); 409 | test_builder_strings_length(457); 410 | } 411 | 412 | static void test_builder_resolve_error(void) { 413 | static char buf[5]; 414 | mpack_writer_t writer; 415 | mpack_writer_init(&writer, buf, sizeof(buf)); 416 | mpack_build_array(&writer); 417 | mpack_write_cstr(&writer, "Hello world!"); 418 | mpack_complete_array(&writer); 419 | TEST_WRITER_DESTROY_ERROR(&writer, mpack_error_too_big); 420 | } 421 | 422 | void test_builder(void) { 423 | test_builder_basic(); 424 | test_builder_repeat(); 425 | test_builder_nested(); 426 | test_builder_deep(); 427 | test_builder_large(); 428 | test_builder_content(); 429 | test_builder_strings(); 430 | test_builder_resolve_error(); 431 | } 432 | #endif 433 | -------------------------------------------------------------------------------- /docs/reader.md: -------------------------------------------------------------------------------- 1 | # Using the Reader API 2 | 3 | The Reader API is used to parse MessagePack incrementally with no per-element memory usage. Elements can be read one-by-one, and the content of strings and binary blobs can be read in chunks. 4 | 5 | Reading incrementally is much more difficult and verbose than using the [Node API](docs/node.md). If you are not constrained in memory or performance, you should use the Node API. 6 | 7 | ## The Basics 8 | 9 | A reader is first initialized against a data source. This can be a chunk of data in memory, or it can be a file or stream. A reader that uses a file or stream will read data in chunks into an internal buffer. This allows parsing very large messages efficiently with minimal memory usage. 10 | 11 | Once initialized, the fundamental operation of a reader is to read a tag. A tag is a value struct of type `mpack_tag_t` that contains the value of a single element, or the metadata for a compound element. 12 | 13 | Here's a minimal example that parses the first element out of a chunk of MessagePack data. 14 | 15 | ```C 16 | bool parse_first_element(const char* data, size_t length) { 17 | mpack_reader_t reader; 18 | mpack_reader_init_data(&reader, data, length); 19 | 20 | mpack_tag_t tag = mpack_read_tag(&reader); 21 | do_something_with_tag(&tag); 22 | 23 | return mpack_reader_destroy(&reader) == mpack_ok; 24 | }; 25 | ``` 26 | 27 | The struct `mpack_tag_t` contains the element type, accessible with `mpack_tag_type()`. Depending on the type, you can access the value with tag accessor functions such as `mpack_tag_bool_value()` or `mpack_tag_array_count()`. 28 | 29 | Of course, parsing single values isn't terribly useful. You'll need to know how to parse strings, maps and arrays. 30 | 31 | ## Compound Types 32 | 33 | Compound types are either containers (map, array) or data chunks (strings, binary blobs). For any compound type, the tag only contains the compound type's size. You still need to read the contained data, and then you need to let the reader know that you're done reading the element (so it can check that you did it correctly.) 34 | 35 | ### Containers 36 | 37 | To parse a container, you must read as many additional elements as are specified by the tag's count. Note that a map tag specifies the number of key value pairs it contains, so you must actually read double the tag's count. You must then call `mpack_done_array()` or `mpack_done_map()` so that the reader can verify that you read the correct number of elements. 38 | 39 | Here's an example of a function that reads a tag from a reader, then reads all of the contained elements recursively: 40 | 41 | ```C 42 | void parse_element(mpack_reader_t* reader, int depth) { 43 | if (depth >= 32) { // critical check! 44 | mpack_reader_flag_error(reader, mpack_error_too_big); 45 | return; 46 | } 47 | 48 | mpack_tag_t tag = mpack_read_tag(&reader); 49 | if (mpack_reader_error(reader) != mpack_ok) 50 | return; 51 | 52 | do_something_with_tag(&tag); 53 | 54 | if (mpack_tag_type(&tag) == mpack_type_array) { 55 | for (uint32_t i = mpack_tag_array_count(&tag); i > 0; --i) { 56 | parse_element(reader, depth + 1); 57 | if (mpack_reader_error(reader) != mpack_ok) // critical check! 58 | break; 59 | } 60 | mpack_done_array(reader); 61 | } 62 | 63 | if (mpack_tag_type(&tag) == mpack_type_map) { 64 | for (uint32_t i = mpack_tag_map_count(&tag); i > 0; --i) { 65 | parse_element(reader, depth + 1); 66 | parse_element(reader, depth + 1); 67 | if (mpack_reader_error(reader) != mpack_ok) // critical check! 68 | break; 69 | } 70 | mpack_done_map(reader); 71 | } 72 | } 73 | ``` 74 | 75 | WARNING: It is critical that we check for errors during each iteration of our array and map loops. If we skip these checks, malicious data could claim to contain billions of elements, throwing us into a very long loop. These checks allow us to break out as soon as we run out of data. Since this function is recursive, we've also added a depth limit to prevent bad data from causing a stack overflow. 76 | 77 | (The [Node API](docs/node.md) is not vulnerable to such problems. In fact, it can break out of such bad data even sooner. The Node API will automatically flag an error as soon as it realizes that there aren't enough bytes left in the message to fill the remaining elements of all open compound types. If possible, consider using the Node API.) 78 | 79 | Unfortunately, the above code has a problem: it does not handle strings or binary blobs. If it encounters them, it will fail because it's not reading their contents. 80 | 81 | ### Compound Data Chunks 82 | 83 | The data for chunks such as strings and binary blobs must be read separately. Bytes can be read across multiple read calls, as long the total number of bytes you read matches the number of bytes contained in the element. And as with containers, you must call `mpack_done_str()` or `mpack_done_bin()` so that the reader can verify that you read the correct total number of bytes. 84 | 85 | We can add support for strings to the previous example with the below code. In this example, we read the string incrementally in 128 byte chunks to keep memory usage at an absolute minimum: 86 | 87 | ```C 88 | if (mpack_tag_type(&tag) == mpack_type_str) { 89 | uint32_t length = mpack_tag_str_length(&tag); 90 | char buffer[128]; 91 | 92 | while (length > 0) { 93 | size_t step = (length < sizeof(buffer)) ? length : sizeof(buffer); 94 | mpack_read_bytes(reader, buffer, sizeof(buffer); 95 | if (mpack_reader_error(reader) != mpack_ok) // critical check! 96 | break; 97 | 98 | do_something_with_partial_string(buffer, step); 99 | } 100 | 101 | mpack_done_str(reader); 102 | } 103 | ``` 104 | 105 | As above, we need a safety check to ensure that bad data cannot get us stuck in a loop. 106 | 107 | Code for `mpack_type_bin` is identical except that the type and function names contain `bin` instead of `str`. 108 | 109 | ### In-Place strings 110 | 111 | The MPack reader supports accessing the data contained in compound data types directly out of the reader's buffer. This can provide zero-copy access to strings and binary blobs, provided they fit within the buffer. 112 | 113 | To access string or bin data in-place, use `mpack_read_bytes_inplace()` (or `mpack_read_utf8_inplace()` to also check a string's UTF-8 encoding.) This provides a pointer directly into the reader's buffer, moving data around as necessary to make it fit. The previous code could be replaced with: 114 | 115 | ```C 116 | if (mpack_tag_type(&tag) == mpack_type_str) { 117 | uint32_t length = mpack_tag_str_length(&tag); 118 | const char* data = mpack_read_bytes_inplace(reader, length); 119 | if (mpack_reader_error(reader) != mpack_ok) 120 | return; 121 | do_something_with_string(data, length); 122 | mpack_done_str(reader); 123 | } 124 | ``` 125 | 126 | There's an important caveat here though: this will flag an error if the string does not fit in the buffer. If you are decoding a chunk of MessagePack data in memory (without a fill function), then this is not a problem, as it would simply mean that the message was truncated. But if you are decoding from a file or stream, you need to account for the fact that the string may not fit in the buffer. 127 | 128 | To work around this, you can provide two paths for reading data depending on the size of the string. MPack provides a helper `mpack_should_read_bytes_inplace()` to tell you if it's a good idea to read in-place. Here's another example that uses zero-copy strings where possible, and falls back to an allocation otherwise: 129 | 130 | ```C 131 | if (mpack_tag_type(&tag) == mpack_type_str) { 132 | uint32_t length = mpack_tag_str_length(&tag); 133 | if (length >= 16 * 1024) { // critical check! limit length to avoid a huge allocation 134 | mpack_reader_flag_error(reader, mpack_error_too_big); 135 | return; 136 | } 137 | 138 | if (mpack_should_read_bytes_inplace(reader, length)) { 139 | const char* data = mpack_read_bytes_inplace(reader, length); 140 | if (mpack_reader_error(reader) != mpack_ok) 141 | return; 142 | do_something_with_string(data, length); 143 | 144 | } else { 145 | char* data = malloc(length); 146 | mpack_read_bytes(reader, data, length); 147 | if (mpack_reader_error(reader) != mpack_ok) { 148 | free(data); 149 | return; 150 | } 151 | do_something_with_string(data, length); 152 | free(data); 153 | } 154 | 155 | mpack_done_str(reader); 156 | } 157 | ``` 158 | 159 | ## Expected Types 160 | 161 | When parsing data of expected types from a dynamic stream, you will likely want to hardcode type checks before each access. For example: 162 | 163 | ```C 164 | // get a bool 165 | mpack_tag_t tag = mpack_read_tag(reader); 166 | if (mpack_tag_type(&tag) != mpack_type_bool) { 167 | perror("not a bool!"); 168 | return false; 169 | } 170 | bool value = mpack_tag_bool_value(&tag); 171 | ``` 172 | 173 | MPack provides helper functions that perform these checks in the [Expect API](docs/expect.md). The above code for example could be replaced by a single call to `mpack_expect_bool()`. If the value is not a bool, it will flag an error and return `false`. This is the incremental reader analogue to `mpack_write_bool()`. 174 | 175 | These helpers are strongly recommended because they will perform range checks and lossless conversions where needed. For example, suppose you want to read an `int`. A positive integer could be stored in a tag as two different types (`mpack_type_int` or `mpack_type_uint`), and the value of that integer could be outside the range of an `int`. The functions `mpack_expect_int()` supports both types and performs the appropriate range checks. It will convert losslessly as needed and flag an error if the value doesn't fit. 176 | 177 | Note that when using the expect functions, you still need to read the contents of compound types and call the corresponding done function. A call to an expect function only replaces a call to `mpack_read_tag()`. For example: 178 | 179 | ```C 180 | uint32_t length = mpack_expect_str(reader); 181 | const char* data = mpack_read_bytes_inplace(reader, length); 182 | if (mpack_reader_error(reader) == mpack_ok) 183 | do_something_with_string(data, length); 184 | mpack_done_str(reader); 185 | ``` 186 | 187 | If you want to always check that types match what you expect, head on over to the [Expect API](docs/expect.md) and use these type checking functions. But if you are still interested in handling the dynamic runtime type of elements, read on. 188 | 189 | ## Event-Based Parser 190 | 191 | In this example, we'll implement a SAX-style event-based parser for blobs of MessagePack using the Reader API. This is a good example of non-trivial MPack usage that handles the dynamic type of incrementally parsed MessagePack data. It's easy to convert any incremental parser to an event-based parser, as we'll soon see. 192 | 193 | First let's define a header file with a list of callbacks for our parser, and our single function to launch the parser: 194 | 195 | ```C 196 | typedef struct sax_callbacks_t { 197 | void (*nil_element)(void* context); 198 | void (*bool_element)(void* context, int64_t value); 199 | void (*int_element)(void* context, int64_t value); 200 | void (*uint_element)(void* context, uint64_t value); 201 | void (*string_element)(void* context, const char* data, uint32_t length); 202 | 203 | void (*start_map)(void* context, uint32_t pair_count); 204 | void (*start_array)(void* context, uint32_t element_count); 205 | void (*finish_map)(void* context); 206 | void (*finish_array)(void* context); 207 | } sax_callbacks_t; 208 | 209 | /** 210 | * Parse a blob of MessagePack data, calling the appropriate callback for each 211 | * element encountered. 212 | * 213 | * @return true if successful, false if any error occurs. 214 | */ 215 | bool parse_messagepack(const char* data, size_t length, 216 | const sax_callbacks_t* callbacks, void* context); 217 | ``` 218 | 219 | Next we'll define our `parse_messagepack()` function in a corresponding source file to set our our reader. This will wrap another function called `parse_element()`. 220 | 221 | ```C 222 | static void parse_element(mpack_reader_t* reader, int depth, 223 | const sax_callbacks_t* callbacks, void* context); 224 | 225 | bool parse_messagepack(const char* data, size_t length, 226 | const sax_callbacks_t* callbacks, void* context) 227 | { 228 | mpack_reader_t reader; 229 | mpack_reader_init_data(&reader, data, length); 230 | parse_element(&reader, 0, callbacks, context); 231 | return mpack_ok == mpack_reader_destroy(&reader); 232 | } 233 | ``` 234 | 235 | We'll make `parse_element()` recursive to keep things simple. This makes it extremely straightforward to implement. We just parse a tag, switch on the type, and call the appropriate callback function. 236 | 237 | Note that we can access all strings in-place because the source data is a single chunk of contiguous memory. 238 | 239 | ```C 240 | static void parse_element(mpack_reader_t* reader, int depth, 241 | const sax_callbacks_t* callbacks, void* context) 242 | { 243 | if (depth >= 32) { // critical check! 244 | mpack_reader_flag_error(reader, mpack_error_too_big); 245 | return; 246 | } 247 | 248 | mpack_tag_t tag = mpack_read_tag(reader); 249 | if (mpack_reader_error(reader) != mpack_ok) 250 | return; 251 | 252 | switch (mpack_tag_type(&tag)) { 253 | case mpack_type_nil: 254 | callbacks->nil_element(context); 255 | break; 256 | case mpack_type_bool: 257 | callbacks->bool_element(context, mpack_tag_bool_value(&tag)); 258 | break; 259 | case mpack_type_int: 260 | callbacks->int_element(context, mpack_tag_int_value(&tag)); 261 | break; 262 | case mpack_type_uint: 263 | callbacks->uint_element(context, mpack_tag_uint_value(&tag)); 264 | break; 265 | 266 | case mpack_type_str: { 267 | uint32_t length = mpack_tag_str_length(&tag); 268 | const char* data = mpack_read_bytes_inplace(reader, length); 269 | callbacks->string_element(context, data, length); 270 | mpack_done_str(reader); 271 | break; 272 | } 273 | 274 | case mpack_type_array: { 275 | uint32_t count = mpack_tag_array_count(&tag); 276 | callbacks->start_array(context, count); 277 | while (count-- > 0) { 278 | parse_element(reader, depth + 1, callbacks, context); 279 | if (mpack_reader_error(reader) != mpack_ok) // critical check! 280 | break; 281 | } 282 | callbacks->finish_array(context); 283 | mpack_done_array(reader); 284 | break; 285 | } 286 | 287 | case mpack_type_map: { 288 | uint32_t count = mpack_tag_map_count(&tag); 289 | callbacks->start_map(context, count); 290 | while (count-- > 0) { 291 | parse_element(reader, depth + 1, callbacks, context); 292 | parse_element(reader, depth + 1, callbacks, context); 293 | if (mpack_reader_error(reader) != mpack_ok) // critical check! 294 | break; 295 | } 296 | callbacks->finish_map(context); 297 | mpack_done_map(reader); 298 | break; 299 | } 300 | 301 | default: 302 | mpack_reader_flag_error(reader, mpack_error_unsupported); 303 | break; 304 | } 305 | } 306 | ``` 307 | 308 | As above, the error checks within loops are critical to keep the parser safe against untrusted data. 309 | 310 | This is all that is needed to convert MPack into an event-based parser. 311 | -------------------------------------------------------------------------------- /test/unit/src/test-buffer.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | * this software and associated documentation files (the "Software"), to deal in 6 | * the Software without restriction, including without limitation the rights to 7 | * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 8 | * the Software, and to permit persons to whom the Software is furnished to do so, 9 | * subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 16 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 17 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 18 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 19 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 20 | */ 21 | 22 | #include "test-buffer.h" 23 | 24 | #include "test-expect.h" 25 | #include "test-write.h" 26 | 27 | static const char test_numbers[] = 28 | 29 | "\x02" // 2 30 | "\x11" // 17 31 | "\x1d" // 29 32 | "\x2b" // 43 33 | "\x3b" // 59 34 | "\x47" // 71 35 | "\x59" // 89 36 | "\x65" // 101 37 | 38 | "\xcc\x83" // 131 39 | "\xcc\x95" // 149 40 | "\xcc\x9d" // 157 41 | "\xcc\xad" // 173 42 | "\xcc\xbf" // 191 43 | "\xcc\xc7" // 199 44 | "\xcc\xdf" // 223 45 | "\xcc\xe3" // 227 46 | 47 | "\xcd\x01\x01" // 257 48 | "\xcd\x1d\x5d" // 7517 49 | "\xcd\x39\xaf" // 14767 50 | "\xcd\x56\x0b" // 22027 51 | "\xcd\x72\x55" // 29269 52 | "\xcd\x8e\xab" // 36523 53 | "\xcd\xab\x01" // 43777 54 | "\xcd\xc7\x57" // 51031 55 | 56 | "\xce\x00\x01\x00\x01" // 65537 57 | "\xce\x1c\x72\xaa\xb3" // 477276851 58 | "\xce\x38\xe4\x55\x59" // 954488153 59 | "\xce\x55\x56\x00\x19" // 1431699481 60 | "\xce\x71\xc7\xaa\xab" // 1908910763 61 | "\xce\x8e\x39\x55\x77" // 2386122103 62 | "\xce\xaa\xab\x00\x17" // 2863333399 63 | "\xce\xc7\x1c\xaa\xa9" // 3340544681 64 | 65 | "\xcf\x00\x00\x00\x01\x00\x00\x00\x0f" // 4294967311 66 | "\xcf\x1a\xf2\x86\xbd\x86\xbc\xa2\x67" // 1941762537917555303 67 | "\xcf\x35\xe5\x0d\x7a\x0d\x79\x44\x0f" // 3883525071540143119 68 | "\xcf\x50\xd7\x94\x36\x94\x35\xe4\x51" // 5825287605162730577 69 | "\xcf\x6b\xca\x1a\xf3\x1a\xf2\x88\x31" // 7767050138785318961 70 | "\xcf\x86\xbc\xa1\xaf\xa1\xaf\x28\x3f" // 9708812672407906367 71 | "\xcf\xa1\xaf\x28\x6c\x28\x6b\xc8\x11" // 11650575206030493713 72 | "\xcf\xbc\xa1\xaf\x28\xaf\x28\x68\x03" // 13592337739653081091 73 | 74 | ; 75 | 76 | static const char test_strings[] = 77 | "\x9F" 78 | "\xA0" 79 | "\xA1""a" 80 | "\xA2""ab" 81 | "\xA3""abc" 82 | "\xA4""abcd" 83 | "\xA5""abcde" 84 | "\xA6""abcdef" 85 | "\xA7""abcdefg" 86 | "\xA8""abcdefgh" 87 | "\xA9""abcdefghi" 88 | "\xAA""abcdefghij" 89 | "\xAB""abcdefghijk" 90 | "\xAC""abcdefghijkl" 91 | "\xAD""abcdefghijklm" 92 | "\xAE""abcdefghijklmn"; 93 | 94 | 95 | // a semi-random list of buffer sizes we will test with. each buffer 96 | // test is run with each of these buffer sizes to test the fill and 97 | // flush functions. 98 | static const size_t test_buffer_sizes[] = { 99 | 32, 33, 34, 35, 36, 37, 39, 43, 48, 51, 100 | 52, 53, 57, 59, 64, 67, 89, 127, 128, 101 | 129, 131, 160, 163, 191, 192, 193, 102 | 251, 256, 257, 509, 512, 521, 103 | 1021, 1024, 1031, 2039, 2048, 2053, 104 | #ifndef __AVR__ 105 | 4093, 4096, 4099, 7919, 8192, 106 | 6384, 32768, 107 | #endif 108 | }; 109 | 110 | #if MPACK_READER 111 | typedef struct test_fill_state_t { 112 | const char* data; 113 | size_t remaining; 114 | } test_fill_state_t; 115 | 116 | static size_t test_buffer_fill(mpack_reader_t* reader, char* buffer, size_t count) { 117 | test_fill_state_t* state = (test_fill_state_t*)reader->context; 118 | if (state->remaining < count) 119 | count = state->remaining; 120 | memcpy(buffer, state->data, count); 121 | state->data += count; 122 | state->remaining -= count; 123 | return count; 124 | } 125 | #endif 126 | 127 | #if MPACK_WRITER 128 | typedef struct test_flush_state_t { 129 | char* data; 130 | size_t remaining; 131 | } test_flush_state_t; 132 | 133 | static void test_buffer_flush(mpack_writer_t* writer, const char* buffer, size_t count) { 134 | test_flush_state_t* state = (test_flush_state_t*)writer->context; 135 | if (state->remaining < count) { 136 | mpack_writer_flag_error(writer, mpack_error_too_big); 137 | return; 138 | } 139 | memcpy(state->data, buffer, count); 140 | state->data += count; 141 | state->remaining -= count; 142 | } 143 | #endif 144 | 145 | #if MPACK_EXPECT 146 | static void test_expect_buffer_values(mpack_reader_t* reader) { 147 | TEST_READ_NOERROR(reader, 2 == mpack_expect_u8(reader)); 148 | TEST_READ_NOERROR(reader, 17 == mpack_expect_u8(reader)); 149 | TEST_READ_NOERROR(reader, 29 == mpack_expect_u8(reader)); 150 | TEST_READ_NOERROR(reader, 43 == mpack_expect_u8(reader)); 151 | TEST_READ_NOERROR(reader, 59 == mpack_expect_u8(reader)); 152 | TEST_READ_NOERROR(reader, 71 == mpack_expect_u8(reader)); 153 | TEST_READ_NOERROR(reader, 89 == mpack_expect_u8(reader)); 154 | TEST_READ_NOERROR(reader, 101 == mpack_expect_u8(reader)); 155 | 156 | TEST_READ_NOERROR(reader, 131 == mpack_expect_u8(reader)); 157 | TEST_READ_NOERROR(reader, 149 == mpack_expect_u8(reader)); 158 | TEST_READ_NOERROR(reader, 157 == mpack_expect_u8(reader)); 159 | TEST_READ_NOERROR(reader, 173 == mpack_expect_u8(reader)); 160 | TEST_READ_NOERROR(reader, 191 == mpack_expect_u8(reader)); 161 | TEST_READ_NOERROR(reader, 199 == mpack_expect_u8(reader)); 162 | TEST_READ_NOERROR(reader, 223 == mpack_expect_u8(reader)); 163 | TEST_READ_NOERROR(reader, 227 == mpack_expect_u8(reader)); 164 | 165 | TEST_READ_NOERROR(reader, 257 == mpack_expect_u16(reader)); 166 | TEST_READ_NOERROR(reader, 7517 == mpack_expect_u16(reader)); 167 | TEST_READ_NOERROR(reader, 14767 == mpack_expect_u16(reader)); 168 | TEST_READ_NOERROR(reader, 22027 == mpack_expect_u16(reader)); 169 | TEST_READ_NOERROR(reader, 29269 == mpack_expect_u16(reader)); 170 | TEST_READ_NOERROR(reader, 36523 == mpack_expect_u16(reader)); 171 | TEST_READ_NOERROR(reader, 43777 == mpack_expect_u16(reader)); 172 | TEST_READ_NOERROR(reader, 51031 == mpack_expect_u16(reader)); 173 | 174 | TEST_READ_NOERROR(reader, 65537 == mpack_expect_u32(reader)); 175 | TEST_READ_NOERROR(reader, 477276851 == mpack_expect_u32(reader)); 176 | TEST_READ_NOERROR(reader, 954488153 == mpack_expect_u32(reader)); 177 | TEST_READ_NOERROR(reader, 1431699481 == mpack_expect_u32(reader)); 178 | TEST_READ_NOERROR(reader, 1908910763 == mpack_expect_u32(reader)); 179 | 180 | // when using UINT32_C() and compiling the test suite as c++, gcc complains: 181 | // error: this decimal constant is unsigned only in ISO C90 [-Werror] 182 | TEST_READ_NOERROR(reader, MPACK_UINT64_C(2386122103) == mpack_expect_u32(reader)); 183 | TEST_READ_NOERROR(reader, MPACK_UINT64_C(2863333399) == mpack_expect_u32(reader)); 184 | TEST_READ_NOERROR(reader, MPACK_UINT64_C(3340544681) == mpack_expect_u32(reader)); 185 | 186 | TEST_READ_NOERROR(reader, MPACK_UINT64_C(4294967311) == mpack_expect_u64(reader)); 187 | TEST_READ_NOERROR(reader, MPACK_UINT64_C(1941762537917555303) == mpack_expect_u64(reader)); 188 | TEST_READ_NOERROR(reader, MPACK_UINT64_C(3883525071540143119) == mpack_expect_u64(reader)); 189 | TEST_READ_NOERROR(reader, MPACK_UINT64_C(5825287605162730577) == mpack_expect_u64(reader)); 190 | TEST_READ_NOERROR(reader, MPACK_UINT64_C(7767050138785318961) == mpack_expect_u64(reader)); 191 | TEST_READ_NOERROR(reader, MPACK_UINT64_C(9708812672407906367) == mpack_expect_u64(reader)); 192 | TEST_READ_NOERROR(reader, MPACK_UINT64_C(11650575206030493713) == mpack_expect_u64(reader)); 193 | TEST_READ_NOERROR(reader, MPACK_UINT64_C(13592337739653081091) == mpack_expect_u64(reader)); 194 | } 195 | #endif 196 | 197 | #if MPACK_WRITER 198 | static void test_write_buffer_values(mpack_writer_t* writer) { 199 | TEST_WRITE_NOERROR(writer, mpack_write_u8(writer, 2)); 200 | TEST_WRITE_NOERROR(writer, mpack_write_u8(writer, 17)); 201 | TEST_WRITE_NOERROR(writer, mpack_write_u8(writer, 29)); 202 | TEST_WRITE_NOERROR(writer, mpack_write_u8(writer, 43)); 203 | TEST_WRITE_NOERROR(writer, mpack_write_u8(writer, 59)); 204 | TEST_WRITE_NOERROR(writer, mpack_write_u8(writer, 71)); 205 | TEST_WRITE_NOERROR(writer, mpack_write_u8(writer, 89)); 206 | TEST_WRITE_NOERROR(writer, mpack_write_u8(writer, 101)); 207 | 208 | TEST_WRITE_NOERROR(writer, mpack_write_u8(writer, 131)); 209 | TEST_WRITE_NOERROR(writer, mpack_write_u8(writer, 149)); 210 | TEST_WRITE_NOERROR(writer, mpack_write_u8(writer, 157)); 211 | TEST_WRITE_NOERROR(writer, mpack_write_u8(writer, 173)); 212 | TEST_WRITE_NOERROR(writer, mpack_write_u8(writer, 191)); 213 | TEST_WRITE_NOERROR(writer, mpack_write_u8(writer, 199)); 214 | TEST_WRITE_NOERROR(writer, mpack_write_u8(writer, 223)); 215 | TEST_WRITE_NOERROR(writer, mpack_write_u8(writer, 227)); 216 | 217 | TEST_WRITE_NOERROR(writer, mpack_write_u16(writer, 257)); 218 | TEST_WRITE_NOERROR(writer, mpack_write_u16(writer, 7517)); 219 | TEST_WRITE_NOERROR(writer, mpack_write_u16(writer, 14767)); 220 | TEST_WRITE_NOERROR(writer, mpack_write_u16(writer, 22027)); 221 | TEST_WRITE_NOERROR(writer, mpack_write_u16(writer, 29269)); 222 | TEST_WRITE_NOERROR(writer, mpack_write_u16(writer, 36523)); 223 | TEST_WRITE_NOERROR(writer, mpack_write_u16(writer, 43777)); 224 | TEST_WRITE_NOERROR(writer, mpack_write_u16(writer, 51031)); 225 | 226 | TEST_WRITE_NOERROR(writer, mpack_write_u32(writer, 65537)); 227 | TEST_WRITE_NOERROR(writer, mpack_write_u32(writer, 477276851)); 228 | TEST_WRITE_NOERROR(writer, mpack_write_u32(writer, 954488153)); 229 | TEST_WRITE_NOERROR(writer, mpack_write_u32(writer, 1431699481)); 230 | TEST_WRITE_NOERROR(writer, mpack_write_u32(writer, 1908910763)); 231 | 232 | // when using UINT32_C() and compiling the test suite as c++, gcc complains: 233 | // error: this decimal constant is unsigned only in ISO C90 [-Werror] 234 | TEST_WRITE_NOERROR(writer, mpack_write_u32(writer, MPACK_UINT64_C(2386122103))); 235 | TEST_WRITE_NOERROR(writer, mpack_write_u32(writer, MPACK_UINT64_C(2863333399))); 236 | TEST_WRITE_NOERROR(writer, mpack_write_u32(writer, MPACK_UINT64_C(3340544681))); 237 | 238 | TEST_WRITE_NOERROR(writer, mpack_write_u64(writer, MPACK_UINT64_C(4294967311))); 239 | TEST_WRITE_NOERROR(writer, mpack_write_u64(writer, MPACK_UINT64_C(1941762537917555303))); 240 | TEST_WRITE_NOERROR(writer, mpack_write_u64(writer, MPACK_UINT64_C(3883525071540143119))); 241 | TEST_WRITE_NOERROR(writer, mpack_write_u64(writer, MPACK_UINT64_C(5825287605162730577))); 242 | TEST_WRITE_NOERROR(writer, mpack_write_u64(writer, MPACK_UINT64_C(7767050138785318961))); 243 | TEST_WRITE_NOERROR(writer, mpack_write_u64(writer, MPACK_UINT64_C(9708812672407906367))); 244 | TEST_WRITE_NOERROR(writer, mpack_write_u64(writer, MPACK_UINT64_C(11650575206030493713))); 245 | TEST_WRITE_NOERROR(writer, mpack_write_u64(writer, MPACK_UINT64_C(13592337739653081091))); 246 | } 247 | #endif 248 | 249 | #if MPACK_EXPECT 250 | static void test_expect_buffer(void) { 251 | size_t i; 252 | for (i = 0; i < sizeof(test_buffer_sizes) / sizeof(test_buffer_sizes[0]); ++i) { 253 | 254 | // initialize the reader with our buffer reader function 255 | mpack_reader_t reader; 256 | size_t size = test_buffer_sizes[i]; 257 | char* buffer = (char*)malloc(size); 258 | test_fill_state_t state = {test_numbers, sizeof(test_numbers) - 1}; 259 | mpack_reader_init(&reader, buffer, size, 0); 260 | mpack_reader_set_fill(&reader, test_buffer_fill); 261 | mpack_reader_set_context(&reader, &state); 262 | 263 | // read and destroy, ensuring no errors 264 | test_expect_buffer_values(&reader); 265 | TEST_READER_DESTROY_NOERROR(&reader); 266 | free(buffer); 267 | 268 | } 269 | } 270 | #endif 271 | 272 | #if MPACK_WRITER 273 | static void test_write_buffer(void) { 274 | size_t i; 275 | for (i = 0; i < sizeof(test_buffer_sizes) / sizeof(test_buffer_sizes[0]); ++i) { 276 | size_t size = test_buffer_sizes[i]; 277 | size_t output_size = 278 | #ifdef __AVR__ 279 | 0xfff 280 | #else 281 | 0xfffff 282 | #endif 283 | ; 284 | char* output = (char*)malloc(output_size); 285 | 286 | // initialize the writer with our buffer writer function 287 | mpack_writer_t writer; 288 | char* buffer = (char*)malloc(size); 289 | char* pos = output; 290 | test_flush_state_t state = {output, output_size}; 291 | mpack_writer_init(&writer, buffer, size); 292 | mpack_writer_set_flush(&writer, test_buffer_flush); 293 | mpack_writer_set_context(&writer, &state); 294 | 295 | // read and destroy, ensuring no errors 296 | test_write_buffer_values(&writer); 297 | TEST_WRITER_DESTROY_NOERROR(&writer); 298 | free(buffer); 299 | 300 | // check output 301 | TEST_TRUE(output_size - state.remaining == sizeof(test_numbers) - 1, 302 | "output contains %i bytes but %i were expected", 303 | (int)(pos - output), (int)sizeof(test_numbers) - 1); 304 | TEST_TRUE(memcmp(output, test_numbers, sizeof(test_numbers) - 1) == 0, 305 | "output does not match test buffer"); 306 | free(output); 307 | } 308 | } 309 | #endif 310 | 311 | #if MPACK_READER 312 | static void test_inplace_buffer(void) { 313 | size_t i; 314 | for (i = 0; i < sizeof(test_buffer_sizes) / sizeof(test_buffer_sizes[0]); ++i) { 315 | 316 | // initialize the reader with our buffer reader function 317 | mpack_reader_t reader; 318 | size_t size = test_buffer_sizes[i]; 319 | char* buffer = (char*)malloc(size); 320 | test_fill_state_t state = {test_strings, sizeof(test_strings) - 1}; 321 | mpack_reader_init(&reader, buffer, size, 0); 322 | mpack_reader_set_fill(&reader, test_buffer_fill); 323 | mpack_reader_set_context(&reader, &state); 324 | 325 | // read the array 326 | mpack_tag_t tag = mpack_read_tag(&reader); 327 | TEST_TRUE(tag.type == mpack_type_array, "wrong type: %i %s", (int)tag.type, mpack_type_to_string(tag.type)); 328 | TEST_TRUE(tag.v.n == 15, "wrong array count: %i", tag.v.n); 329 | 330 | // check each string, using inplace if it's less than or equal to the 331 | // length of the buffer size 332 | static const char* ref = "abcdefghijklmn"; 333 | const char* val; 334 | char r[15]; 335 | size_t j; 336 | for (j = 0; j < 15; ++j) { 337 | mpack_tag_t peek = mpack_peek_tag(&reader); 338 | tag = mpack_read_tag(&reader); 339 | TEST_TRUE(mpack_tag_equal(peek, tag), "peeked tag does not match read tag"); 340 | TEST_TRUE(tag.type == mpack_type_str, "wrong type: %i %s", (int)tag.type, mpack_type_to_string(tag.type)); 341 | TEST_TRUE(tag.v.l == j, "string is the wrong length: %i bytes", (int)tag.v.l); 342 | if (tag.v.l <= reader.size) { 343 | val = mpack_read_bytes_inplace(&reader, tag.v.l); 344 | } else { 345 | mpack_read_bytes(&reader, r, tag.v.l); 346 | val = r; 347 | } 348 | TEST_TRUE(memcmp(val, ref, tag.v.l) == 0, "strings do not match!"); 349 | mpack_done_str(&reader); 350 | } 351 | 352 | // destroy, ensuring no errors 353 | mpack_done_array(&reader); 354 | TEST_READER_DESTROY_NOERROR(&reader); 355 | free(buffer); 356 | 357 | } 358 | } 359 | #endif 360 | 361 | void test_buffers(void) { 362 | MPACK_UNUSED(test_numbers); 363 | MPACK_UNUSED(test_strings); 364 | MPACK_UNUSED(test_buffer_sizes); 365 | 366 | #if MPACK_EXPECT 367 | test_expect_buffer(); 368 | #endif 369 | #if MPACK_WRITER 370 | test_write_buffer(); 371 | #endif 372 | #if MPACK_READER 373 | test_inplace_buffer(); 374 | #endif 375 | } 376 | 377 | --------------------------------------------------------------------------------