├── debian ├── compat ├── docs ├── source │ └── format ├── libmtbl1.install ├── mtbl-bin.install ├── rules ├── libmtbl-dev.install ├── gbp.conf ├── copyright └── control ├── m4 ├── pkg.m4 ├── ax_pthread.m4 ├── ld-version-script.m4 └── .gitignore ├── mtbl ├── .gitignore ├── libmtbl.pc.in ├── crc32c_wrap.c ├── bytes.h ├── iter.c ├── threadpool.h ├── fixed.c ├── source.c ├── libmtbl.sym └── metadata.c ├── autogen.sh ├── src ├── .gitignore └── mtbl_info.c ├── t ├── fileset-partition │ ├── test.fileset │ ├── file1.mtbl │ ├── file2.mtbl │ └── file3.mtbl ├── test-gh1-lz4.data ├── test-gh1-zlib.data ├── fileset-filter-data │ ├── animals.fileset │ ├── animals-1.mtbl │ ├── animals-2.mtbl │ └── animals-3.mtbl ├── test-deb716628.data ├── test-gh1-lz4hc.data ├── test-gh1-snappy.data ├── test-verify-bad1.data ├── test-verify-bad2.data ├── test-verify-good1.data ├── test-foreign-prefix.data ├── test-verify-bad-index-block-offset.data ├── .gitignore ├── test-fileset-filter.sh ├── test-fileset-partition.sh ├── test-deb716628.sh ├── test-foreign-prefix.sh ├── test-gh1.sh ├── test-compression.sh ├── test-verify.sh ├── test-fixed.c ├── test-metadata.c ├── test-fileset-partition.c ├── test-vector.c └── test-block_builder.c ├── libmy ├── .gitignore ├── threadnum.h ├── crc32c.h ├── spooldir.h ├── threadnum.c ├── getenv_int.h ├── varint.h ├── atomic_ptr.h ├── lookup3.h ├── zonefile.h ├── read_bytes.h ├── COPYRIGHT ├── my_memory_barrier.h ├── my_fileset.h ├── b64_decode.h ├── b64_encode.h ├── my_byteorder.h ├── heap.h ├── my_rate.h ├── print_string.h ├── ubuf-pb.h ├── my_queue.c ├── m4 │ ├── pcap.m4 │ ├── my_pkg_config_files.m4 │ ├── protobuf-c.m4 │ ├── ld-version-script.m4 │ ├── ax_define_dir.m4 │ └── ax_prog_xsltproc.m4 ├── string_replace.h ├── crc32c.c ├── my_alloc.h ├── hex_decode.h ├── my_time.h ├── b32_decode.h ├── b32_encode.h ├── b64_encode.c ├── ubuf.h ├── b64_decode.c ├── my_queue.h ├── atomic.h ├── atomic_64.h ├── my_queue_mb.c ├── ip_arith.h ├── my_queue_mutex.c ├── zonefile.c ├── heap.c └── b32_encode.c ├── man ├── asciidoc.conf ├── mtbl_crc32c.3.txt ├── mtbl_verify.1.txt ├── mtbl_threadpool.3.txt ├── mtbl_fixed.3.txt ├── mtbl_iter.3.txt ├── mtbl_info.1.txt ├── mtbl_crc32c.3 ├── mtbl_varint.3.txt ├── mtbl_dump.1.txt ├── mtbl_verify.1 ├── mtbl_source.3.txt ├── mtbl_threadpool.3 ├── mtbl_fixed.3 ├── mtbl_metadata.3.txt ├── mtbl.7.txt ├── mtbl_iter.3 ├── mtbl_merge.1.txt ├── mtbl_info.1 ├── mtbl_reader.3.txt ├── mtbl_varint.3 ├── mtbl_dump.1 ├── mtbl_source.3 └── mtbl_metadata.3 ├── .gitignore ├── .travis.yml ├── mtbl.spec ├── COPYRIGHT ├── README.md └── configure.ac /debian/compat: -------------------------------------------------------------------------------- 1 | 10 2 | -------------------------------------------------------------------------------- /debian/docs: -------------------------------------------------------------------------------- 1 | README.md 2 | -------------------------------------------------------------------------------- /m4/pkg.m4: -------------------------------------------------------------------------------- 1 | ../libmy/m4/pkg.m4 -------------------------------------------------------------------------------- /mtbl/.gitignore: -------------------------------------------------------------------------------- 1 | libmtbl.pc 2 | -------------------------------------------------------------------------------- /debian/source/format: -------------------------------------------------------------------------------- 1 | 3.0 (quilt) 2 | -------------------------------------------------------------------------------- /m4/ax_pthread.m4: -------------------------------------------------------------------------------- 1 | ../libmy/m4/ax_pthread.m4 -------------------------------------------------------------------------------- /debian/libmtbl1.install: -------------------------------------------------------------------------------- 1 | usr/lib/*/*.so.* 2 | -------------------------------------------------------------------------------- /autogen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | exec autoreconf -fvi 3 | -------------------------------------------------------------------------------- /m4/ld-version-script.m4: -------------------------------------------------------------------------------- 1 | ../libmy/m4/ld-version-script.m4 -------------------------------------------------------------------------------- /debian/mtbl-bin.install: -------------------------------------------------------------------------------- 1 | usr/bin/* 2 | usr/share/man/man1/*.1 3 | -------------------------------------------------------------------------------- /src/.gitignore: -------------------------------------------------------------------------------- 1 | mtbl_dump 2 | mtbl_info 3 | mtbl_merge 4 | mtbl_verify 5 | -------------------------------------------------------------------------------- /debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | 3 | %: 4 | dh $@ --with autoreconf 5 | -------------------------------------------------------------------------------- /t/fileset-partition/test.fileset: -------------------------------------------------------------------------------- 1 | file1.mtbl 2 | file2.mtbl 3 | file3.mtbl 4 | -------------------------------------------------------------------------------- /libmy/.gitignore: -------------------------------------------------------------------------------- 1 | *.lo 2 | *.o 3 | .*swp 4 | .deps/ 5 | .dirstamp 6 | .libs/ 7 | -------------------------------------------------------------------------------- /t/test-gh1-lz4.data: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/farsightsec/mtbl/HEAD/t/test-gh1-lz4.data -------------------------------------------------------------------------------- /t/test-gh1-zlib.data: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/farsightsec/mtbl/HEAD/t/test-gh1-zlib.data -------------------------------------------------------------------------------- /m4/.gitignore: -------------------------------------------------------------------------------- 1 | libtool.m4 2 | ltoptions.m4 3 | ltsugar.m4 4 | ltversion.m4 5 | lt~obsolete.m4 6 | -------------------------------------------------------------------------------- /t/fileset-filter-data/animals.fileset: -------------------------------------------------------------------------------- 1 | animals-1.mtbl 2 | animals-2.mtbl 3 | animals-3.mtbl 4 | -------------------------------------------------------------------------------- /t/test-deb716628.data: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/farsightsec/mtbl/HEAD/t/test-deb716628.data -------------------------------------------------------------------------------- /t/test-gh1-lz4hc.data: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/farsightsec/mtbl/HEAD/t/test-gh1-lz4hc.data -------------------------------------------------------------------------------- /t/test-gh1-snappy.data: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/farsightsec/mtbl/HEAD/t/test-gh1-snappy.data -------------------------------------------------------------------------------- /t/test-verify-bad1.data: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/farsightsec/mtbl/HEAD/t/test-verify-bad1.data -------------------------------------------------------------------------------- /t/test-verify-bad2.data: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/farsightsec/mtbl/HEAD/t/test-verify-bad2.data -------------------------------------------------------------------------------- /t/test-verify-good1.data: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/farsightsec/mtbl/HEAD/t/test-verify-good1.data -------------------------------------------------------------------------------- /t/test-foreign-prefix.data: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/farsightsec/mtbl/HEAD/t/test-foreign-prefix.data -------------------------------------------------------------------------------- /t/fileset-partition/file1.mtbl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/farsightsec/mtbl/HEAD/t/fileset-partition/file1.mtbl -------------------------------------------------------------------------------- /t/fileset-partition/file2.mtbl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/farsightsec/mtbl/HEAD/t/fileset-partition/file2.mtbl -------------------------------------------------------------------------------- /t/fileset-partition/file3.mtbl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/farsightsec/mtbl/HEAD/t/fileset-partition/file3.mtbl -------------------------------------------------------------------------------- /t/fileset-filter-data/animals-1.mtbl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/farsightsec/mtbl/HEAD/t/fileset-filter-data/animals-1.mtbl -------------------------------------------------------------------------------- /t/fileset-filter-data/animals-2.mtbl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/farsightsec/mtbl/HEAD/t/fileset-filter-data/animals-2.mtbl -------------------------------------------------------------------------------- /t/fileset-filter-data/animals-3.mtbl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/farsightsec/mtbl/HEAD/t/fileset-filter-data/animals-3.mtbl -------------------------------------------------------------------------------- /t/test-verify-bad-index-block-offset.data: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/farsightsec/mtbl/HEAD/t/test-verify-bad-index-block-offset.data -------------------------------------------------------------------------------- /debian/libmtbl-dev.install: -------------------------------------------------------------------------------- 1 | usr/include/mtbl.h 2 | usr/lib/*/*.a 3 | usr/lib/*/*.so 4 | usr/lib/*/pkgconfig/libmtbl.pc 5 | usr/share/man/man3/*.3 6 | usr/share/man/man7/*.7 7 | -------------------------------------------------------------------------------- /libmy/threadnum.h: -------------------------------------------------------------------------------- 1 | #ifndef MY_THREADNUM_H 2 | #define MY_THREADNUM_H 3 | 4 | void my_threadnum_register(void); 5 | int my_threadnum(void); 6 | 7 | #endif /* MY_THREADNUM_H */ 8 | -------------------------------------------------------------------------------- /debian/gbp.conf: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | debian-branch = fsi-debian 3 | upstream-branch = master 4 | upstream-tag = tags/v%(version)s 5 | pristine-tar = False 6 | 7 | [dch] 8 | commit-msg = debian/changelog: %(version)s 9 | -------------------------------------------------------------------------------- /t/.gitignore: -------------------------------------------------------------------------------- 1 | test-block_builder 2 | test-compression 3 | test-crc32c 4 | test-fileset-partition 5 | test-fixed 6 | test-iter-seek 7 | test-metadata 8 | test-sorted-merge 9 | test-varint 10 | test-vector 11 | test-fileset-filter 12 | -------------------------------------------------------------------------------- /libmy/crc32c.h: -------------------------------------------------------------------------------- 1 | #ifndef MY_CRC32C_H 2 | #define MY_CRC32C_H 3 | 4 | #include 5 | 6 | typedef uint32_t (*my_crc32c_fp)(const uint8_t *buffer, size_t length); 7 | 8 | extern my_crc32c_fp my_crc32c; 9 | 10 | #endif /* MY_CRC32C_H */ 11 | -------------------------------------------------------------------------------- /man/asciidoc.conf: -------------------------------------------------------------------------------- 1 | ifdef::backend-docbook[] 2 | [link-inlinemacro] 3 | {0%{target}} 4 | {0#} 5 | {0#{target}{0}} 6 | {0#} 7 | endif::backend-docbook[] 8 | 9 | [quotes] 10 | ^=strong 11 | *= 12 | -------------------------------------------------------------------------------- /libmy/spooldir.h: -------------------------------------------------------------------------------- 1 | #ifndef MY_SPOOLDIR_H 2 | #define MY_SPOOLDIR_H 3 | 4 | struct spooldir; 5 | 6 | struct spooldir *spooldir_init(const char *path); 7 | void spooldir_destroy(struct spooldir **); 8 | char *spooldir_next(struct spooldir *); 9 | 10 | #endif /* MY_SPOOLDIR_H */ 11 | -------------------------------------------------------------------------------- /mtbl/libmtbl.pc.in: -------------------------------------------------------------------------------- 1 | prefix=@prefix@ 2 | exec_prefix=@exec_prefix@ 3 | libdir=@libdir@ 4 | includedir=@includedir@ 5 | 6 | Name: libmtbl 7 | Description: immutable sorted string table library 8 | Version: @VERSION@ 9 | Libs: -L${libdir} -lmtbl 10 | Libs.private: 11 | Cflags: -I${includedir} 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | .*swp 3 | *.gcda 4 | *.gcno 5 | *.la 6 | *.lo 7 | *.log 8 | *.o 9 | *.tar.gz 10 | *.trs 11 | .deps/ 12 | .dirstamp 13 | .libs/ 14 | /aclocal.m4 15 | /autom4te.cache 16 | /build-aux 17 | /config.* 18 | /configure 19 | /libtool 20 | /stamp-h1 21 | all.coverage 22 | coverage-html 23 | Makefile 24 | Makefile.in 25 | report.coverage 26 | TAGS 27 | -------------------------------------------------------------------------------- /t/test-fileset-filter.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [ -z "${top_srcdir}" ]; then 4 | echo "top_srcdir variable not set" 5 | exit 1 6 | fi 7 | 8 | if [ -z "${top_builddir}" ]; then 9 | echo "top_builddir variable not set" 10 | exit 1 11 | fi 12 | 13 | exec "${top_builddir}/t/test-fileset-filter" "${top_srcdir}/t/fileset-filter-data/animals.fileset" 14 | -------------------------------------------------------------------------------- /t/test-fileset-partition.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [ -z "${top_srcdir}" ]; then 4 | echo "top_srcdir variable not set" 5 | exit 1 6 | fi 7 | 8 | if [ -z "${top_builddir}" ]; then 9 | echo "top_builddir variable not set" 10 | exit 1 11 | fi 12 | 13 | exec "${top_builddir}/t/test-fileset-partition" "${top_srcdir}/t/fileset-partition/test.fileset" 14 | -------------------------------------------------------------------------------- /libmy/threadnum.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "threadnum.h" 4 | 5 | static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; 6 | static int next_threadnum = 0; 7 | static __thread int threadnum; 8 | 9 | void 10 | my_threadnum_register(void) { 11 | pthread_mutex_lock(&lock); 12 | threadnum = next_threadnum; 13 | next_threadnum += 1; 14 | pthread_mutex_unlock(&lock); 15 | } 16 | 17 | int 18 | my_threadnum(void) { 19 | return (threadnum); 20 | } 21 | -------------------------------------------------------------------------------- /libmy/getenv_int.h: -------------------------------------------------------------------------------- 1 | #ifndef MY_GETENV_INT_H 2 | #define MY_GETENV_INT_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | static inline bool 9 | getenv_int(const char *name, uint64_t *value) { 10 | char *s, *t; 11 | 12 | s = getenv(name); 13 | if (s == NULL) 14 | return (false); 15 | 16 | *value = strtoul(s, &t, 0); 17 | if (*t != '\0') 18 | return (false); 19 | return (true); 20 | } 21 | 22 | #endif /* MY_GETENV_INT_H */ 23 | -------------------------------------------------------------------------------- /t/test-deb716628.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [ -z "${top_srcdir}" ]; then 4 | echo "top_srcdir variable not set" 5 | exit 1 6 | fi 7 | 8 | if [ -z "${top_builddir}" ]; then 9 | echo "top_builddir variable not set" 10 | exit 1 11 | fi 12 | 13 | ulimit -c 0 14 | 15 | echo "The following line should be an error message." 16 | 17 | "${top_builddir}/src/mtbl_dump" "${top_srcdir}/t/deb716628.data" 2>&1 18 | if [ "$?" -eq "1" ]; then 19 | exit 0 20 | fi 21 | 22 | exit 1 23 | -------------------------------------------------------------------------------- /libmy/varint.h: -------------------------------------------------------------------------------- 1 | #ifndef MY_VARINT_H 2 | #define MY_VARINT_H 3 | 4 | #include 5 | #include 6 | 7 | unsigned varint_length(uint64_t v); 8 | unsigned varint_length_packed(const uint8_t *buf, size_t len_buf); 9 | size_t varint_encode32(uint8_t *ptr, uint32_t value); 10 | size_t varint_encode64(uint8_t *ptr, uint64_t value); 11 | size_t varint_decode32(const uint8_t *ptr, uint32_t *value); 12 | size_t varint_decode64(const uint8_t *ptr, uint64_t *value); 13 | 14 | #endif /* MY_VARINT_H */ 15 | -------------------------------------------------------------------------------- /man/mtbl_crc32c.3.txt: -------------------------------------------------------------------------------- 1 | = mtbl_crc32c(3) = 2 | 3 | == NAME == 4 | 5 | mtbl_crc32c - calculate CRC32C checksum 6 | 7 | == SYNOPSIS == 8 | 9 | ^#include ^ 10 | 11 | ^uint32_t mtbl_crc32c(const uint8_t *'buffer', size_t 'length');^ 12 | 13 | == DESCRIPTION == 14 | 15 | The ^mtbl_crc32c^() function calculates the CRC32C checksum of a sequence of 16 | bytes of length _length_. The _buffer_ argument points to the start of the 17 | sequence. 18 | 19 | == RETURN VALUE == 20 | 21 | The CRC32C checksum. 22 | -------------------------------------------------------------------------------- /t/test-foreign-prefix.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | 3 | if [ -z "${top_srcdir}" ]; then 4 | echo "top_srcdir variable not set" 5 | exit 1 6 | fi 7 | 8 | if [ -z "${top_builddir}" ]; then 9 | echo "top_builddir variable not set" 10 | exit 1 11 | fi 12 | 13 | test "Hello, world!" = \ 14 | "`head -c 13 "${top_srcdir}/t/test-foreign-prefix.data"`" 15 | test '"some key\x0a" "1234567890123456"' = \ 16 | "`"${top_builddir}/src/mtbl_dump" \ 17 | "${top_srcdir}/t/test-foreign-prefix.data"`" 18 | -------------------------------------------------------------------------------- /libmy/atomic_ptr.h: -------------------------------------------------------------------------------- 1 | #ifndef MY_ATOMIC_PTR_H 2 | #define MY_ATOMIC_PTR_H 3 | 4 | typedef struct { 5 | volatile void *ptr; 6 | } atomic_ptr_t; 7 | 8 | static inline void * 9 | atomic_ptr_get(atomic_ptr_t *v) { 10 | return ((void *) (__sync_fetch_and_add(&v->ptr, 0))); 11 | } 12 | 13 | static inline bool 14 | atomic_ptr_set(atomic_ptr_t *v, void *new_ptr) { 15 | void *old_ptr = atomic_ptr_get(v); 16 | return (__sync_bool_compare_and_swap(&v->ptr, old_ptr, new_ptr)); 17 | } 18 | 19 | #endif /* MY_ATOMIC_PTR_H */ 20 | -------------------------------------------------------------------------------- /libmy/lookup3.h: -------------------------------------------------------------------------------- 1 | #ifndef MY_LOOKUP3_H 2 | #define MY_LOOKUP3_H 3 | 4 | #include 5 | 6 | uint32_t my_hashword(const uint32_t *key, size_t length, uint32_t initval); 7 | uint32_t my_hashlittle(const void *key, size_t length, uint32_t initval); 8 | uint32_t my_hashbig(const void *key, size_t length, uint32_t initval); 9 | void my_hashword2(const uint32_t *key, size_t length, uint32_t *pc, uint32_t *pb); 10 | void my_hashlittle2(const void *key, size_t length, uint32_t *pc, uint32_t *pb); 11 | 12 | #endif /* MY_LOOKUP3_H */ 13 | -------------------------------------------------------------------------------- /libmy/zonefile.h: -------------------------------------------------------------------------------- 1 | #ifndef MY_ZONEFILE_H 2 | #define MY_ZONEFILE_H 3 | 4 | #include 5 | 6 | struct zonefile; 7 | 8 | struct zonefile * 9 | zonefile_init_fname(const char *fname); 10 | 11 | void 12 | zonefile_destroy(struct zonefile **); 13 | 14 | const ldns_rdf * 15 | zonefile_get_domain(struct zonefile *); 16 | 17 | size_t 18 | zonefile_get_count(struct zonefile *); 19 | 20 | uint32_t 21 | zonefile_get_serial(struct zonefile *); 22 | 23 | ldns_status 24 | zonefile_read(struct zonefile *, ldns_rr **); 25 | 26 | #endif /* MY_ZONEFILE_H */ 27 | -------------------------------------------------------------------------------- /libmy/read_bytes.h: -------------------------------------------------------------------------------- 1 | #ifndef MY_READ_BYTES_H 2 | #define MY_READ_BYTES_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | static inline bool 10 | read_bytes(int fd, uint8_t *buf, size_t bytes_needed) 11 | { 12 | while (bytes_needed > 0) { 13 | ssize_t bytes_read; 14 | 15 | bytes_read = read(fd, buf, bytes_needed); 16 | if (bytes_read == -1 && errno == EINTR) 17 | continue; 18 | else if (bytes_read <= 0) 19 | return false; 20 | bytes_needed -= bytes_read; 21 | buf += bytes_read; 22 | } 23 | return true; 24 | } 25 | 26 | #endif /* MY_READ_BYTES_H */ 27 | -------------------------------------------------------------------------------- /t/test-gh1.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | 3 | if [ -z "${top_srcdir}" ]; then 4 | echo "top_srcdir variable not set" 5 | exit 1 6 | fi 7 | 8 | if [ -z "${top_builddir}" ]; then 9 | echo "top_builddir variable not set" 10 | exit 1 11 | fi 12 | 13 | ulimit -c 0 14 | 15 | "${top_builddir}/src/mtbl_dump" "${top_srcdir}/t/test-gh1-lz4.data" 1>/dev/null 16 | "${top_builddir}/src/mtbl_dump" "${top_srcdir}/t/test-gh1-lz4hc.data" 1>/dev/null 17 | "${top_builddir}/src/mtbl_dump" "${top_srcdir}/t/test-gh1-snappy.data" 1>/dev/null 18 | "${top_builddir}/src/mtbl_dump" "${top_srcdir}/t/test-gh1-zlib.data" 1>/dev/null 19 | 20 | exit 0 21 | -------------------------------------------------------------------------------- /libmy/COPYRIGHT: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012-2014 by Farsight Security, Inc. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /libmy/my_memory_barrier.h: -------------------------------------------------------------------------------- 1 | #ifndef MY_MEMORY_BARRIER_H 2 | #define MY_MEMORY_BARRIER_H 3 | 4 | #if defined(__GNUC__) 5 | # if defined(__x86_64__) 6 | # define MY_HAVE_MEMORY_BARRIERS 1 7 | # define smp_mb() asm volatile("mfence" ::: "memory") 8 | # define smp_rmb() asm volatile("" ::: "memory") 9 | # define smp_wmb() asm volatile("" ::: "memory") 10 | # elif defined(__ia64__) 11 | # define MY_HAVE_MEMORY_BARRIERS 1 12 | # define smp_mb() asm volatile ("mf" ::: "memory") 13 | # define smp_rmb() asm volatile ("mf" ::: "memory") 14 | # define smp_wmb() asm volatile ("mf" ::: "memory") 15 | # endif 16 | #endif 17 | 18 | #endif /* MY_MEMORY_BARRIER_H */ 19 | -------------------------------------------------------------------------------- /libmy/my_fileset.h: -------------------------------------------------------------------------------- 1 | #ifndef MY_FILESET_H 2 | #define MY_FILESET_H 3 | 4 | #include 5 | 6 | struct my_fileset; 7 | 8 | typedef void *(*my_fileset_load_func)(struct my_fileset *, const char *fname); 9 | typedef void (*my_fileset_unload_func)(struct my_fileset *, const char *fname, void *); 10 | 11 | struct my_fileset *my_fileset_init( 12 | const char *setfile, 13 | my_fileset_load_func, 14 | my_fileset_unload_func, 15 | void *user); 16 | void my_fileset_destroy(struct my_fileset **); 17 | void *my_fileset_user(struct my_fileset *); 18 | void my_fileset_reload(struct my_fileset *); 19 | bool my_fileset_get(struct my_fileset *, size_t, const char **, void **); 20 | 21 | #endif /* MY_FILESET_H */ 22 | -------------------------------------------------------------------------------- /man/mtbl_verify.1.txt: -------------------------------------------------------------------------------- 1 | = mtbl_verify(1) = 2 | 3 | == NAME == 4 | 5 | mtbl_verify - verify integrity of an MTBL file's data and index blocks 6 | 7 | == SYNOPSIS == 8 | 9 | ^mtbl_verify^ 'FILE' ['FILE']... 10 | 11 | == DESCRIPTION == 12 | 13 | ^mtbl_verify^(1) verifies the embedded data and index block checksums in the 14 | MTBL files specified on the command line. 15 | 16 | Each filename specified on the command line will be verified, and a message 17 | indicating whether each file passed or failed will be printed to stdout. If 18 | a file fails, additional information about the failure will be printed to 19 | stderr. 20 | 21 | If stdout is a terminal or the ^-p^ option is provided, periodic progress 22 | indications will be printed. 23 | -------------------------------------------------------------------------------- /libmy/b64_decode.h: -------------------------------------------------------------------------------- 1 | /* 2 | cdecode.h - c header for a base64 decoding algorithm 3 | 4 | This is part of the libb64 project, and has been placed in the public domain. 5 | For details, see http://sourceforge.net/projects/libb64 6 | */ 7 | 8 | #ifndef BASE64_CDECODE_H 9 | #define BASE64_CDECODE_H 10 | 11 | typedef enum 12 | { 13 | step_a, step_b, step_c, step_d 14 | } base64_decodestep; 15 | 16 | typedef struct 17 | { 18 | base64_decodestep step; 19 | char plainchar; 20 | } base64_decodestate; 21 | 22 | void base64_init_decodestate(base64_decodestate* state_in); 23 | 24 | int base64_decode_value(char value_in); 25 | 26 | int base64_decode_block(const char* code_in, const int length_in, char* plaintext_out, base64_decodestate* state_in); 27 | 28 | #endif /* BASE64_CDECODE_H */ 29 | -------------------------------------------------------------------------------- /t/test-compression.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | 3 | if [ -z "${top_srcdir}" ]; then 4 | echo "top_srcdir variable not set" 5 | exit 1 6 | fi 7 | 8 | if [ -z "${top_builddir}" ]; then 9 | echo "top_builddir variable not set" 10 | exit 1 11 | fi 12 | 13 | ulimit -c 0 14 | 15 | for compression_type in \ 16 | none \ 17 | snappy \ 18 | zlib \ 19 | lz4 \ 20 | lz4hc \ 21 | zstd \ 22 | ; do 23 | for thread_count in \ 24 | 0 \ 25 | 1 \ 26 | 2 \ 27 | 5 \ 28 | 8 \ 29 | 10 \ 30 | 24 \ 31 | 36 \ 32 | 50 \ 33 | 100 \ 34 | 500 \ 35 | 1000 \ 36 | ; do 37 | echo "$0: Testing compression type ${compression_type} ${thread_count}" 38 | "${top_builddir}/t/test-compression" "${compression_type}" "${thread_count}" 39 | done 40 | done 41 | 42 | exit 0 43 | -------------------------------------------------------------------------------- /libmy/b64_encode.h: -------------------------------------------------------------------------------- 1 | /* 2 | cencode.h - c header for a base64 encoding algorithm 3 | 4 | This is part of the libb64 project, and has been placed in the public domain. 5 | For details, see http://sourceforge.net/projects/libb64 6 | */ 7 | 8 | #ifndef BASE64_CENCODE_H 9 | #define BASE64_CENCODE_H 10 | 11 | typedef enum 12 | { 13 | step_A, step_B, step_C 14 | } base64_encodestep; 15 | 16 | typedef struct 17 | { 18 | base64_encodestep step; 19 | char result; 20 | } base64_encodestate; 21 | 22 | void base64_init_encodestate(base64_encodestate* state_in); 23 | 24 | char base64_encode_value(char value_in); 25 | 26 | int base64_encode_block(const char* plaintext_in, int length_in, char* code_out, base64_encodestate* state_in); 27 | 28 | int base64_encode_blockend(char* code_out, base64_encodestate* state_in); 29 | 30 | #endif /* BASE64_CENCODE_H */ 31 | -------------------------------------------------------------------------------- /mtbl/crc32c_wrap.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2012, 2014 by Farsight Security, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "mtbl-private.h" 18 | #include "libmy/crc32c.h" 19 | 20 | uint32_t 21 | mtbl_crc32c(const uint8_t *buf, size_t size) 22 | { 23 | return (my_crc32c(buf, size)); 24 | } 25 | -------------------------------------------------------------------------------- /libmy/my_byteorder.h: -------------------------------------------------------------------------------- 1 | #ifndef MY_BYTEORDER_H 2 | #define MY_BYTEORDER_H 3 | 4 | #include "config.h" 5 | 6 | #ifdef HAVE_ENDIAN_H 7 | # include 8 | #else 9 | # ifdef HAVE_SYS_ENDIAN_H 10 | # include 11 | # endif 12 | #endif 13 | 14 | #if defined(__APPLE__) 15 | # include 16 | # define be16toh OSSwapBigToHostInt16 17 | # define be32toh OSSwapBigToHostInt32 18 | # define be64toh OSSwapBigToHostInt64 19 | # define htobe16 OSSwapHostToBigInt16 20 | # define htobe32 OSSwapHostToBigInt32 21 | # define htobe64 OSSwapHostToBigInt64 22 | # define htole16 OSSwapHostToLittleInt16 23 | # define htole32 OSSwapHostToLittleInt32 24 | # define htole64 OSSwapHostToLittleInt64 25 | # define le16toh OSSwapLittleToHostInt16 26 | # define le32toh OSSwapLittleToHostInt32 27 | # define le64toh OSSwapLittleToHostInt64 28 | #endif 29 | 30 | #endif /* MY_BYTEORDER_H */ 31 | -------------------------------------------------------------------------------- /libmy/heap.h: -------------------------------------------------------------------------------- 1 | #ifndef MY_HEAP_H 2 | #define MY_HEAP_H 3 | 4 | struct heap; 5 | 6 | typedef int (*heap_compare_func)(const void *a, const void *b, void *clos); 7 | 8 | struct heap *heap_init(heap_compare_func, void *); 9 | void heap_destroy(struct heap **); 10 | void heap_reset(struct heap *); 11 | void heap_clip(struct heap *, size_t); 12 | void heap_heapify(struct heap *); 13 | /* you should use heap_add if you are adding more than n/log(n) elements 14 | * at a time. you must call heap_heapify after calling heap_add. this 15 | * reduces the time complexity from O(n*log(n)) to O(n)*/ 16 | void heap_add(struct heap *, void *); 17 | void heap_push(struct heap *, void *); 18 | void *heap_pop(struct heap *); 19 | void *heap_replace(struct heap *, void *); 20 | void *heap_peek(struct heap *); 21 | void *heap_get(struct heap *, size_t); 22 | size_t heap_size(struct heap *); 23 | 24 | #endif /* MY_HEAP_H */ 25 | -------------------------------------------------------------------------------- /libmy/my_rate.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2008, 2009, 2013, 2014 by Farsight Security, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef MY_RATE_H 18 | #define MY_RATE_H 19 | 20 | struct my_rate; 21 | 22 | struct my_rate * 23 | my_rate_init(unsigned rate, unsigned freq); 24 | 25 | void 26 | my_rate_destroy(struct my_rate **); 27 | 28 | void 29 | my_rate_sleep(struct my_rate *); 30 | 31 | #endif /* MY_RATE_H */ 32 | -------------------------------------------------------------------------------- /t/test-verify.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [ -z "${top_srcdir}" ]; then 4 | echo "top_srcdir variable not set" 5 | exit 1 6 | fi 7 | 8 | if [ -z "${top_builddir}" ]; then 9 | echo "top_builddir variable not set" 10 | exit 1 11 | fi 12 | 13 | ulimit -c 0 14 | 15 | for good_fname in \ 16 | "${top_srcdir}/t/test-gh1-snappy.data" \ 17 | "${top_srcdir}/t/test-gh1-zlib.data" \ 18 | "${top_srcdir}/t/test-foreign-prefix.data" \ 19 | "${top_srcdir}/t/test-verify-good1.data" \ 20 | ; do 21 | "${top_builddir}/src/mtbl_verify" "${good_fname}" 2>&1 22 | if [ "$?" -ne "0" ]; then 23 | exit 1 24 | fi 25 | done 26 | 27 | echo "The following lines should be error messages." 28 | 29 | for bad_fname in \ 30 | "${top_srcdir}/t/test-verify-bad1.data" \ 31 | "${top_srcdir}/t/test-verify-bad2.data" \ 32 | "${top_srcdir}/t/test-verify-bad-index-block-offset.data" \ 33 | ; do 34 | "${top_builddir}/src/mtbl_verify" "${bad_fname}" 2>&1 35 | if [ "$?" -ne "1" ]; then 36 | exit 1 37 | fi 38 | done 39 | 40 | exit 0 41 | -------------------------------------------------------------------------------- /man/mtbl_threadpool.3.txt: -------------------------------------------------------------------------------- 1 | = mtbl_threadpool(3) = 2 | 3 | == NAME == 4 | 5 | mtbl_threadpool - create a shared worker threadpool 6 | 7 | == SYNOPSIS == 8 | 9 | ^#include ^ 10 | 11 | [verse] 12 | ^struct mtbl_threadpool * 13 | mtbl_threadpool_init(size_t 'thread_count');^ 14 | 15 | [verse] 16 | ^void 17 | mtbl_threadpool_destroy(struct mtbl_threadpool **'pool');^ 18 | 19 | == DESCRIPTION == 20 | 21 | Certain MTBL "option" structures accept an ^mtbl_threadpool^ option (e.g. 22 | ^mtbl_writer_options^, ^mtbl_sorter_options^) to enable internal concurrency. 23 | The user-provided ^mtbl_threadpool^ object must be initialized before use by 24 | calling ^mtbl_threadpool_init()^, and must be destroyed after use by calling 25 | ^mtbl_threadpool_destroy()^. 26 | 27 | If the _thread_count_ parameter to ^mtbl_threadpool_init()^ is 0, 28 | multithreading will be disabled. Regardless, a non-NULL ^mtbl_threadpool^ 29 | object will be returned from ^mtbl_threadpool_init()^. 30 | 31 | === Threadpool options === 32 | 33 | ==== thread_count ==== 34 | The maximum number of worker threads that the threadpool will open. 35 | 36 | == RETURN VALUE == 37 | 38 | ^mtbl_threadpool_init^() returns NULL on failure, and non-NULL on success. 39 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: c 2 | 3 | os: 4 | - linux 5 | - osx 6 | 7 | matrix: 8 | include: 9 | - os: linux 10 | dist: trusty 11 | sudo: required 12 | 13 | before_install: 14 | # Linux 15 | - if [ "$TRAVIS_OS_NAME" == "linux" ]; then sudo sh -c 'wget -O /etc/apt/trusted.gpg.d/debian-farsightsec.gpg https://dl.farsightsecurity.com/debian/archive.pubkey'; fi 16 | - if [ "$TRAVIS_OS_NAME" == "linux" ]; then sudo sh -c 'echo "deb [arch=amd64] http://dl.farsightsecurity.com/debian wheezy-farsightsec main" > /etc/apt/sources.list.d/debian-farsightsec.list'; fi 17 | - if [ "$TRAVIS_OS_NAME" == "linux" ]; then sudo apt-get update -qq; fi 18 | 19 | # OS X 20 | - if [ "$TRAVIS_OS_NAME" == "osx" ]; then brew update; fi 21 | 22 | install: 23 | # Linux 24 | - if [ "$TRAVIS_OS_NAME" == "linux" ]; then sudo apt-get -qq install liblz4-dev libsnappy-dev libzstd-dev zlib1g-dev; fi 25 | 26 | # OS X 27 | - if [ "$TRAVIS_OS_NAME" == "osx" ]; then brew install lz4; fi 28 | - if [ "$TRAVIS_OS_NAME" == "osx" ]; then brew install snappy; fi 29 | - if [ "$TRAVIS_OS_NAME" == "osx" ]; then brew install zstd; fi 30 | 31 | script: 32 | - ./autogen.sh 33 | - ./configure 34 | - make 35 | - make distcheck VERBOSE=1 36 | -------------------------------------------------------------------------------- /libmy/print_string.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2012 by Farsight Security, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef MY_PRINT_STRING_H 18 | #define MY_PRINT_STRING_H 19 | 20 | #include 21 | #include 22 | #include 23 | 24 | static inline void 25 | print_string(const void *data, size_t len, FILE *out) 26 | { 27 | uint8_t *str = (uint8_t *) data; 28 | fputc('"', out); 29 | while (len-- != 0) { 30 | unsigned c = *(str++); 31 | if (isprint(c)) { 32 | if (c == '"') 33 | fputs("\\\"", out); 34 | else 35 | fputc(c, out); 36 | } else { 37 | fprintf(out, "\\x%02x", c); 38 | } 39 | } 40 | fputc('"', out); 41 | } 42 | 43 | #endif /* MY_PRINT_STRING_H */ 44 | -------------------------------------------------------------------------------- /libmy/ubuf-pb.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 by Farsight Security, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef MY_UBUF_PB_H 18 | #define MY_UBUF_PB_H 19 | 20 | #include 21 | 22 | #include "ubuf.h" 23 | 24 | struct ubuf_protobuf_c_buffer { 25 | ProtobufCBuffer base; 26 | ubuf *u; 27 | }; 28 | 29 | static inline void 30 | ubuf_protobuf_c_buffer_append(ProtobufCBuffer *buffer, 31 | size_t len, 32 | const uint8_t *data) 33 | { 34 | ubuf *u = ((struct ubuf_protobuf_c_buffer *) buffer)->u; 35 | ubuf_append(u, data, len); 36 | } 37 | 38 | #define UBUF_PROTOBUF_C_BUFFER_INIT(u) { { ubuf_protobuf_c_buffer_append }, u } 39 | 40 | #endif /* MY_UBUF_PB_H */ 41 | -------------------------------------------------------------------------------- /man/mtbl_fixed.3.txt: -------------------------------------------------------------------------------- 1 | = mtbl_fixed(3) = 2 | 3 | == NAME == 4 | 5 | mtbl_fixed - Fixed-width encoding and decoding of 32 and 64 bit integers 6 | 7 | == SYNOPSIS == 8 | 9 | ^#include ^ 10 | 11 | ^size_t mtbl_fixed_encode32(uint8_t *'dst', uint32_t 'value');^ 12 | 13 | ^size_t mtbl_fixed_encode64(uint8_t *'dst', uint64_t 'value');^ 14 | 15 | ^uint32_t mtbl_fixed_decode32(const uint8_t *'ptr');^ 16 | 17 | ^uint64_t mtbl_fixed_decode64(const uint8_t *'ptr');^ 18 | 19 | == DESCRIPTION == 20 | 21 | ^mtbl_fixed_encode32^() and ^mtbl_fixed_encode64^() write the 32 or 64 bit 22 | quantity, respectively, in the argument _value_ to the buffer in the argument 23 | _dst_. The quantity will be written in little endian order, regardless of host 24 | architecture. 25 | 26 | ^mtbl_fixed_decode32^() and ^mtbl_fixed_decode64^() read and return the 32 27 | or 64 bit quantity, respectively, in the argument _ptr_. The quantity will be 28 | read in little endian order, regardless of host architecture. 29 | 30 | Bounds checking must be performed by the caller. 31 | 32 | == RETURN VALUE == 33 | 34 | ^mtbl_fixed_encode32^() and ^mtbl_fixed_encode64^() return the number of 35 | bytes written to the buffer. (4 or 8, respectively.) 36 | 37 | ^mtbl_fixed_decode32^() and ^mtbl_fixed_decode64^() return the decoded 38 | quantity. 39 | -------------------------------------------------------------------------------- /libmy/my_queue.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, 2014 by Farsight Security, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "my_memory_barrier.h" 18 | 19 | #ifdef MY_HAVE_MEMORY_BARRIERS 20 | # define my_queue_mb_init my_queue_init 21 | # define my_queue_mb_destroy my_queue_destroy 22 | # define my_queue_mb_impl_type my_queue_impl_type 23 | # define my_queue_mb_insert my_queue_insert 24 | # define my_queue_mb_remove my_queue_remove 25 | # include "my_queue_mb.c" 26 | #else 27 | # define my_queue_mutex_init my_queue_init 28 | # define my_queue_mutex_destroy my_queue_destroy 29 | # define my_queue_mutex_impl_type my_queue_impl_type 30 | # define my_queue_mutex_insert my_queue_insert 31 | # define my_queue_mutex_remove my_queue_remove 32 | # include "my_queue_mutex.c" 33 | #endif 34 | -------------------------------------------------------------------------------- /mtbl/bytes.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "libmy/ubuf.h" 5 | 6 | static inline void 7 | bytes_shortest_separator(ubuf *start, const uint8_t *limit, size_t len_limit) 8 | { 9 | 10 | size_t min_length = ubuf_size(start) < len_limit ? ubuf_size(start) : len_limit; 11 | size_t diff_index = 0; 12 | while ((diff_index < min_length) && 13 | (ubuf_data(start)[diff_index] == limit[diff_index])) 14 | { 15 | diff_index++; 16 | } 17 | 18 | if (diff_index >= min_length) 19 | return; 20 | 21 | uint8_t diff_byte = ubuf_data(start)[diff_index]; 22 | if (diff_byte < 0xFF && diff_byte + 1 < limit[diff_index]) { 23 | ubuf_data(start)[diff_index]++; 24 | ubuf_clip(start, diff_index + 1); 25 | } else if (diff_index + sizeof(uint16_t) < min_length) { 26 | /* awww yeah, big endian arithmetic on strings */ 27 | uint16_t u_start, u_limit, u_between; 28 | memcpy(&u_start, &ubuf_data(start)[diff_index], sizeof(u_start)); 29 | memcpy(&u_limit, &limit[diff_index], sizeof(u_limit)); 30 | u_start = be16toh(u_start); 31 | u_limit = be16toh(u_limit); 32 | u_between = u_start + 1; 33 | if (u_start <= u_between && u_between <= u_limit) { 34 | u_between = htobe16(u_between); 35 | memcpy(&ubuf_data(start)[diff_index], &u_between, sizeof(u_between)); 36 | ubuf_clip(start, diff_index + sizeof(uint16_t)); 37 | } 38 | } 39 | 40 | assert(bytes_compare(ubuf_data(start), ubuf_size(start), limit, len_limit) < 0); 41 | } 42 | -------------------------------------------------------------------------------- /man/mtbl_iter.3.txt: -------------------------------------------------------------------------------- 1 | = mtbl_iter(3) = 2 | 3 | == NAME == 4 | 5 | mtbl_iter - iterate over a sequence of key-value pairs 6 | 7 | == SYNOPSIS == 8 | 9 | ^#include ^ 10 | 11 | [verse] 12 | ^mtbl_res 13 | mtbl_iter_next(struct mtbl_iter *'it', 14 | const uint8_t \**'key', size_t *'len_key', 15 | const uint8_t **'val', size_t *'len_val');^ 16 | 17 | [verse] 18 | ^void 19 | mtbl_iter_destroy(struct mtbl_iter **'it');^ 20 | 21 | [verse] 22 | ^mtbl_res 23 | mtbl_iter_seek(struct mtbl_iter *'it', 24 | const uint8_t *'key', size_t 'len_key');^ 25 | 26 | == DESCRIPTION == 27 | 28 | The ^mtbl_iter^ interface is used to return a sequence of one or more key-value 29 | pairs. Once the caller obtains an ^mtbl_iter^ object, ^mtbl_iter_next^() should 30 | be repeatedly called on it until there are no more key-value entries to 31 | retrieve, at which point the iterator object must be freed by calling 32 | ^mtbl_iter_destroy^(). ^mtbl_iter_seek^() can be called on the iterator to seek 33 | to a different location in the index without having to destroy the iterator 34 | and create a new one. 35 | 36 | == RETURN VALUE == 37 | 38 | ^mtbl_iter_next^() returns ^mtbl_res_success^ if a key-value entry was 39 | successfully retrieved, in which case _key_ and _val_ will point to buffers of 40 | length _len_key_ and _len_val_ respectively. The value ^mtbl_res_failure^ is 41 | returned if there are no more entries to read, or if the _it_ argument is NULL. 42 | 43 | == SEE ALSO == 44 | 45 | link:mtbl_source[3] 46 | -------------------------------------------------------------------------------- /libmy/m4/pcap.m4: -------------------------------------------------------------------------------- 1 | AC_DEFUN([MY_CHECK_LIBPCAP], [ 2 | libpcap_CFLAGS="" 3 | libpcap_LIBS="-lpcap" 4 | 5 | AC_ARG_WITH( 6 | [libpcap], 7 | AC_HELP_STRING([--with-libpcap=DIR], [libpcap installation path]), 8 | [], 9 | [withval="yes"] 10 | ) 11 | if test "$withval" = "yes"; then 12 | withval="/usr /usr/local" 13 | fi 14 | 15 | libpcap_dir="" 16 | 17 | AC_MSG_CHECKING([for libpcap headers]) 18 | for dir in $withval; do 19 | if test -f "$dir/include/pcap.h"; then 20 | libpcap_dir="$dir" 21 | if test "$dir" != "/usr"; then 22 | libpcap_CFLAGS="-I$dir/include" 23 | fi 24 | break 25 | fi 26 | done 27 | if test -n "$libpcap_dir"; then 28 | AC_MSG_RESULT([$libpcap_dir]) 29 | else 30 | AC_MSG_ERROR([cannot find pcap.h in $withval]) 31 | fi 32 | 33 | save_LDFLAGS="$LDFLAGS" 34 | save_LIBS="$LIBS" 35 | if test "$libpcap_dir" != "/usr"; then 36 | libpcap_LIBS="$libpcap_LIBS -L$libpcap_dir/lib" 37 | LDFLAGS="-L$libpcap_dir/lib" 38 | fi 39 | AC_CHECK_LIB( 40 | [pcap], 41 | [pcap_open_offline], 42 | [], 43 | [AC_MSG_ERROR([required library not found])] 44 | ) 45 | AC_SEARCH_LIBS( 46 | [pcap_create], 47 | [pcap], 48 | AC_DEFINE([HAVE_PCAP_CREATE], [1], [Define to 1 if pcap_create() is available.]) 49 | ) 50 | LDFLAGS="$save_LDFLAGS" 51 | LIBS="$save_LIBS" 52 | 53 | AC_SUBST([libpcap_CFLAGS]) 54 | AC_SUBST([libpcap_LIBS]) 55 | ]) 56 | -------------------------------------------------------------------------------- /libmy/m4/my_pkg_config_files.m4: -------------------------------------------------------------------------------- 1 | # SYNOPSIS 2 | # 3 | # my_PKG_CONFIG_FILES(pc-files-variable, pc-files-value) 4 | # 5 | # DESCRIPTION 6 | # 7 | # Wrapper for PKG_INSTALLDIR that allows disabling the installation of .pc 8 | # files when explicitly configured with --without-pkgconfigdir. It also 9 | # selects a more sensible default for pkgconfigdir on platforms that place 10 | # pkg-config files in a "libdata" directory. 11 | # 12 | # If .pc file installation is enabled, pc-files-variable will be set to 13 | # pc-files-value. 14 | # 15 | # Example: 16 | # 17 | # In configure.ac: 18 | # 19 | # my_PKG_CONFIG_FILES([LIBEXAMPLE_PC], [src/libexample.pc]) 20 | # 21 | # In Makefile.am: 22 | # 23 | # pkgconfig_DATA = ${LIBEXAMPLE_PC} 24 | # CLEANFILES += ${LIBEXAMPLE_PC} 25 | # 26 | # Here, ${LIBEXAMPLE_PC} will normally expand to "src/libexample.pc", unless 27 | # configure was called with --without-pkgconfigdir, in which case it will 28 | # expand to "". 29 | # 30 | 31 | AC_DEFUN([my_PKG_CONFIG_FILES], 32 | [ 33 | PKG_PROG_PKG_CONFIG 34 | 35 | pkgconfig_dir='${libdir}/pkgconfig' 36 | if test -n "$PKG_CONFIG"; then 37 | if $PKG_CONFIG --variable=pc_path pkg-config 2>/dev/null | grep -q /libdata/; then 38 | pkgconfig_dir='${prefix}/libdata/pkgconfig' 39 | fi 40 | fi 41 | PKG_INSTALLDIR([$pkgconfig_dir]) 42 | 43 | if test "x$pkgconfigdir" != "xno"; then 44 | if test -z "$PKG_CONFIG"; then 45 | AC_MSG_ERROR([pkg-config is required!]) 46 | fi 47 | $1=$2 48 | AC_SUBST([$1]) 49 | fi 50 | ]) 51 | -------------------------------------------------------------------------------- /libmy/string_replace.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2012 by Farsight Security, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef MY_STRING_REPLACE_H 18 | #define MY_STRING_REPLACE_H 19 | 20 | #include 21 | 22 | #include "ubuf.h" 23 | 24 | static inline char * 25 | string_replace(const char *str, const char *old, const char *new) 26 | { 27 | const char *end; 28 | char *ret; 29 | size_t retsz; 30 | 31 | if (strstr(str, old) == NULL) { 32 | char *s = strdup(str); 33 | assert(s != NULL); 34 | return (s); 35 | } 36 | ubuf *u = ubuf_new(); 37 | 38 | end = str + strlen(str) + 1; 39 | while ((ret = strstr(str, old)) != NULL) { 40 | size_t offset = (size_t) (ret - str); 41 | ubuf_append(u, (uint8_t *) str, offset); 42 | ubuf_append(u, (uint8_t *) new, strlen(new)); 43 | str += offset; 44 | if (str + strlen(old) >= end) 45 | break; 46 | str += strlen(old); 47 | } 48 | ubuf_append(u, (uint8_t *) str, strlen(str)); 49 | 50 | ubuf_cterm(u); 51 | ubuf_detach(u, (uint8_t **) &ret, &retsz); 52 | ubuf_destroy(&u); 53 | return (ret); 54 | } 55 | 56 | #endif /* MY_STRING_REPLACE_H */ 57 | -------------------------------------------------------------------------------- /man/mtbl_info.1.txt: -------------------------------------------------------------------------------- 1 | = mtbl_info(1) = 2 | 3 | == NAME == 4 | 5 | mtbl_info - display information about an MTBL file 6 | 7 | == SYNOPSIS == 8 | 9 | ^mtbl_info^ 'FILE' ['FILE']... 10 | 11 | == DESCRIPTION == 12 | 13 | ^mtbl_info^(1) displays the following information about the MTBL files 14 | specified on the command line. 15 | 16 | 'file name' -- the name of the MTBL file. 17 | 18 | 'file size' -- the total size of the MTBL file, in bytes. 19 | 20 | 'index bytes' -- the total number of bytes and proportion of the total file 21 | size consumed by the index. 22 | 23 | 'data block bytes' -- the total number of bytes and proportion of the total 24 | file size consumed by data blocks. 25 | 26 | 'data block size' -- the maximum size of an uncompressed data block. 27 | 28 | 'data block count' -- the total number of data blocks. 29 | 30 | 'entry count' -- the total number of key-value entries. 31 | 32 | 'key bytes' -- the total number of bytes that all keys in the file would 33 | occupy if stored end-to-end in a byte array with no delimiters. 34 | 35 | 'value bytes' -- the total number of bytes that all values in the file would 36 | occupy if stored end-to-end in a byte array with no delimiters. 37 | 38 | 'compression algorithm' -- the algorithm used to compress data blocks. 39 | Possible values are "none", "snappy", "zlib", "lz4", "lz4hc", and "zstd". 40 | 41 | 'compactness' -- a rough metric comparing the total number of bytes in the 42 | key-value entries with the total size of the MTBL file. It is calculated as 43 | (file size) / (key bytes + value bytes), and thus takes into account the gains 44 | of data block compression and prefix key compression against the overhead of 45 | the index, metadata, and data block offset arrays. 46 | -------------------------------------------------------------------------------- /libmy/crc32c.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 by Farsight Security, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include 18 | #include 19 | #include 20 | 21 | #include 22 | 23 | #include "crc32c.h" 24 | 25 | static uint32_t my_crc32c_first(const uint8_t *buf, size_t len); 26 | 27 | my_crc32c_fp my_crc32c = my_crc32c_first; 28 | 29 | /* crc32c-slicing.c */ 30 | uint32_t my_crc32c_slicing(const uint8_t *, size_t); 31 | 32 | /* crc32c-sse42.c */ 33 | #if __GNUC__ >= 3 && defined(__x86_64__) 34 | bool my_crc32c_sse42_supported(void); 35 | uint32_t my_crc32c_sse42(const uint8_t *, size_t); 36 | #endif 37 | 38 | #if defined(__GNUC__) 39 | __attribute__((constructor)) 40 | #endif 41 | static void 42 | my_crc32c_runtime_detection(void) 43 | { 44 | #if __GNUC__ >= 3 && defined(__x86_64__) 45 | if (my_crc32c_sse42_supported()) { 46 | my_crc32c = my_crc32c_sse42; 47 | } else { 48 | my_crc32c = my_crc32c_slicing; 49 | } 50 | #else 51 | my_crc32c = my_crc32c_slicing; 52 | #endif 53 | } 54 | 55 | static uint32_t 56 | my_crc32c_first(const uint8_t *buf, size_t len) { 57 | my_crc32c_runtime_detection(); 58 | return my_crc32c(buf, len); 59 | } 60 | -------------------------------------------------------------------------------- /mtbl.spec: -------------------------------------------------------------------------------- 1 | Name: mtbl 2 | Version: 1.7.1 3 | Release: 1%{?dist} 4 | Summary: immutable sorted string table utilities 5 | 6 | License: Apache-2.0 7 | URL: https://github.com/farsightsec/%{name} 8 | Source0: https://dl.farsightsecurity.com/dist/%{name}/%{name}-%{version}.tar.gz 9 | 10 | 11 | BuildRequires: zlib-devel lz4-devel libzstd-devel snappy-devel 12 | #Requires: 13 | # TODO: will Requires be set automatically? 14 | 15 | %description 16 | mtbl is a C library implementation of the Sorted String Table (SSTable) 17 | data structure. mtbl exposes primitives for creating, searching and 18 | merging SSTable files. 19 | 20 | This package contains the shared library for libmtbl and the mtbl 21 | command-line tools. 22 | 23 | %package devel 24 | Summary: immutable sorted string table library (development files) 25 | Requires: %{name}%{?_isa} = %{version}-%{release} zlib lz4 libzstd snappy 26 | 27 | %description devel 28 | mtbl is a C library implementation of the Sorted String Table (SSTable) 29 | data structure. mtbl exposes primitives for creating, searching and 30 | merging SSTable files. 31 | 32 | This package contains the static library, headers, and development 33 | documentation for libmtbl. 34 | 35 | %prep 36 | %setup -q 37 | 38 | %build 39 | [ -x configure ] || autoreconf -fvi 40 | %configure 41 | make %{?_smp_mflags} 42 | 43 | 44 | %install 45 | rm -rf $RPM_BUILD_ROOT 46 | %make_install 47 | 48 | 49 | %files 50 | %defattr(-,root,root,-) 51 | %{_libdir}/*.so.* 52 | %exclude %{_libdir}/libmtbl.la 53 | %_bindir/* 54 | %_mandir/man1/* 55 | 56 | %files devel 57 | %{_libdir}/*.so 58 | %{_libdir}/*.a 59 | %{_libdir}/pkgconfig/* 60 | %{_includedir}/* 61 | %_mandir/man3/* 62 | %_mandir/man7/* 63 | 64 | 65 | %doc 66 | 67 | 68 | 69 | %changelog 70 | -------------------------------------------------------------------------------- /man/mtbl_crc32c.3: -------------------------------------------------------------------------------- 1 | '\" t 2 | .\" Title: mtbl_crc32c 3 | .\" Author: [FIXME: author] [see http://docbook.sf.net/el/author] 4 | .\" Generator: DocBook XSL Stylesheets v1.78.1 5 | .\" Date: 01/31/2014 6 | .\" Manual: \ \& 7 | .\" Source: \ \& 8 | .\" Language: English 9 | .\" 10 | .TH "MTBL_CRC32C" "3" "01/31/2014" "\ \&" "\ \&" 11 | .\" ----------------------------------------------------------------- 12 | .\" * Define some portability stuff 13 | .\" ----------------------------------------------------------------- 14 | .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 15 | .\" http://bugs.debian.org/507673 16 | .\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html 17 | .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 18 | .ie \n(.g .ds Aq \(aq 19 | .el .ds Aq ' 20 | .\" ----------------------------------------------------------------- 21 | .\" * set default formatting 22 | .\" ----------------------------------------------------------------- 23 | .\" disable hyphenation 24 | .nh 25 | .\" disable justification (adjust text to left margin only) 26 | .ad l 27 | .\" ----------------------------------------------------------------- 28 | .\" * MAIN CONTENT STARTS HERE * 29 | .\" ----------------------------------------------------------------- 30 | .SH "NAME" 31 | mtbl_crc32c \- calculate CRC32C checksum 32 | .SH "SYNOPSIS" 33 | .sp 34 | \fB#include \fR 35 | .sp 36 | \fBuint32_t mtbl_crc32c(const uint8_t *\fR\fB\fIbuffer\fR\fR\fB, size_t \fR\fB\fIlength\fR\fR\fB);\fR 37 | .SH "DESCRIPTION" 38 | .sp 39 | The \fBmtbl_crc32c\fR() function calculates the CRC32C checksum of a sequence of bytes of length \fIlength\fR\&. The \fIbuffer\fR argument points to the start of the sequence\&. 40 | .SH "RETURN VALUE" 41 | .sp 42 | The CRC32C checksum\&. 43 | -------------------------------------------------------------------------------- /libmy/my_alloc.h: -------------------------------------------------------------------------------- 1 | #ifndef MY_ALLOC_H 2 | #define MY_ALLOC_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | static inline void * 9 | my_calloc(size_t nmemb, size_t size) 10 | { 11 | void *ptr = calloc(nmemb, size); 12 | assert(ptr != NULL); 13 | return (ptr); 14 | } 15 | 16 | static inline void * 17 | my_malloc(size_t size) 18 | { 19 | void *ptr = malloc(size); 20 | assert(ptr != NULL); 21 | return (ptr); 22 | } 23 | 24 | static inline void * 25 | my_realloc(void *ptr, size_t size) 26 | { 27 | ptr = realloc(ptr, size); 28 | assert(ptr != NULL); 29 | return (ptr); 30 | } 31 | 32 | static inline char * 33 | my_strdup(const char *s) 34 | { 35 | char *ptr = strdup(s); 36 | assert(ptr != NULL); 37 | return (ptr); 38 | } 39 | 40 | #define my_free(ptr) do { free(ptr); (ptr) = NULL; } while (0) 41 | 42 | #if defined(MY_ALLOC_WARN_DEPRECATED) 43 | 44 | static inline void *my_calloc_deprecated(size_t, size_t) 45 | __attribute__ ((deprecated("use my_calloc, not calloc"))); 46 | 47 | static inline void *my_malloc_deprecated(size_t) 48 | __attribute__ ((deprecated("use my_malloc, not malloc"))); 49 | 50 | static inline void *my_realloc_deprecated(void *, size_t) 51 | __attribute__ ((deprecated("use my_realloc, not realloc"))); 52 | 53 | static inline void * 54 | my_calloc_deprecated(size_t nmemb, size_t size) 55 | { 56 | return calloc(nmemb, size); 57 | } 58 | 59 | static inline void * 60 | my_malloc_deprecated(size_t size) 61 | { 62 | return malloc(size); 63 | } 64 | 65 | static inline void * 66 | my_realloc_deprecated(void *ptr, size_t size) 67 | { 68 | return realloc(ptr, size); 69 | } 70 | 71 | #define calloc my_calloc_deprecated 72 | #define malloc my_malloc_deprecated 73 | #define realloc my_realloc_deprecated 74 | 75 | #endif /* MY_ALLOC_WARN_DEPRECATED */ 76 | 77 | #endif /* MY_ALLOC_H */ 78 | -------------------------------------------------------------------------------- /man/mtbl_varint.3.txt: -------------------------------------------------------------------------------- 1 | = mtbl_varint(3) = 2 | 3 | == NAME == 4 | 5 | mtbl_varint - Variable-width encoding and decoding of 32 and 64 bit integers 6 | 7 | == SYNOPSIS == 8 | 9 | ^#include ^ 10 | 11 | ^unsigned mtbl_varint_length(uint64_t 'value');^ 12 | 13 | ^unsigned mtbl_varint_length_packed(const uint8_t *'buf', size_t 'len_buf');^ 14 | 15 | ^size_t mtbl_varint_encode32(uint8_t *'ptr', uint32_t 'value');^ 16 | 17 | ^size_t mtbl_varint_encode64(uint8_t *'ptr', uint64_t 'value');^ 18 | 19 | ^size_t mtbl_varint_decode32(const uint8_t *'ptr', uint32_t *'value');^ 20 | 21 | ^size_t mtbl_varint_decode64(const uint8_t *'ptr', uint64_t *'value');^ 22 | 23 | == DESCRIPTION == 24 | 25 | ^mtbl_varint_encode32^() and ^mtbl_varint_encode64^() write the 32 or 64 bit 26 | quantity, respectively, in the argument _value_ to the buffer in the argument 27 | _dst_. The quantity will be written in using a variable-width encoding that uses 28 | at most 5 bytes for a 32 bit quantity or 10 bytes for a 64 bit quantity. 29 | 30 | ^mtbl_varint_decode32^() and ^mtbl_varint_decode64^() read the 32 31 | or 64 bit varint quantity, respectively, in the argument _ptr_. The quantity 32 | read will be placed in the argument _value_. 33 | 34 | Bounds checking must be performed by the caller. 35 | 36 | == RETURN VALUE == 37 | 38 | ^mtbl_varint_encode32^() and ^mtbl_varint_encode64^() return the number of 39 | bytes written to _dst_. 40 | 41 | ^mtbl_varint_decode32^() and ^mtbl_varint_decode64^() return the number of 42 | bytes read from _ptr_. 43 | 44 | ^mtbl_varint_length^() returns the number of bytes that its argument _value_ 45 | would require in the variable-width encoding. 46 | 47 | ^mtbl_varint_length_packed^() returns the number of bytes consumed by the 48 | variable-width encoded quantity at its argument _data_. It will read at most 49 | _len_buf_ bytes from _data_. The value 0 is returned if a valid varint is not 50 | present. 51 | -------------------------------------------------------------------------------- /libmy/m4/protobuf-c.m4: -------------------------------------------------------------------------------- 1 | AC_DEFUN([MY_CHECK_LIBPROTOBUF_C], [ 2 | libprotobuf_c_CFLAGS="" 3 | libprotobuf_c_LIBS="-lprotobuf-c" 4 | 5 | AC_ARG_WITH( 6 | [libprotobuf_c], 7 | AC_HELP_STRING([--with-libprotobuf_c=DIR], [libprotobuf-c installation path]), 8 | [], 9 | [withval="yes"] 10 | ) 11 | if test "$withval" = "yes"; then 12 | withval="/usr /usr/local" 13 | fi 14 | 15 | libprotobuf_c_dir="" 16 | 17 | AC_MSG_CHECKING([for libprotobuf-c headers]) 18 | for dir in $withval; do 19 | if test -f "$dir/include/protobuf-c/protobuf-c.h"; then 20 | libprotobuf_c_dir="$dir" 21 | if test "$dir" != "/usr"; then 22 | libprotobuf_c_CFLAGS="-I$dir/include" 23 | fi 24 | break 25 | elif test -f "$dir/include/google/protobuf-c/protobuf-c.h"; then 26 | libprotobuf_c_dir="$dir" 27 | libprotobuf_c_CFLAGS="-I$dir/include/google" 28 | break 29 | fi 30 | done 31 | if test -n "$libprotobuf_c_dir"; then 32 | AC_MSG_RESULT([$libprotobuf_c_dir]) 33 | else 34 | AC_MSG_ERROR([cannot find protobuf-c.h in $withval]) 35 | fi 36 | 37 | save_LDFLAGS="$LDFLAGS" 38 | save_LIBS="$LIBS" 39 | if test "$libprotobuf_c_dir" != "/usr"; then 40 | libprotobuf_c_LIBS="$libprotobuf_c_LIBS -L$libprotobuf_c_dir/lib" 41 | LDFLAGS="-L$libprotobuf_c_dir/lib" 42 | fi 43 | AC_CHECK_LIB( 44 | [protobuf-c], 45 | [protobuf_c_message_pack], 46 | [], 47 | [AC_MSG_ERROR([required library not found])] 48 | ) 49 | LDFLAGS="$save_LDFLAGS" 50 | LIBS="$save_LIBS" 51 | 52 | AC_SUBST([libprotobuf_c_CFLAGS]) 53 | AC_SUBST([libprotobuf_c_LIBS]) 54 | 55 | AC_PATH_PROG([PROTOC_C], [protoc-c]) 56 | if test -z "$PROTOC_C"; then 57 | AC_MSG_ERROR([The protoc-c program was not found. Please install the protobuf-c compiler!]) 58 | fi 59 | ]) 60 | -------------------------------------------------------------------------------- /libmy/m4/ld-version-script.m4: -------------------------------------------------------------------------------- 1 | # ld-version-script.m4 serial 3 2 | dnl Copyright (C) 2008-2014 Free Software Foundation, Inc. 3 | dnl This file is free software; the Free Software Foundation 4 | dnl gives unlimited permission to copy and/or distribute it, 5 | dnl with or without modifications, as long as this notice is preserved. 6 | 7 | dnl From Simon Josefsson 8 | 9 | # FIXME: The test below returns a false positive for mingw 10 | # cross-compiles, 'local:' statements does not reduce number of 11 | # exported symbols in a DLL. Use --disable-ld-version-script to work 12 | # around the problem. 13 | 14 | # gl_LD_VERSION_SCRIPT 15 | # -------------------- 16 | # Check if LD supports linker scripts, and define automake conditional 17 | # HAVE_LD_VERSION_SCRIPT if so. 18 | AC_DEFUN([gl_LD_VERSION_SCRIPT], 19 | [ 20 | AC_ARG_ENABLE([ld-version-script], 21 | AS_HELP_STRING([--enable-ld-version-script], 22 | [enable linker version script (default is enabled when possible)]), 23 | [have_ld_version_script=$enableval], []) 24 | if test -z "$have_ld_version_script"; then 25 | AC_MSG_CHECKING([if LD -Wl,--version-script works]) 26 | save_LDFLAGS="$LDFLAGS" 27 | LDFLAGS="$LDFLAGS -Wl,--version-script=conftest.map" 28 | cat > conftest.map < conftest.map < 21 | #include 22 | #include 23 | #include 24 | 25 | #include "my_alloc.h" 26 | 27 | static inline bool 28 | hex_to_int(char hex, uint8_t *val) 29 | { 30 | hex = toupper(hex); 31 | switch (hex) { 32 | case '0': 33 | case '1': 34 | case '2': 35 | case '3': 36 | case '4': 37 | case '5': 38 | case '6': 39 | case '7': 40 | case '8': 41 | case '9': 42 | *val = (hex - '0'); 43 | return (true); 44 | case 'A': 45 | case 'B': 46 | case 'C': 47 | case 'D': 48 | case 'E': 49 | case 'F': 50 | *val = (hex - 55); 51 | return (true); 52 | default: 53 | return (false); 54 | } 55 | } 56 | 57 | static inline bool 58 | hex_decode(const char *hex, uint8_t **raw, size_t *len) 59 | { 60 | size_t hexlen = strlen(hex); 61 | uint8_t *p; 62 | 63 | if (hexlen == 0 || (hexlen % 2) != 0) 64 | return (false); 65 | *len = hexlen / 2; 66 | p = *raw = my_malloc(*len); 67 | while (hexlen != 0) { 68 | uint8_t val[2]; 69 | 70 | if (!hex_to_int(*hex, &val[0])) 71 | goto err; 72 | hex++; 73 | if (!hex_to_int(*hex, &val[1])) 74 | goto err; 75 | hex++; 76 | 77 | *p = (val[0] << 4) | val[1]; 78 | p++; 79 | 80 | hexlen -= 2; 81 | } 82 | return (true); 83 | err: 84 | my_free(*raw); 85 | return (false); 86 | } 87 | 88 | #endif /* MY_HEX_DECODE_H */ 89 | -------------------------------------------------------------------------------- /mtbl/iter.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2012-2016 by Farsight Security, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "mtbl-private.h" 18 | 19 | struct mtbl_iter { 20 | mtbl_iter_seek_func iter_seek; 21 | mtbl_iter_next_func iter_next; 22 | mtbl_iter_free_func iter_free; 23 | void *clos; 24 | }; 25 | 26 | struct mtbl_iter * 27 | mtbl_iter_init(mtbl_iter_seek_func iter_seek, 28 | mtbl_iter_next_func iter_next, 29 | mtbl_iter_free_func iter_free, 30 | void *clos) 31 | { 32 | assert(iter_seek != NULL); 33 | assert(iter_next != NULL); 34 | struct mtbl_iter *it = my_calloc(1, sizeof(*it)); 35 | it->iter_seek = iter_seek; 36 | it->iter_next = iter_next; 37 | it->iter_free = iter_free; 38 | it->clos = clos; 39 | return (it); 40 | } 41 | 42 | void 43 | mtbl_iter_destroy(struct mtbl_iter **it) 44 | { 45 | if (*it) { 46 | if ((*it)->iter_free != NULL) 47 | (*it)->iter_free((*it)->clos); 48 | free(*it); 49 | *it = NULL; 50 | } 51 | } 52 | 53 | mtbl_res 54 | mtbl_iter_seek(struct mtbl_iter *it, 55 | const uint8_t *key, size_t len_key) 56 | { 57 | if (it == NULL) 58 | return (mtbl_res_failure); 59 | return (it->iter_seek(it->clos, key, len_key)); 60 | } 61 | 62 | mtbl_res 63 | mtbl_iter_next(struct mtbl_iter *it, 64 | const uint8_t **key, size_t *len_key, 65 | const uint8_t **val, size_t *len_val) 66 | { 67 | if (it == NULL) 68 | return (mtbl_res_failure); 69 | return (it->iter_next(it->clos, key, len_key, val, len_val)); 70 | } 71 | -------------------------------------------------------------------------------- /man/mtbl_verify.1: -------------------------------------------------------------------------------- 1 | '\" t 2 | .\" Title: mtbl_verify 3 | .\" Author: [FIXME: author] [see http://docbook.sf.net/el/author] 4 | .\" Generator: DocBook XSL Stylesheets v1.79.1 5 | .\" Date: 04/01/2019 6 | .\" Manual: \ \& 7 | .\" Source: \ \& 8 | .\" Language: English 9 | .\" 10 | .TH "MTBL_VERIFY" "1" "04/01/2019" "\ \&" "\ \&" 11 | .\" ----------------------------------------------------------------- 12 | .\" * Define some portability stuff 13 | .\" ----------------------------------------------------------------- 14 | .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 15 | .\" http://bugs.debian.org/507673 16 | .\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html 17 | .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 18 | .ie \n(.g .ds Aq \(aq 19 | .el .ds Aq ' 20 | .\" ----------------------------------------------------------------- 21 | .\" * set default formatting 22 | .\" ----------------------------------------------------------------- 23 | .\" disable hyphenation 24 | .nh 25 | .\" disable justification (adjust text to left margin only) 26 | .ad l 27 | .\" ----------------------------------------------------------------- 28 | .\" * MAIN CONTENT STARTS HERE * 29 | .\" ----------------------------------------------------------------- 30 | .SH "NAME" 31 | mtbl_verify \- verify integrity of an MTBL file\*(Aqs data and index blocks 32 | .SH "SYNOPSIS" 33 | .sp 34 | \fBmtbl_verify\fR \fIFILE\fR [\fIFILE\fR]\&... 35 | .SH "DESCRIPTION" 36 | .sp 37 | \fBmtbl_verify\fR(1) verifies the embedded data and index block checksums in the MTBL files specified on the command line\&. 38 | .sp 39 | Each filename specified on the command line will be verified, and a message indicating whether each file passed or failed will be printed to stdout\&. If a file fails, additional information about the failure will be printed to stderr\&. 40 | .sp 41 | If stdout is a terminal or the \fB\-p\fR option is provided, periodic progress indications for successful verification will be printed for every 10,000 blocks and at the final block.\&. 42 | -------------------------------------------------------------------------------- /libmy/m4/ax_define_dir.m4: -------------------------------------------------------------------------------- 1 | # =========================================================================== 2 | # http://www.gnu.org/software/autoconf-archive/ax_define_dir.html 3 | # =========================================================================== 4 | # 5 | # OBSOLETE MACRO 6 | # 7 | # Deprecated because it does not comply with the GNU Coding Standards. See 8 | # the autoconf manual section "Defining Directories" for alternatives. 9 | # 10 | # SYNOPSIS 11 | # 12 | # AX_DEFINE_DIR(VARNAME, DIR [, DESCRIPTION]) 13 | # 14 | # DESCRIPTION 15 | # 16 | # This macro sets VARNAME to the expansion of the DIR variable, taking 17 | # care of fixing up ${prefix} and such. 18 | # 19 | # VARNAME is then offered as both an output variable and a C preprocessor 20 | # symbol. 21 | # 22 | # Example: 23 | # 24 | # AX_DEFINE_DIR([DATADIR], [datadir], [Where data are placed to.]) 25 | # 26 | # LICENSE 27 | # 28 | # Copyright (c) 2008 Stepan Kasal 29 | # Copyright (c) 2008 Andreas Schwab 30 | # Copyright (c) 2008 Guido U. Draheim 31 | # Copyright (c) 2008 Alexandre Oliva 32 | # 33 | # Copying and distribution of this file, with or without modification, are 34 | # permitted in any medium without royalty provided the copyright notice 35 | # and this notice are preserved. This file is offered as-is, without any 36 | # warranty. 37 | 38 | #serial 8 39 | 40 | AU_ALIAS([AC_DEFINE_DIR], [AX_DEFINE_DIR]) 41 | AC_DEFUN([AX_DEFINE_DIR], [ 42 | prefix_NONE= 43 | exec_prefix_NONE= 44 | test "x$prefix" = xNONE && prefix_NONE=yes && prefix=$ac_default_prefix 45 | test "x$exec_prefix" = xNONE && exec_prefix_NONE=yes && exec_prefix=$prefix 46 | dnl In Autoconf 2.60, ${datadir} refers to ${datarootdir}, which in turn 47 | dnl refers to ${prefix}. Thus we have to use `eval' twice. 48 | eval ax_define_dir="\"[$]$2\"" 49 | eval ax_define_dir="\"$ax_define_dir\"" 50 | AC_SUBST($1, "$ax_define_dir") 51 | AC_DEFINE_UNQUOTED($1, "$ax_define_dir", [$3]) 52 | test "$prefix_NONE" && prefix=NONE 53 | test "$exec_prefix_NONE" && exec_prefix=NONE 54 | ]) 55 | -------------------------------------------------------------------------------- /mtbl/threadpool.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2024 DomainTools LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /* Public interface */ 18 | 19 | struct mtbl_threadpool { 20 | struct threadpool *pool; 21 | }; 22 | 23 | /* External API, through mtbl wrapper */ 24 | struct threadpool; 25 | 26 | /* 27 | * Initialize a pool of at most `max_threads` threads. 28 | * This pool can be passed to multiple thread users. 29 | */ 30 | struct threadpool *threadpool_init(size_t max_threads); 31 | 32 | /* Destroy a pool of threads, after waiting for all threads to finish. */ 33 | void threadpool_destroy(struct threadpool **poolp); 34 | 35 | 36 | /* Internal API. */ 37 | struct result_handler; 38 | 39 | typedef void *(*thread_cb)(void *arg); 40 | typedef void (*result_cb)(void *res, void *cbdata); 41 | 42 | /* Initialize a handler for worker thread results. */ 43 | struct result_handler *result_handler_init(result_cb cb, void *cbdata); 44 | 45 | /* 46 | * threadpool_dispatch runs `cb(arg)` in a thread from pool `pool`, 47 | * passing its result to the result handler's callback. 48 | * 49 | * If `ordered` is true, results will be passed to the result handler 50 | * callback in the order `threadpool_dispatch` was called, otherwise 51 | * they will be passed as each worker thread finishes. 52 | */ 53 | 54 | void threadpool_dispatch(struct threadpool *pool, 55 | struct result_handler *rh, 56 | bool ordered, 57 | thread_cb cb, void *arg); 58 | 59 | /* 60 | * Free all resources associated with the result handler *rhp after 61 | * waiting for the result handler thread and all worker threads to 62 | * finish. 63 | */ 64 | void result_handler_destroy(struct result_handler **rhp); 65 | -------------------------------------------------------------------------------- /t/test-fixed.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include 8 | 9 | #define NAME "test-fixed" 10 | 11 | static int 12 | do_64(uint64_t val) 13 | { 14 | uint8_t buf[8]; 15 | 16 | mtbl_fixed_encode64(buf, val); 17 | fprintf(stderr, "%s: buf= 0x%02x%02x%02x%02x%02x%02x%02x%02x val= %" PRIu64 "\n", 18 | __func__, 19 | buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6], buf[7], 20 | val); 21 | if (mtbl_fixed_decode64(buf) != val) { 22 | fprintf(stderr, "%s: val= %" PRIu64 " failed\n", __func__, val); 23 | return (0); 24 | } 25 | 26 | return (1); 27 | } 28 | 29 | static int 30 | test2(void) 31 | { 32 | int ret = 0; 33 | unsigned count = 0; 34 | uint64_t val; 35 | 36 | if (!do_64(0) || !do_64(1) || !do_64(0xffffffffffffffffU)) 37 | ret |= 1; 38 | 39 | for (uint64_t i = 1; i < UINT64_MAX; i *= 17) { 40 | val = i++; 41 | if (!do_64(val)) 42 | ret |= 1; 43 | if (count++ > 10) 44 | break; 45 | } 46 | 47 | return (ret); 48 | } 49 | 50 | static int 51 | do_32(uint32_t val) 52 | { 53 | uint8_t buf[4]; 54 | 55 | mtbl_fixed_encode32(buf, val); 56 | fprintf(stderr, "%s: buf= 0x%02x%02x%02x%02x val= %u\n", 57 | __func__, buf[0], buf[1], buf[2], buf[3], val); 58 | if (mtbl_fixed_decode32(buf) != val) { 59 | fprintf(stderr, "%s: val= %u failed\n", __func__, val); 60 | return (0); 61 | } 62 | 63 | return (1); 64 | } 65 | 66 | static int 67 | test1(void) 68 | { 69 | int ret = 0; 70 | 71 | if (!do_32(0) || !do_32(1) || !do_32(0xffffffffU)) 72 | ret |= 1; 73 | 74 | for (uint64_t i = 1; i < UINT32_MAX; i *= 23) { 75 | uint32_t val = (uint32_t) i++; 76 | if (!do_32(val)) 77 | ret |= 1; 78 | } 79 | 80 | return (ret); 81 | } 82 | 83 | static int 84 | check(int ret, const char *s) 85 | { 86 | if (ret == 0) 87 | fprintf(stderr, NAME ": PASS: %s\n", s); 88 | else 89 | fprintf(stderr, NAME ": FAIL: %s\n", s); 90 | return (ret); 91 | } 92 | 93 | int 94 | main(int argc, char **argv) 95 | { 96 | int ret = 0; 97 | 98 | ret |= check(test1(), "test1"); 99 | ret |= check(test2(), "test2"); 100 | 101 | if (ret) 102 | return (EXIT_FAILURE); 103 | return (EXIT_SUCCESS); 104 | } 105 | -------------------------------------------------------------------------------- /man/mtbl_source.3.txt: -------------------------------------------------------------------------------- 1 | = mtbl_source(3) = 2 | 3 | == NAME == 4 | 5 | mtbl_source - obtain key-value entries from a data source 6 | 7 | == SYNOPSIS == 8 | 9 | ^#include ^ 10 | 11 | [verse] 12 | ^struct mtbl_iter * 13 | mtbl_source_iter(const struct mtbl_source *'s');^ 14 | 15 | [verse] 16 | ^struct mtbl_iter * 17 | mtbl_source_get(const struct mtbl_source *'s', const uint8_t *'key', size_t 'len_key');^ 18 | 19 | [verse] 20 | ^struct mtbl_iter * 21 | mtbl_source_get_prefix( 22 | const struct mtbl_source *'s', 23 | const uint8_t *'prefix', size_t 'len_prefix');^ 24 | 25 | [verse] 26 | ^struct mtbl_iter * 27 | mtbl_source_get_range( 28 | const struct mtbl_source *'s', 29 | const uint8_t *'key0', size_t 'len_key0', 30 | const uint8_t *'key1', size_t 'len_key1');^ 31 | 32 | [verse] 33 | ^mtbl_res 34 | mtbl_source_write(const struct mtbl_source *'s', struct mtbl_writer *'w');^ 35 | 36 | [verse] 37 | ^void 38 | mtbl_source_destroy(struct mtbl_source **'s');^ 39 | 40 | == DESCRIPTION == 41 | 42 | The ^mtbl_source^ interface provides an abstraction for reading key-value 43 | entries from mtbl data sources. 44 | 45 | ^mtbl_source_iter^() provides an iterator over all of the entries in the data 46 | source. 47 | 48 | ^mtbl_source_get^() provides an exact match iterator which returns all entries 49 | whose key matches the key provided in the arguments _key_ and _len_key_. 50 | 51 | ^mtbl_source_get_prefix^() provides a prefix iterator which returns all entries 52 | whose keys start with _prefix_ and are at least _len_prefix_ bytes long. 53 | 54 | ^mtbl_source_get_range^() provides a range iterator which returns all entries 55 | whose keys are between _key0_ and _key1_ inclusive. 56 | 57 | ^mtbl_source_write^() is a convenience function for reading all of the entries 58 | from a source and writing them to an ^mtbl_writer^ object. It is equivalent to 59 | calling ^mtbl_writer_add^() on all of the entries returned from 60 | ^mtbl_source_iter^(). 61 | 62 | == RETURN VALUE == 63 | 64 | ^mtbl_source_iter^(), ^mtbl_source_get^(), ^mtbl_source_get_prefix^(), 65 | and ^mtbl_source_get_range^() return ^mtbl_iter^ objects. 66 | 67 | ^mtbl_source_write^() returns ^mtbl_res_success^ if all of the entries in the 68 | data source were successfully written to the ^mtbl_writer^ argument, and 69 | ^mtbl_res_failure^ otherwise. 70 | 71 | == SEE ALSO == 72 | 73 | link:mtbl_iter[3] 74 | -------------------------------------------------------------------------------- /libmy/my_time.h: -------------------------------------------------------------------------------- 1 | #ifndef MY_TIME_H 2 | #define MY_TIME_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #if HAVE_CLOCK_GETTIME 10 | static inline void 11 | my_gettime(clockid_t clk_id, struct timespec *ts) 12 | { 13 | int res; 14 | res = clock_gettime(clk_id, ts); 15 | assert(res == 0); 16 | } 17 | #else 18 | static inline void 19 | my_gettime(int clk_id __attribute__((unused)), struct timespec *ts) 20 | { 21 | struct timeval tv; 22 | int res; 23 | 24 | res = gettimeofday(&tv, NULL); 25 | assert(res == 0); 26 | 27 | ts->tv_sec = tv.tv_sec; 28 | ts->tv_nsec = tv.tv_usec * 1000; 29 | } 30 | #endif 31 | 32 | static inline void 33 | my_timespec_add(const struct timespec *a, struct timespec *b) { 34 | b->tv_sec += a->tv_sec; 35 | b->tv_nsec += a->tv_nsec; 36 | while (b->tv_nsec >= 1000000000) { 37 | b->tv_sec += 1; 38 | b->tv_nsec -= 1000000000; 39 | } 40 | } 41 | 42 | static inline void 43 | my_timespec_sub(const struct timespec *a, struct timespec *b) 44 | { 45 | b->tv_sec -= a->tv_sec; 46 | b->tv_nsec -= a->tv_nsec; 47 | if (b->tv_nsec < 0) { 48 | b->tv_sec -= 1; 49 | b->tv_nsec += 1000000000; 50 | } 51 | } 52 | 53 | /* 54 | * Comparison function for struct timespec suitable for use with bsearch, 55 | * qsort, etc. Returns an integer less than, equal to, or greater than zero 56 | * if the first argument is considered to be respectively less than, equal 57 | * to, or greater than the second. 58 | */ 59 | static inline int 60 | my_timespec_cmp(const struct timespec *a, const struct timespec *b) 61 | { 62 | if (a->tv_sec < b->tv_sec) 63 | return (-1); 64 | else if (a->tv_sec > b->tv_sec) 65 | return (1); 66 | else if (a->tv_nsec < b->tv_nsec) 67 | return (-1); 68 | else if (a->tv_nsec > b->tv_nsec) 69 | return (1); 70 | else 71 | return (0); 72 | } 73 | 74 | static inline double 75 | my_timespec_to_double(const struct timespec *ts) 76 | { 77 | return (ts->tv_sec + ts->tv_nsec / 1E9); 78 | } 79 | 80 | static inline void 81 | my_timespec_from_double(double seconds, struct timespec *ts) { 82 | ts->tv_sec = (time_t) seconds; 83 | ts->tv_nsec = (long) ((seconds - ((int) seconds)) * 1E9); 84 | } 85 | 86 | static inline void 87 | my_nanosleep(const struct timespec *ts) 88 | { 89 | struct timespec rqt, rmt; 90 | 91 | for (rqt = *ts; nanosleep(&rqt, &rmt) < 0 && errno == EINTR; rqt = rmt) 92 | ; 93 | } 94 | 95 | #endif /* MY_TIME_H */ 96 | -------------------------------------------------------------------------------- /t/test-metadata.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | #include "libmy/b64_encode.h" 11 | 12 | #include "metadata.c" 13 | 14 | #define NAME "test-metadata" 15 | 16 | static int 17 | test2(void) 18 | { 19 | int ret = 0; 20 | struct mtbl_metadata t; 21 | uint8_t tbuf[MTBL_METADATA_SIZE]; 22 | 23 | memset(tbuf, 0, sizeof(tbuf)); 24 | 25 | if (metadata_read(tbuf, &t) != false) { 26 | fprintf(stderr, NAME ": metadata_read() should have failed but didn't\n"); 27 | ret |= 1; 28 | } 29 | 30 | return (ret); 31 | } 32 | 33 | #define TEST_ATTR(x, y, attr) \ 34 | do { \ 35 | if ((x).attr != (y).attr) { \ 36 | fprintf(stderr, NAME ": " #x "." #attr " != " #y "." #attr); \ 37 | ret |= 1; \ 38 | } \ 39 | } while (0); 40 | 41 | static int 42 | test1(void) 43 | { 44 | int ret = 0; 45 | struct mtbl_metadata m1, m2; 46 | uint8_t tbuf[MTBL_METADATA_SIZE]; 47 | 48 | memset(tbuf, 0, sizeof(tbuf)); 49 | 50 | m1.file_version = MTBL_FORMAT_V2; 51 | m1.index_block_offset = 123; 52 | m1.data_block_size = 65536; 53 | m1.compression_algorithm = 1; 54 | m1.count_entries = 2; 55 | m1.count_data_blocks = 3; 56 | m1.bytes_data_blocks = 4; 57 | m1.bytes_index_block = 5; 58 | m1.bytes_keys = 6; 59 | m1.bytes_values = 7; 60 | 61 | metadata_write(&m1, tbuf); 62 | if (!metadata_read(tbuf, &m2)) { 63 | fprintf(stderr, NAME ": metadata_read() failed\n"); 64 | ret |= 1; 65 | } 66 | 67 | TEST_ATTR(m1, m2, file_version); 68 | TEST_ATTR(m1, m2, index_block_offset); 69 | TEST_ATTR(m1, m2, data_block_size); 70 | TEST_ATTR(m1, m2, compression_algorithm); 71 | TEST_ATTR(m1, m2, count_entries); 72 | TEST_ATTR(m1, m2, count_data_blocks); 73 | TEST_ATTR(m1, m2, bytes_data_blocks); 74 | TEST_ATTR(m1, m2, bytes_index_block); 75 | TEST_ATTR(m1, m2, bytes_keys); 76 | TEST_ATTR(m1, m2, bytes_values); 77 | 78 | return (ret); 79 | } 80 | 81 | static int 82 | check(int ret, const char *s) 83 | { 84 | if (ret == 0) 85 | fprintf(stderr, NAME ": PASS: %s\n", s); 86 | else 87 | fprintf(stderr, NAME ": FAIL: %s\n", s); 88 | return (ret); 89 | } 90 | 91 | int 92 | main(int argc, char **argv) 93 | { 94 | int ret = 0; 95 | 96 | ret |= check(test1(), "test1"); 97 | ret |= check(test2(), "test2"); 98 | 99 | if (ret) 100 | return (EXIT_FAILURE); 101 | return (EXIT_SUCCESS); 102 | } 103 | -------------------------------------------------------------------------------- /libmy/b32_decode.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015 by Farsight Security, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /* 18 | * Copyright (c) 2006 Christian Biere 19 | * All rights reserved. 20 | * 21 | * Redistribution and use in source and binary forms, with or without 22 | * modification, are permitted provided that the following conditions 23 | * are met: 24 | * 25 | * 1. Redistributions of source code must retain the above copyright 26 | * notice, this list of conditions and the following disclaimer. 27 | * 2. Redistributions in binary form must reproduce the above copyright 28 | * notice, this list of conditions and the following disclaimer in the 29 | * documentation and/or other materials provided with the distribution. 30 | * 3. Neither the name of the authors nor the names of its contributors 31 | * may be used to endorse or promote products derived from this software 32 | * without specific prior written permission. 33 | * 34 | * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 35 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 36 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 37 | * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 38 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 39 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 40 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 41 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 42 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 43 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 44 | * SUCH DAMAGE. 45 | */ 46 | 47 | #ifndef B32_DECODE_H 48 | #define B32_DECODE_H 49 | 50 | size_t base32_decode(void *dst, size_t size, const char *data, size_t len); 51 | 52 | #endif /* B32_DECODE_H */ 53 | -------------------------------------------------------------------------------- /man/mtbl_threadpool.3: -------------------------------------------------------------------------------- 1 | '\" t 2 | .\" Title: mtbl_threadpool 3 | .\" Author: [FIXME: author] [see http://www.docbook.org/tdg5/en/html/author] 4 | .\" Generator: DocBook XSL Stylesheets vsnapshot 5 | .\" Date: 07/10/2024 6 | .\" Manual: \ \& 7 | .\" Source: \ \& 8 | .\" Language: English 9 | .\" 10 | .TH "MTBL_THREADPOOL" "3" "07/10/2024" "\ \&" "\ \&" 11 | .\" ----------------------------------------------------------------- 12 | .\" * Define some portability stuff 13 | .\" ----------------------------------------------------------------- 14 | .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 15 | .\" http://bugs.debian.org/507673 16 | .\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html 17 | .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 18 | .ie \n(.g .ds Aq \(aq 19 | .el .ds Aq ' 20 | .\" ----------------------------------------------------------------- 21 | .\" * set default formatting 22 | .\" ----------------------------------------------------------------- 23 | .\" disable hyphenation 24 | .nh 25 | .\" disable justification (adjust text to left margin only) 26 | .ad l 27 | .\" ----------------------------------------------------------------- 28 | .\" * MAIN CONTENT STARTS HERE * 29 | .\" ----------------------------------------------------------------- 30 | .SH "NAME" 31 | mtbl_threadpool \- create a shared worker threadpool 32 | .SH "SYNOPSIS" 33 | .sp 34 | \fB#include \fR 35 | .sp 36 | .nf 37 | \fBstruct mtbl_threadpool * 38 | mtbl_threadpool_init(size_t \fR\fB\fIthread_count\fR\fR\fB);\fR 39 | .fi 40 | .sp 41 | .nf 42 | \fBvoid 43 | mtbl_threadpool_destroy(struct mtbl_threadpool **\fR\fB\fIpool\fR\fR\fB);\fR 44 | .fi 45 | .SH "DESCRIPTION" 46 | .sp 47 | Certain MTBL "option" structures accept an \fBmtbl_threadpool\fR option (e\&.g\&. \fBmtbl_writer_options\fR, \fBmtbl_sorter_options\fR) to enable internal concurrency\&. The user\-provided \fBmtbl_threadpool\fR object must be initialized before use by calling \fBmtbl_threadpool_init()\fR, and must be destroyed after use by calling \fBmtbl_threadpool_destroy()\fR\&. 48 | .sp 49 | If the \fIthread_count\fR parameter to \fBmtbl_threadpool_init()\fR is 0, multithreading will be disabled\&. Regardless, a non\-NULL \fBmtbl_threadpool\fR object will be returned from \fBmtbl_threadpool_init()\fR\&. 50 | .SS "Threadpool options" 51 | .sp 52 | .it 1 an-trap 53 | .nr an-no-space-flag 1 54 | .nr an-break-flag 1 55 | .br 56 | .ps +1 57 | \fBthread_count\fR 58 | .RS 4 59 | .sp 60 | The maximum number of worker threads that the threadpool will open\&. 61 | .RE 62 | .SH "RETURN VALUE" 63 | .sp 64 | \fBmtbl_threadpool_init\fR() returns NULL on failure, and non\-NULL on success\&. 65 | -------------------------------------------------------------------------------- /libmy/b32_encode.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 by Internet Systems Consortium, Inc. ("ISC") 3 | * 4 | * Permission to use, copy, modify, and/or distribute this software for any 5 | * purpose with or without fee is hereby granted, provided that the above 6 | * copyright notice and this permission notice appear in all copies. 7 | * 8 | * THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES 9 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 10 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR 11 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 12 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 13 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT 14 | * OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 15 | */ 16 | 17 | /* 18 | * Copyright (c) 2006 Christian Biere 19 | * All rights reserved. 20 | * 21 | * Redistribution and use in source and binary forms, with or without 22 | * modification, are permitted provided that the following conditions 23 | * are met: 24 | * 25 | * 1. Redistributions of source code must retain the above copyright 26 | * notice, this list of conditions and the following disclaimer. 27 | * 2. Redistributions in binary form must reproduce the above copyright 28 | * notice, this list of conditions and the following disclaimer in the 29 | * documentation and/or other materials provided with the distribution. 30 | * 3. Neither the name of the authors nor the names of its contributors 31 | * may be used to endorse or promote products derived from this software 32 | * without specific prior written permission. 33 | * 34 | * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 35 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 36 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 37 | * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 38 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 39 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 40 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 41 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 42 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 43 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 44 | * SUCH DAMAGE. 45 | */ 46 | 47 | #ifndef B32_ENCODE_H 48 | #define B32_ENCODE_H 49 | 50 | size_t base32_encode(char *dst, size_t dst_len, const void *src, size_t src_len); 51 | 52 | #endif /* B32_ENCODE_H */ 53 | -------------------------------------------------------------------------------- /man/mtbl_fixed.3: -------------------------------------------------------------------------------- 1 | '\" t 2 | .\" Title: mtbl_fixed 3 | .\" Author: [FIXME: author] [see http://docbook.sf.net/el/author] 4 | .\" Generator: DocBook XSL Stylesheets v1.78.1 5 | .\" Date: 01/31/2014 6 | .\" Manual: \ \& 7 | .\" Source: \ \& 8 | .\" Language: English 9 | .\" 10 | .TH "MTBL_FIXED" "3" "01/31/2014" "\ \&" "\ \&" 11 | .\" ----------------------------------------------------------------- 12 | .\" * Define some portability stuff 13 | .\" ----------------------------------------------------------------- 14 | .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 15 | .\" http://bugs.debian.org/507673 16 | .\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html 17 | .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 18 | .ie \n(.g .ds Aq \(aq 19 | .el .ds Aq ' 20 | .\" ----------------------------------------------------------------- 21 | .\" * set default formatting 22 | .\" ----------------------------------------------------------------- 23 | .\" disable hyphenation 24 | .nh 25 | .\" disable justification (adjust text to left margin only) 26 | .ad l 27 | .\" ----------------------------------------------------------------- 28 | .\" * MAIN CONTENT STARTS HERE * 29 | .\" ----------------------------------------------------------------- 30 | .SH "NAME" 31 | mtbl_fixed \- Fixed\-width encoding and decoding of 32 and 64 bit integers 32 | .SH "SYNOPSIS" 33 | .sp 34 | \fB#include \fR 35 | .sp 36 | \fBsize_t mtbl_fixed_encode32(uint8_t *\fR\fB\fIdst\fR\fR\fB, uint32_t \fR\fB\fIvalue\fR\fR\fB);\fR 37 | .sp 38 | \fBsize_t mtbl_fixed_encode64(uint8_t *\fR\fB\fIdst\fR\fR\fB, uint64_t \fR\fB\fIvalue\fR\fR\fB);\fR 39 | .sp 40 | \fBuint32_t mtbl_fixed_decode32(const uint8_t *\fR\fB\fIptr\fR\fR\fB);\fR 41 | .sp 42 | \fBuint64_t mtbl_fixed_decode64(const uint8_t *\fR\fB\fIptr\fR\fR\fB);\fR 43 | .SH "DESCRIPTION" 44 | .sp 45 | \fBmtbl_fixed_encode32\fR() and \fBmtbl_fixed_encode64\fR() write the 32 or 64 bit quantity, respectively, in the argument \fIvalue\fR to the buffer in the argument \fIdst\fR\&. The quantity will be written in little endian order, regardless of host architecture\&. 46 | .sp 47 | \fBmtbl_fixed_decode32\fR() and \fBmtbl_fixed_decode64\fR() read and return the 32 or 64 bit quantity, respectively, in the argument \fIptr\fR\&. The quantity will be read in little endian order, regardless of host architecture\&. 48 | .sp 49 | Bounds checking must be performed by the caller\&. 50 | .SH "RETURN VALUE" 51 | .sp 52 | \fBmtbl_fixed_encode32\fR() and \fBmtbl_fixed_encode64\fR() return the number of bytes written to the buffer\&. (4 or 8, respectively\&.) 53 | .sp 54 | \fBmtbl_fixed_decode32\fR() and \fBmtbl_fixed_decode64\fR() return the decoded quantity\&. 55 | -------------------------------------------------------------------------------- /libmy/b64_encode.c: -------------------------------------------------------------------------------- 1 | /* 2 | cencoder.c - c source to a base64 encoding algorithm implementation 3 | 4 | This is part of the libb64 project, and has been placed in the public domain. 5 | For details, see http://sourceforge.net/projects/libb64 6 | */ 7 | 8 | #include "b64_encode.h" 9 | 10 | void base64_init_encodestate(base64_encodestate* state_in) 11 | { 12 | state_in->step = step_A; 13 | state_in->result = 0; 14 | } 15 | 16 | char base64_encode_value(char value_in) 17 | { 18 | static const char* encoding = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; 19 | if (value_in > 63) return '='; 20 | return encoding[(int)value_in]; 21 | } 22 | 23 | int base64_encode_block(const char* plaintext_in, int length_in, char* code_out, base64_encodestate* state_in) 24 | { 25 | const char* plainchar = plaintext_in; 26 | const char* const plaintextend = plaintext_in + length_in; 27 | char* codechar = code_out; 28 | char result; 29 | char fragment; 30 | 31 | result = state_in->result; 32 | 33 | switch (state_in->step) 34 | { 35 | while (1) 36 | { 37 | case step_A: 38 | if (plainchar == plaintextend) 39 | { 40 | state_in->result = result; 41 | state_in->step = step_A; 42 | return codechar - code_out; 43 | } 44 | fragment = *plainchar++; 45 | result = (fragment & 0x0fc) >> 2; 46 | *codechar++ = base64_encode_value(result); 47 | result = (fragment & 0x003) << 4; 48 | case step_B: 49 | if (plainchar == plaintextend) 50 | { 51 | state_in->result = result; 52 | state_in->step = step_B; 53 | return codechar - code_out; 54 | } 55 | fragment = *plainchar++; 56 | result |= (fragment & 0x0f0) >> 4; 57 | *codechar++ = base64_encode_value(result); 58 | result = (fragment & 0x00f) << 2; 59 | case step_C: 60 | if (plainchar == plaintextend) 61 | { 62 | state_in->result = result; 63 | state_in->step = step_C; 64 | return codechar - code_out; 65 | } 66 | fragment = *plainchar++; 67 | result |= (fragment & 0x0c0) >> 6; 68 | *codechar++ = base64_encode_value(result); 69 | result = (fragment & 0x03f) >> 0; 70 | *codechar++ = base64_encode_value(result); 71 | } 72 | } 73 | /* control should not reach here */ 74 | return codechar - code_out; 75 | } 76 | 77 | int base64_encode_blockend(char* code_out, base64_encodestate* state_in) 78 | { 79 | char* codechar = code_out; 80 | 81 | switch (state_in->step) 82 | { 83 | case step_B: 84 | *codechar++ = base64_encode_value(state_in->result); 85 | *codechar++ = '='; 86 | *codechar++ = '='; 87 | break; 88 | case step_C: 89 | *codechar++ = base64_encode_value(state_in->result); 90 | *codechar++ = '='; 91 | break; 92 | case step_A: 93 | break; 94 | } 95 | 96 | return codechar - code_out; 97 | } 98 | 99 | -------------------------------------------------------------------------------- /man/mtbl_metadata.3.txt: -------------------------------------------------------------------------------- 1 | = MTBL_METADATA(3) = 2 | 3 | == NAME == 4 | 5 | mtbl_metadata - get MTBL file metadata 6 | 7 | == SYNOPSIS == 8 | 9 | ^#include ^ 10 | 11 | [verse] 12 | ^mtbl_file_version 13 | mtbl_metadata_file_version(const struct mtbl_metadata *'m');^ 14 | 15 | [verse] 16 | ^uint64_t 17 | mtbl_metadata_index_block_offset(const struct mtbl_metadata *'m');^ 18 | 19 | [verse] 20 | ^uint64_t 21 | mtbl_metadata_data_block_size(const struct mtbl_metadata *'m');^ 22 | 23 | [verse] 24 | ^mtbl_compression_type 25 | mtbl_metadata_compression_algorithm(const struct mtbl_metadata *'m');^ 26 | 27 | [verse] 28 | ^uint64_t 29 | mtbl_metadata_count_entries(const struct mtbl_metadata *'m');^ 30 | 31 | [verse] 32 | ^uint64_t 33 | mtbl_metadata_count_data_blocks(const struct mtbl_metadata *'m');^ 34 | 35 | [verse] 36 | ^uint64_t 37 | mtbl_metadata_bytes_data_blocks(const struct mtbl_metadata *'m');^ 38 | 39 | [verse] 40 | ^uint64_t 41 | mtbl_metadata_bytes_index_block(const struct mtbl_metadata *'m');^ 42 | 43 | [verse] 44 | ^uint64_t 45 | mtbl_metadata_bytes_keys(const struct mtbl_metadata *'m');^ 46 | 47 | [verse] 48 | ^uint64_t 49 | mtbl_metadata_bytes_values(const struct mtbl_metadata *'m');^ 50 | 51 | == DESCRIPTION == 52 | 53 | An ^mtbl_metadata^ object may be obtained from an ^mtbl_reader^(3). 54 | Its accessors export attributes and file statistics that are recorded in the 55 | metadata block. 56 | 57 | == RETURN VALUE == 58 | 59 | === mtbl_metadata_file_version() === 60 | 61 | File format version of the MTBL file. Either MTBL_FORMAT_V1 or MTBL_FORMAT_V2. 62 | 63 | === mtbl_metadata_index_block_offset() === 64 | 65 | Byte offset in the MTBL file where the index begins. 66 | 67 | === mtbl_metadata_data_block_size() === 68 | 69 | Maximum size of an uncompressed data block, see ^mtbl_writer^(3). 70 | 71 | === mtbl_metadata_compression_algorithm() === 72 | 73 | One of the ^compression^ values allowed by ^mtbl_writer^(3). 74 | 75 | === mtbl_metadata_count_entries() === 76 | 77 | Total number of key-value entries. 78 | 79 | === mtbl_metadata_count_data_blocks() === 80 | 81 | Total number of data blocks. 82 | 83 | === mtbl_metadata_bytes_data_blocks() === 84 | 85 | Total number of bytes consumed by data blocks. 86 | 87 | === mtbl_metadata_bytes_index_block() === 88 | 89 | Total number of bytes consumed by the index. 90 | 91 | === mtbl_metadata_bytes_keys() === 92 | 93 | Total number of bytes that all keys would occupy if stored end-to-end in a byte 94 | array with no delimiters. 95 | 96 | === mtbl_metadata_bytes_values() === 97 | 98 | Total number of bytes that all values in the file would occupy if stored 99 | end-to-end in a byte array with no delimiters. 100 | -------------------------------------------------------------------------------- /libmy/ubuf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2012 by Farsight Security, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef MY_UBUF_H 18 | #define MY_UBUF_H 19 | 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | #include "vector.h" 28 | 29 | VECTOR_GENERATE(ubuf, uint8_t); 30 | 31 | static inline ubuf * 32 | ubuf_new(void) 33 | { 34 | return (ubuf_init(64)); 35 | } 36 | 37 | static inline ubuf * 38 | ubuf_dup_cstr(const char *s) 39 | { 40 | size_t len = strlen(s); 41 | ubuf *u = ubuf_init(len + 1); 42 | ubuf_append(u, (const uint8_t *) s, len); 43 | return (u); 44 | } 45 | 46 | static inline void 47 | ubuf_add_cstr(ubuf *u, const char *s) 48 | { 49 | if (ubuf_size(u) > 0 && ubuf_value(u, ubuf_size(u) - 1) == '\x00') 50 | ubuf_clip(u, ubuf_size(u) - 1); 51 | ubuf_append(u, (const uint8_t *) s, strlen(s)); 52 | } 53 | 54 | static inline void 55 | ubuf_cterm(ubuf *u) 56 | { 57 | if (ubuf_size(u) == 0 || 58 | (ubuf_size(u) > 0 && ubuf_value(u, ubuf_size(u) - 1) != '\x00')) 59 | { 60 | ubuf_append(u, (const uint8_t *) "\x00", 1); 61 | } 62 | } 63 | 64 | static inline char * 65 | ubuf_cstr(ubuf *u) 66 | { 67 | ubuf_cterm(u); 68 | return ((char *) ubuf_data(u)); 69 | } 70 | 71 | static inline void 72 | ubuf_add_fmt(ubuf *u, const char *fmt, ...) 73 | { 74 | va_list args, args_copy; 75 | int status, needed; 76 | 77 | if (ubuf_size(u) > 0 && ubuf_value(u, ubuf_size(u) - 1) == '\x00') 78 | ubuf_clip(u, ubuf_size(u) - 1); 79 | 80 | va_start(args, fmt); 81 | 82 | va_copy(args_copy, args); 83 | needed = vsnprintf(NULL, 0, fmt, args_copy); 84 | assert(needed >= 0); 85 | va_end(args_copy); 86 | 87 | ubuf_reserve(u, ubuf_size(u) + needed + 1); 88 | status = vsnprintf((char *) ubuf_ptr(u), needed + 1, fmt, args); 89 | assert(status >= 0); 90 | ubuf_advance(u, needed); 91 | 92 | va_end(args); 93 | } 94 | 95 | static inline void 96 | ubuf_rstrip(ubuf *u, char s) 97 | { 98 | if (ubuf_size(u) > 0 && 99 | ubuf_value(u, ubuf_size(u) - 1) == ((uint8_t) s)) 100 | { 101 | ubuf_clip(u, ubuf_size(u) - 1); 102 | } 103 | } 104 | 105 | #endif /* MY_UBUF_H */ 106 | -------------------------------------------------------------------------------- /libmy/b64_decode.c: -------------------------------------------------------------------------------- 1 | /* 2 | cdecoder.c - c source to a base64 decoding algorithm implementation 3 | 4 | This is part of the libb64 project, and has been placed in the public domain. 5 | For details, see http://sourceforge.net/projects/libb64 6 | */ 7 | 8 | #include "b64_decode.h" 9 | 10 | int base64_decode_value(char value_in) 11 | { 12 | static const char decoding[] = {62,-1,-1,-1,63,52,53,54,55,56,57,58,59,60,61,-1,-1,-1,-2,-1,-1,-1,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,-1,-1,-1,-1,-1,-1,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51}; 13 | static const char decoding_size = sizeof(decoding); 14 | value_in -= 43; 15 | if (value_in < 0 || value_in > decoding_size) return -1; 16 | return decoding[(int)value_in]; 17 | } 18 | 19 | void base64_init_decodestate(base64_decodestate* state_in) 20 | { 21 | state_in->step = step_a; 22 | state_in->plainchar = 0; 23 | } 24 | 25 | int base64_decode_block(const char* code_in, const int length_in, char* plaintext_out, base64_decodestate* state_in) 26 | { 27 | const char* codechar = code_in; 28 | char* plainchar = plaintext_out; 29 | char fragment; 30 | 31 | *plainchar = state_in->plainchar; 32 | 33 | switch (state_in->step) 34 | { 35 | while (1) 36 | { 37 | case step_a: 38 | do { 39 | if (codechar == code_in+length_in) 40 | { 41 | state_in->step = step_a; 42 | state_in->plainchar = *plainchar; 43 | return plainchar - plaintext_out; 44 | } 45 | fragment = (char)base64_decode_value(*codechar++); 46 | } while (fragment < 0); 47 | *plainchar = (fragment & 0x03f) << 2; 48 | case step_b: 49 | do { 50 | if (codechar == code_in+length_in) 51 | { 52 | state_in->step = step_b; 53 | state_in->plainchar = *plainchar; 54 | return plainchar - plaintext_out; 55 | } 56 | fragment = (char)base64_decode_value(*codechar++); 57 | } while (fragment < 0); 58 | *plainchar++ |= (fragment & 0x030) >> 4; 59 | *plainchar = (fragment & 0x00f) << 4; 60 | case step_c: 61 | do { 62 | if (codechar == code_in+length_in) 63 | { 64 | state_in->step = step_c; 65 | state_in->plainchar = *plainchar; 66 | return plainchar - plaintext_out; 67 | } 68 | fragment = (char)base64_decode_value(*codechar++); 69 | } while (fragment < 0); 70 | *plainchar++ |= (fragment & 0x03c) >> 2; 71 | *plainchar = (fragment & 0x003) << 6; 72 | case step_d: 73 | do { 74 | if (codechar == code_in+length_in) 75 | { 76 | state_in->step = step_d; 77 | state_in->plainchar = *plainchar; 78 | return plainchar - plaintext_out; 79 | } 80 | fragment = (char)base64_decode_value(*codechar++); 81 | } while (fragment < 0); 82 | *plainchar++ |= (fragment & 0x03f); 83 | } 84 | } 85 | /* control should not reach here */ 86 | return plainchar - plaintext_out; 87 | } 88 | -------------------------------------------------------------------------------- /COPYRIGHT: -------------------------------------------------------------------------------- 1 | Copyright (c) 2022-2023 DomainTools LLC 2 | Copyright (c) 2012-2021 by Farsight Security, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | 16 | 17 | Copyright (c) 2011 The LevelDB Authors. All rights reserved. 18 | 19 | Redistribution and use in source and binary forms, with or without 20 | modification, are permitted provided that the following conditions are 21 | met: 22 | 23 | * Redistributions of source code must retain the above copyright 24 | notice, this list of conditions and the following disclaimer. 25 | 26 | * Redistributions in binary form must reproduce the above 27 | copyright notice, this list of conditions and the following disclaimer 28 | in the documentation and/or other materials provided with the 29 | distribution. 30 | 31 | * Neither the name of Google Inc. nor the names of its 32 | contributors may be used to endorse or promote products derived from 33 | this software without specific prior written permission. 34 | 35 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 36 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 37 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 38 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 39 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 40 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 41 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 42 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 43 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 44 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 45 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 46 | 47 | 48 | Copyright 2008, Dave Benson. 49 | 50 | Licensed under the Apache License, Version 2.0 (the "License"); you 51 | may not use this file except in compliance with the License. You may 52 | obtain a copy of the License at 53 | 54 | http://www.apache.org/licenses/LICENSE-2.0. 55 | 56 | Unless required by applicable law or agreed to in writing, software 57 | distributed under the License is distributed on an "AS IS" BASIS, 58 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 59 | implied. See the License for the specific language governing 60 | permissions and limitations under the License. 61 | -------------------------------------------------------------------------------- /man/mtbl.7.txt: -------------------------------------------------------------------------------- 1 | = mtbl(7) = 2 | 3 | == NAME == 4 | 5 | mtbl - immutable sorted string library 6 | 7 | == SYNOPSIS == 8 | 9 | ^#include ^ 10 | 11 | ^gcc ['flags'] 'files' -lmtbl ['libraries']^ 12 | 13 | == DESCRIPTION == 14 | 15 | The mtbl library provides interfaces for creating, searching, and merging 16 | Sorted String Table (SSTable) files in the _MTBL_ format, which provide an 17 | immutable mapping of keys to values. Sorted String Tables are compact and 18 | provide fast random access to keys and key ranges. Keys and values are 19 | arbitrary byte arrays, and MTBL SSTables may not contain duplicate keys. 20 | 21 | The six main interfaces provided by the mtbl library are: 22 | 23 | link:mtbl_iter[3]:: 24 | Iterator objects provide a consistent interface for iterating over the 25 | key-value entries returned by other interfaces. 26 | 27 | link:mtbl_source[3]:: 28 | Source objects provide functions for obtaining iterators from an underlying 29 | data source. The ^mtbl_reader^ and ^mtbl_merger^ interfaces provide functions 30 | for obtaining references to a source object. The source methods return an 31 | ^mtbl_iter^ object. 32 | 33 | link:mtbl_reader[3]:: 34 | Reader objects provide read-only access to _MTBL_ files. 35 | 36 | link:mtbl_writer[3]:: 37 | Writer objects initialize a new _MTBL_ file from a sequence of key-value 38 | entries provided by the caller. Keys must be in sorted order based on 39 | lexicographical byte value, and keys may not be duplicated. 40 | 41 | link:mtbl_merger[3]:: 42 | Merger objects receive multiple sequences of key-value entries from one or 43 | more ^mtbl_source^ objects and combine them into a single, sorted sequence. 44 | The combined, merged output sequence is provided via the ^mtbl_source^ 45 | interface. 46 | 47 | link:mtbl_sorter[3]:: 48 | Sorter objects receive a sequence of key-value entries provided by the caller 49 | and return them in sorted order. The caller must provide a callback function 50 | to merge values in the case of entries with duplicate keys. The sorted output 51 | sequence may be retrieved via the ^mtbl_iter^ interface or be dumped to an 52 | ^mtbl_writer^ object. 53 | 54 | link:mtbl_fileset[3]:: 55 | Fileset objects automatically maintain an ^mtbl_source^ built on top of the 56 | ^mtbl_merger^ and ^mtbl_reader^ interfaces. The set of underlying ^mtbl_reader^ 57 | objects is kept synchronized with a "setfile" on disk listing _MTBL_ files. 58 | 59 | Additionally, several utility interfaces are provided: 60 | 61 | link:mtbl_crc32c[3]:: 62 | Calculates the CRC32C checksum of a byte array. 63 | 64 | link:mtbl_fixed[3]:: 65 | Functions for fixed-width encoding and decoding of 32 and 64 bit integers. 66 | 67 | link:mtbl_varint[3]:: 68 | Functions for varint encoding and decoding of 32 and 64 bit integers. 69 | 70 | link:mtbl_threadpool[3]:: 71 | Provides an interface for enabling multithreading on ^mtbl_writer^ and 72 | ^mtbl_sorter^ objects via ^mtbl_threadpool^ objects. 73 | -------------------------------------------------------------------------------- /man/mtbl_iter.3: -------------------------------------------------------------------------------- 1 | '\" t 2 | .\" Title: mtbl_iter 3 | .\" Author: [FIXME: author] [see http://docbook.sf.net/el/author] 4 | .\" Generator: DocBook XSL Stylesheets v1.79.1 5 | .\" Date: 03/29/2019 6 | .\" Manual: \ \& 7 | .\" Source: \ \& 8 | .\" Language: English 9 | .\" 10 | .TH "MTBL_ITER" "3" "03/29/2019" "\ \&" "\ \&" 11 | .\" ----------------------------------------------------------------- 12 | .\" * Define some portability stuff 13 | .\" ----------------------------------------------------------------- 14 | .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 15 | .\" http://bugs.debian.org/507673 16 | .\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html 17 | .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 18 | .ie \n(.g .ds Aq \(aq 19 | .el .ds Aq ' 20 | .\" ----------------------------------------------------------------- 21 | .\" * set default formatting 22 | .\" ----------------------------------------------------------------- 23 | .\" disable hyphenation 24 | .nh 25 | .\" disable justification (adjust text to left margin only) 26 | .ad l 27 | .\" ----------------------------------------------------------------- 28 | .\" * MAIN CONTENT STARTS HERE * 29 | .\" ----------------------------------------------------------------- 30 | .SH "NAME" 31 | mtbl_iter \- iterate over a sequence of key\-value pairs 32 | .SH "SYNOPSIS" 33 | .sp 34 | \fB#include \fR 35 | .sp 36 | .nf 37 | \fBmtbl_res 38 | mtbl_iter_next(struct mtbl_iter *\fR\fB\fIit\fR\fR\fB, 39 | const uint8_t **\fR\fB\fIkey\fR\fR\fB, size_t *\fR\fB\fIlen_key\fR\fR\fB, 40 | const uint8_t **\fR\fB\fIval\fR\fR\fB, size_t *\fR\fB\fIlen_val\fR\fR\fB);\fR 41 | .fi 42 | .sp 43 | .nf 44 | \fBvoid 45 | mtbl_iter_destroy(struct mtbl_iter **\fR\fB\fIit\fR\fR\fB);\fR 46 | .fi 47 | .sp 48 | .nf 49 | \fBmtbl_res 50 | mtbl_iter_seek(struct mtbl_iter *\fR\fB\fIit\fR\fR\fB, 51 | const uint8_t *\fR\fB\fIkey\fR\fR\fB, size_t \fR\fB\fIlen_key\fR\fR\fB);\fR 52 | .fi 53 | .SH "DESCRIPTION" 54 | .sp 55 | The \fBmtbl_iter\fR interface is used to return a sequence of one or more key\-value pairs\&. Once the caller obtains an \fBmtbl_iter\fR object, \fBmtbl_iter_next\fR() should be repeatedly called on it until there are no more key\-value entries to retrieve, at which point the iterator object must be freed by calling \fBmtbl_iter_destroy\fR()\&. \fBmtbl_iter_seek\fR() can be called on the iterator to seek to a different location in the index without having to destroy the iterator and create a new one\&. 56 | .SH "RETURN VALUE" 57 | .sp 58 | \fBmtbl_iter_next\fR() returns \fBmtbl_res_success\fR if a key\-value entry was successfully retrieved, in which case \fIkey\fR and \fIval\fR will point to buffers of length \fIlen_key\fR and \fIlen_val\fR respectively\&. The value \fBmtbl_res_failure\fR is returned if there are no more entries to read, or if the \fIit\fR argument is NULL\&. 59 | .SH "SEE ALSO" 60 | .sp 61 | \fBmtbl_source\fR(3) 62 | -------------------------------------------------------------------------------- /man/mtbl_merge.1.txt: -------------------------------------------------------------------------------- 1 | = mtbl_merge(1) = 2 | 3 | == NAME == 4 | 5 | mtbl_merge - merge MTBL data from multiple input files into a single output file 6 | 7 | == SYNOPSIS == 8 | 9 | User-provided functions: 10 | 11 | [verse] 12 | ^typedef void * 13 | (*mtbl_merge_init_func)(void);^ 14 | 15 | [verse] 16 | ^typedef void 17 | (*mtbl_merge_free_func)(void *clos);^ 18 | 19 | [verse] 20 | ^typedef void 21 | (*mtbl_merge_func)(void *'clos', 22 | const uint8_t *'key', size_t 'len_key', 23 | const uint8_t *'val0', size_t 'len_val0', 24 | const uint8_t *'val1', size_t 'len_val1', 25 | uint8_t **'merged_val', size_t *'len_merged_val');^ 26 | 27 | Command line tool: 28 | 29 | [verse] 30 | ^export MTBL_MERGE_DSO="'libexample.so.0'"^ 31 | ^export MTBL_MERGE_FUNC_PREFIX="'example_merge'"^ 32 | ^mtbl_merge^ [^-b^ 'SIZE'] [^-c^ 'COMPRESSION'] [^-l^ 'LEVEL'] 'INPUT' ['INPUT']... 'OUTPUT' 33 | 34 | == DESCRIPTION == 35 | 36 | ^mtbl_merge^(1) is a command-line driver for the ^mtbl_merger^(3) interface. 37 | The ^mtbl_merger^(3) interface requires a user-provided merge function, which 38 | is loaded from a shared object whose filename is specified in the environment 39 | variable 'MTBL_MERGE_DSO'. 40 | 41 | The user-provided merge function must have the same type as the 42 | 'mtbl_merge_func' function type given above in the synopsis. The symbol name 43 | of the merge function to be loaded from the user-provided DSO will be 44 | constructed by appending "_func" to the string provided in the 45 | 'MTBL_MERGE_FUNC_PREFIX' environment variable, which must be non-empty. 46 | 47 | Additionally, two optional functions may be provided: an "init" function whose 48 | symbol name is "_init" appended to the function prefix, and a "free" function 49 | whose symbol name is "_free" appended to the function prefix. If the "init" 50 | function exists, it will be called at the beginning, before any calls to the 51 | merge function, and the return value from the init function will be passed as 52 | the first argument to the merge function. If the "free" function exists, it will 53 | be called at the end, after any calls to the merge function, and its argument 54 | will be the return value of the "init" function. 55 | 56 | The environment variable 'MTBL_MERGE_BLOCK_SIZE' may optionally be set in order 57 | to configure the MTBL block size (in bytes) of the output file. 58 | 59 | == OPTIONS == 60 | 61 | ^-b^ 'SIZE':: 62 | The uncompressed data block size hint for the output file, in bytes. The 63 | default value if unspecified is 8192 bytes (8 kilobytes). 64 | 65 | ^-c^ 'COMPRESSION':: 66 | The compression algorithm to use for data blocks in the output file. The 67 | default value if unspecified is ^zlib^. See the ^mtbl_info^(1) manpage for 68 | the list of possible compression algorithms. 69 | 70 | ^-l^ 'LEVEL':: 71 | The numeric compression level passed to the compression algorithm. The 72 | default value and valid range depend on the chosen compression algorithm. 73 | 74 | == SEE ALSO == 75 | 76 | ^mtbl_info^(1), ^mtbl_merger^(3) 77 | -------------------------------------------------------------------------------- /man/mtbl_info.1: -------------------------------------------------------------------------------- 1 | '\" t 2 | .\" Title: mtbl_info 3 | .\" Author: [FIXME: author] [see http://docbook.sf.net/el/author] 4 | .\" Generator: DocBook XSL Stylesheets v1.79.1 5 | .\" Date: 12/11/2016 6 | .\" Manual: \ \& 7 | .\" Source: \ \& 8 | .\" Language: English 9 | .\" 10 | .TH "MTBL_INFO" "1" "12/11/2016" "\ \&" "\ \&" 11 | .\" ----------------------------------------------------------------- 12 | .\" * Define some portability stuff 13 | .\" ----------------------------------------------------------------- 14 | .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 15 | .\" http://bugs.debian.org/507673 16 | .\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html 17 | .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 18 | .ie \n(.g .ds Aq \(aq 19 | .el .ds Aq ' 20 | .\" ----------------------------------------------------------------- 21 | .\" * set default formatting 22 | .\" ----------------------------------------------------------------- 23 | .\" disable hyphenation 24 | .nh 25 | .\" disable justification (adjust text to left margin only) 26 | .ad l 27 | .\" ----------------------------------------------------------------- 28 | .\" * MAIN CONTENT STARTS HERE * 29 | .\" ----------------------------------------------------------------- 30 | .SH "NAME" 31 | mtbl_info \- display information about an MTBL file 32 | .SH "SYNOPSIS" 33 | .sp 34 | \fBmtbl_info\fR \fIFILE\fR [\fIFILE\fR]\&... 35 | .SH "DESCRIPTION" 36 | .sp 37 | \fBmtbl_info\fR(1) displays the following information about the MTBL files specified on the command line\&. 38 | .sp 39 | \fIfile name\fR \(em the name of the MTBL file\&. 40 | .sp 41 | \fIfile size\fR \(em the total size of the MTBL file, in bytes\&. 42 | .sp 43 | \fIindex bytes\fR \(em the total number of bytes and proportion of the total file size consumed by the index\&. 44 | .sp 45 | \fIdata block bytes\fR \(em the total number of bytes and proportion of the total file size consumed by data blocks\&. 46 | .sp 47 | \fIdata block size\fR \(em the maximum size of an uncompressed data block\&. 48 | .sp 49 | \fIdata block count\fR \(em the total number of data blocks\&. 50 | .sp 51 | \fIentry count\fR \(em the total number of key\-value entries\&. 52 | .sp 53 | \fIkey bytes\fR \(em the total number of bytes that all keys in the file would occupy if stored end\-to\-end in a byte array with no delimiters\&. 54 | .sp 55 | \fIvalue bytes\fR \(em the total number of bytes that all values in the file would occupy if stored end\-to\-end in a byte array with no delimiters\&. 56 | .sp 57 | \fIcompression algorithm\fR \(em the algorithm used to compress data blocks\&. Possible values are "none", "snappy", "zlib", "lz4", "lz4hc", and "zstd"\&. 58 | .sp 59 | \fIcompactness\fR \(em a rough metric comparing the total number of bytes in the key\-value entries with the total size of the MTBL file\&. It is calculated as (file size) / (key bytes + value bytes), and thus takes into account the gains of data block compression and prefix key compression against the overhead of the index, metadata, and data block offset arrays\&. 60 | -------------------------------------------------------------------------------- /mtbl/fixed.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2012 by Farsight Security, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 18 | // 19 | // Redistribution and use in source and binary forms, with or without 20 | // modification, are permitted provided that the following conditions are 21 | // met: 22 | // 23 | // * Redistributions of source code must retain the above copyright 24 | // notice, this list of conditions and the following disclaimer. 25 | // 26 | // * Redistributions in binary form must reproduce the above 27 | // copyright notice, this list of conditions and the following disclaimer 28 | // in the documentation and/or other materials provided with the 29 | // distribution. 30 | // 31 | // * Neither the name of Google Inc. nor the names of its 32 | // contributors may be used to endorse or promote products derived from 33 | // this software without specific prior written permission. 34 | // 35 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 36 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 37 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 38 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 39 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 40 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 41 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 42 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 43 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 44 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 45 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 46 | 47 | #include "mtbl-private.h" 48 | 49 | size_t 50 | mtbl_fixed_encode32(uint8_t *dst, uint32_t value) 51 | { 52 | value = htole32(value); 53 | memcpy(dst, &value, sizeof(value)); 54 | return (sizeof(uint32_t)); 55 | } 56 | 57 | size_t 58 | mtbl_fixed_encode64(uint8_t *dst, uint64_t value) 59 | { 60 | value = htole64(value); 61 | memcpy(dst, &value, sizeof(value)); 62 | return (sizeof(uint64_t)); 63 | } 64 | 65 | uint32_t 66 | mtbl_fixed_decode32(const uint8_t *ptr) 67 | { 68 | uint32_t result; 69 | memcpy(&result, ptr, sizeof(result)); 70 | return (le32toh(result)); 71 | } 72 | 73 | uint64_t 74 | mtbl_fixed_decode64(const uint8_t *ptr) 75 | { 76 | uint64_t result; 77 | memcpy(&result, ptr, sizeof(result)); 78 | return (le64toh(result)); 79 | } 80 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/farsightsec/mtbl.png?branch=master)](https://travis-ci.org/farsightsec/mtbl) 2 | 3 | `mtbl`: immutable sorted string table library 4 | ============================================= 5 | 6 | Introduction 7 | ------------ 8 | 9 | `mtbl` is a C library implementation of the Sorted String Table (SSTable) data 10 | structure, based on the SSTable implementation in the open source [Google 11 | LevelDB library](http://code.google.com/p/leveldb/). An SSTable is a file 12 | containing an immutable mapping of keys to values. Keys are stored in sorted 13 | order, with an index at the end of the file allowing keys to be located quickly. 14 | 15 | `mtbl` is not a database library. It does not provide an updateable key-value 16 | data store, but rather exposes primitives for creating, searching and merging 17 | SSTable files. Unlike databases which use the SSTable data structure internally 18 | as part of their data store, management of SSTable files -- creation, merging, 19 | deletion, combining of search results from multiple SSTables -- is left to the 20 | discretion of the `mtbl` library user. 21 | 22 | `mtbl` SSTable files consist of a sequence of data blocks containing sorted 23 | key-value pairs, where keys and values are arbitrary byte arrays. Data blocks 24 | are optionally compressed using the 25 | [zlib](http://www.zlib.net/), [LZ4](https://github.com/Cyan4973/lz4), 26 | [zstd](https://github.com/facebook/zstd), or 27 | [Snappy](http://google.github.io/snappy/) compression algorithms. The data 28 | blocks are followed by an index block, allowing for fast searches over the 29 | keyspace. 30 | 31 | The basic `mtbl` interface is the writer, which receives a sequence of key-value 32 | pairs in sorted order with no duplicate keys, and writes them to data blocks in 33 | the SSTable output file. An index containing offsets to data blocks and the last 34 | key in each data block is buffered in memory until the writer object is closed, 35 | at which point the index is written to the end of the SSTable file. This allows 36 | SSTable files to be written in a single pass with sequential I/O operations 37 | only. 38 | 39 | Once written, SSTable files can be searched using the `mtbl` reader interface. 40 | Searches can retrieve key-value pairs based on an exact key match, a key prefix 41 | match, or a key range. Results are retrieved using a simple iterator interface. 42 | 43 | The `mtbl` library also provides two utility interfaces which facilitate a 44 | sort-and-merge workflow for bulk data loading. The sorter interface receives 45 | arbitrarily ordered key-value pairs and provides them in sorted order, buffering 46 | to disk as needed. The merger interface reads from multiple SSTables 47 | simultaneously and provides the key-value pairs from the combined inputs in 48 | sorted order. Since `mtbl` does not allow duplicate keys in an SSTable file, 49 | both the sorter and merger interfaces require a caller-provided merge function 50 | which will be called to merge multiple values for the same key. These interfaces 51 | also make use of sequential I/O operations only. 52 | 53 | The `mtbl` file format was changed in version 1.0.0. Older versions cannot read 54 | the new file format, but newer versions can read both formats. 55 | -------------------------------------------------------------------------------- /libmy/my_queue.h: -------------------------------------------------------------------------------- 1 | #ifndef MY_QUEUE_H 2 | #define MY_QUEUE_H 3 | 4 | #include 5 | 6 | /** 7 | * \file 8 | * 9 | * Fixed-size single-producer / single-consumer queue. 10 | * 11 | * This is a generic queue that supports a single producer thread and a 12 | * single consumer thread. The implementation uses a fixed power-of-2 size 13 | * circular buffer. 14 | * 15 | * The my_queue_insert() and my_queue_remove() functions are "non-blocking"; 16 | * that is, the policies for queue full / queue empty conditions are left to 17 | * the caller. These functions return a boolean indicating whether the queue 18 | * operation succeeded or not. For example, a producer that spins until an 19 | * element is successfully enqueued might look like: 20 | * 21 | * void *item; 22 | * produce_item(&item); 23 | * while (!my_queue_insert(q, item, NULL)); 24 | * 25 | * And a consumer that spins until an element is successfully dequeued 26 | * might look like: 27 | * 28 | * void *item; 29 | * while (!my_queue_remove(q, &item, NULL)); 30 | * consume_item(item); 31 | * 32 | * The my_queue_insert() and my_queue_remove() functions take an optional third 33 | * parameter for returning the spaces remaining in the queue or the count of 34 | * elements remaining, respectively. This allows for more complicated 35 | * coordination between producer and consumer, for instance a consumer thread 36 | * that sleeps when the queue is empty and is woken by the producer when it 37 | * adds an element to an empty queue. 38 | */ 39 | 40 | struct my_queue; 41 | 42 | /** 43 | * Initialize a new queue. 44 | * 45 | * \param[in] num_entries Number of elements in the queue. Must be >=2, and a power-of-2. 46 | * \param[in] size_entry Size in bytes of each queue entry. 47 | * \return Opaque pointer that is NULL on failure or non-NULL on success. 48 | */ 49 | struct my_queue * 50 | my_queue_init(unsigned num_entries, unsigned size_entry); 51 | 52 | /** 53 | * Destroy a queue. 54 | */ 55 | void 56 | my_queue_destroy(struct my_queue **q); 57 | 58 | /** 59 | * Describe the queue implementation type. 60 | */ 61 | const char * 62 | my_queue_impl_type(void); 63 | 64 | /** 65 | * Insert an element into the queue. 66 | * 67 | * \param[in] q Queue object. 68 | * \param[in] elem Element object. 69 | * \param[out] space If non-NULL, pointer to store the number of remaining 70 | * spaces in the queue. 71 | * \return true if the element was inserted into the queue, 72 | * false if the queue is full. 73 | */ 74 | bool 75 | my_queue_insert(struct my_queue *q, void *elem, unsigned *space); 76 | 77 | /** 78 | * Remove an element from the queue. 79 | * 80 | * \param[in] q Queue object. 81 | * \param[out] elem Where the element object will be copied. 82 | * \param[out] count If non-NULL, pointer to store the count of elements 83 | * remaining in the queue. 84 | * \return true if an element was removed from the queue, 85 | * false if the queue is empty. 86 | */ 87 | bool 88 | my_queue_remove(struct my_queue *q, void *elem, unsigned *count); 89 | 90 | struct my_queue_ops { 91 | struct my_queue *(*init)(unsigned, unsigned); 92 | void (*destroy)(struct my_queue **); 93 | const char *(*impl_type)(void); 94 | bool (*insert)(struct my_queue *, void *, unsigned *); 95 | bool (*remove)(struct my_queue *, void *, unsigned *); 96 | }; 97 | 98 | #endif /* MY_QUEUE_H */ 99 | -------------------------------------------------------------------------------- /mtbl/source.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2012 by Farsight Security, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "mtbl-private.h" 18 | 19 | struct mtbl_source { 20 | mtbl_source_iter_func source_iter; 21 | mtbl_source_get_func source_get; 22 | mtbl_source_get_prefix_func source_get_prefix; 23 | mtbl_source_get_range_func source_get_range; 24 | mtbl_source_free_func source_free; 25 | void *clos; 26 | }; 27 | 28 | struct mtbl_source * 29 | mtbl_source_init(mtbl_source_iter_func source_iter, 30 | mtbl_source_get_func source_get, 31 | mtbl_source_get_prefix_func source_get_prefix, 32 | mtbl_source_get_range_func source_get_range, 33 | mtbl_source_free_func source_free, 34 | void *clos) 35 | { 36 | assert(source_iter != NULL); 37 | assert(source_get != NULL); 38 | assert(source_get_prefix != NULL); 39 | assert(source_get_range != NULL); 40 | struct mtbl_source *s = my_calloc(1, sizeof(*s)); 41 | s->source_iter = source_iter; 42 | s->source_get = source_get; 43 | s->source_get_prefix = source_get_prefix; 44 | s->source_get_range = source_get_range; 45 | s->source_free = source_free; 46 | s->clos = clos; 47 | return (s); 48 | } 49 | 50 | void 51 | mtbl_source_destroy(struct mtbl_source **s) 52 | { 53 | if (*s) { 54 | if ((*s)->source_free != NULL) 55 | (*s)->source_free((*s)->clos); 56 | free(*s); 57 | *s = NULL; 58 | } 59 | } 60 | 61 | struct mtbl_iter * 62 | mtbl_source_iter(const struct mtbl_source *s) 63 | { 64 | return (s->source_iter(s->clos)); 65 | } 66 | 67 | struct mtbl_iter * 68 | mtbl_source_get(const struct mtbl_source *s, 69 | const uint8_t *key, size_t len_key) 70 | { 71 | return (s->source_get(s->clos, key, len_key)); 72 | } 73 | 74 | struct mtbl_iter * 75 | mtbl_source_get_prefix(const struct mtbl_source *s, 76 | const uint8_t *key, size_t len_key) 77 | { 78 | return (s->source_get_prefix(s->clos, key, len_key)); 79 | } 80 | 81 | struct mtbl_iter * 82 | mtbl_source_get_range(const struct mtbl_source *s, 83 | const uint8_t *key0, size_t len_key0, 84 | const uint8_t *key1, size_t len_key1) 85 | { 86 | return (s->source_get_range(s->clos, key0, len_key0, key1, len_key1)); 87 | } 88 | 89 | mtbl_res 90 | mtbl_source_write(const struct mtbl_source *s, struct mtbl_writer *w) 91 | { 92 | const uint8_t *key, *val; 93 | size_t len_key, len_val; 94 | struct mtbl_iter *it = mtbl_source_iter(s); 95 | mtbl_res res = mtbl_res_success; 96 | 97 | if (it == NULL) 98 | return (mtbl_res_failure); 99 | while (mtbl_iter_next(it, &key, &len_key, &val, &len_val) == mtbl_res_success) { 100 | res = mtbl_writer_add(w, key, len_key, val, len_val); 101 | if (res != mtbl_res_success) 102 | break; 103 | } 104 | mtbl_iter_destroy(&it); 105 | return (res); 106 | } 107 | -------------------------------------------------------------------------------- /man/mtbl_reader.3.txt: -------------------------------------------------------------------------------- 1 | = MTBL_READER(3) = 2 | 3 | == NAME == 4 | 5 | mtbl_reader - read an MTBL file 6 | 7 | == SYNOPSIS == 8 | 9 | ^#include ^ 10 | 11 | Reader objects: 12 | 13 | [verse] 14 | ^struct mtbl_reader * 15 | mtbl_reader_init(const char *'fname', const struct mtbl_reader_options *'ropt');^ 16 | 17 | [verse] 18 | ^struct mtbl_reader * 19 | mtbl_reader_init_fd(int 'fd', const struct mtbl_reader_options *'ropt');^ 20 | 21 | [verse] 22 | ^void 23 | mtbl_reader_destroy(struct mtbl_reader **'r');^ 24 | 25 | [verse] 26 | ^const struct mtbl_source * 27 | mtbl_reader_source(struct mtbl_reader *'r');^ 28 | 29 | [verse] 30 | ^const struct mtbl_metadata * 31 | mtbl_reader_metadata(struct mtbl_reader *'r');^ 32 | 33 | Reader options: 34 | 35 | [verse] 36 | ^struct mtbl_reader_options * 37 | mtbl_reader_options_init(void);^ 38 | 39 | [verse] 40 | ^void 41 | mtbl_reader_options_destroy(struct mtbl_reader_options **'ropt');^ 42 | 43 | [verse] 44 | ^void 45 | mtbl_reader_options_set_verify_checksums( 46 | struct mtbl_reader_options *'ropt', 47 | bool 'verify_checksums');^ 48 | 49 | [verse] 50 | ^void 51 | mtbl_reader_options_set_madvise_random( 52 | struct mtbl_reader_options *'ropt', 53 | bool 'madvise_random');^ 54 | 55 | == DESCRIPTION == 56 | 57 | MTBL files are accessed by creating an ^mtbl_reader^ object, calling 58 | ^mtbl_reader_source^() to obtain an ^mtbl_source^ handle, and using the 59 | ^mtbl_source^(3) interface to read entries. 60 | 61 | ^mtbl_reader^ objects may be created by calling ^mtbl_reader_init^() with an 62 | _fname_ argument specifying the filename to be opened, or 63 | ^mtbl_reader_init_fd^() may be called with an _fd_ argument specifying an open, 64 | readable file descriptor. Since MTBL files are immutable, the same MTBL file 65 | may be opened and read from concurrently by independent threads or processes. 66 | 67 | If the _ropt_ parameter to ^mtbl_reader_init^() or ^mtbl_reader_init_fd^() is 68 | non-NULL, the parameters specified in the ^mtbl_reader_options^ object will be 69 | configured into the ^mtbl_reader^ object. 70 | 71 | File-level metadata may be accessed using the ^mtbl_metadata^(3) interface, 72 | using the object returned by ^mtbl_reader_metadata^(). Note that the 73 | metadata object is valid only as long as the reader object exists. 74 | 75 | === Reader options === 76 | 77 | ==== verify_checksums ==== 78 | 79 | Specifies whether or not the CRC32C checksum on the index block and each 80 | data block should be verified or not. If _verify_checksums_ is enabled, a 81 | checksum mismatch will cause a fatal runtime error. The default is to not verify 82 | index or data block checksums. 83 | 84 | ==== madvise_random ==== 85 | 86 | Specifies whether the kernel should be advised if the data access patterns are 87 | expected to be random or not. This may hurt some workloads but help others. 88 | The default is to not set this advisory information. 89 | 90 | This option can be explicitly overridden by setting the environment variable 91 | ^MTBL_READER_MADVISE_RANDOM^ to the string ^"0"^ (force disable) or ^"1"^ 92 | (force enable). 93 | 94 | This option only has any effect on systems that have the ^posix_madvise^ or 95 | ^madvise^ system calls. 96 | 97 | == RETURN VALUE == 98 | 99 | ^mtbl_reader_init^() and ^mtbl_reader_init_fd^() return NULL on failure, and 100 | non-NULL on success. 101 | -------------------------------------------------------------------------------- /man/mtbl_varint.3: -------------------------------------------------------------------------------- 1 | '\" t 2 | .\" Title: mtbl_varint 3 | .\" Author: [FIXME: author] [see http://docbook.sf.net/el/author] 4 | .\" Generator: DocBook XSL Stylesheets v1.78.1 5 | .\" Date: 01/31/2014 6 | .\" Manual: \ \& 7 | .\" Source: \ \& 8 | .\" Language: English 9 | .\" 10 | .TH "MTBL_VARINT" "3" "01/31/2014" "\ \&" "\ \&" 11 | .\" ----------------------------------------------------------------- 12 | .\" * Define some portability stuff 13 | .\" ----------------------------------------------------------------- 14 | .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 15 | .\" http://bugs.debian.org/507673 16 | .\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html 17 | .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 18 | .ie \n(.g .ds Aq \(aq 19 | .el .ds Aq ' 20 | .\" ----------------------------------------------------------------- 21 | .\" * set default formatting 22 | .\" ----------------------------------------------------------------- 23 | .\" disable hyphenation 24 | .nh 25 | .\" disable justification (adjust text to left margin only) 26 | .ad l 27 | .\" ----------------------------------------------------------------- 28 | .\" * MAIN CONTENT STARTS HERE * 29 | .\" ----------------------------------------------------------------- 30 | .SH "NAME" 31 | mtbl_varint \- Variable\-width encoding and decoding of 32 and 64 bit integers 32 | .SH "SYNOPSIS" 33 | .sp 34 | \fB#include \fR 35 | .sp 36 | \fBunsigned mtbl_varint_length(uint64_t \fR\fB\fIvalue\fR\fR\fB);\fR 37 | .sp 38 | \fBunsigned mtbl_varint_length_packed(const uint8_t *\fR\fB\fIbuf\fR\fR\fB, size_t \fR\fB\fIlen_buf\fR\fR\fB);\fR 39 | .sp 40 | \fBsize_t mtbl_varint_encode32(uint8_t *\fR\fB\fIptr\fR\fR\fB, uint32_t \fR\fB\fIvalue\fR\fR\fB);\fR 41 | .sp 42 | \fBsize_t mtbl_varint_encode64(uint8_t *\fR\fB\fIptr\fR\fR\fB, uint64_t \fR\fB\fIvalue\fR\fR\fB);\fR 43 | .sp 44 | \fBsize_t mtbl_varint_decode32(const uint8_t *\fR\fB\fIptr\fR\fR\fB, uint32_t *\fR\fB\fIvalue\fR\fR\fB);\fR 45 | .sp 46 | \fBsize_t mtbl_varint_decode64(const uint8_t *\fR\fB\fIptr\fR\fR\fB, uint64_t *\fR\fB\fIvalue\fR\fR\fB);\fR 47 | .SH "DESCRIPTION" 48 | .sp 49 | \fBmtbl_varint_encode32\fR() and \fBmtbl_varint_encode64\fR() write the 32 or 64 bit quantity, respectively, in the argument \fIvalue\fR to the buffer in the argument \fIdst\fR\&. The quantity will be written in using a variable\-width encoding that uses at most 5 bytes for a 32 bit quantity or 10 bytes for a 64 bit quantity\&. 50 | .sp 51 | \fBmtbl_varint_decode32\fR() and \fBmtbl_varint_decode64\fR() read the 32 or 64 bit varint quantity, respectively, in the argument \fIptr\fR\&. The quantity read will be placed in the argument \fIvalue\fR\&. 52 | .sp 53 | Bounds checking must be performed by the caller\&. 54 | .SH "RETURN VALUE" 55 | .sp 56 | \fBmtbl_varint_encode32\fR() and \fBmtbl_varint_encode64\fR() return the number of bytes written to \fIdst\fR\&. 57 | .sp 58 | \fBmtbl_varint_decode32\fR() and \fBmtbl_varint_decode64\fR() return the number of bytes read from \fIptr\fR\&. 59 | .sp 60 | \fBmtbl_varint_length\fR() returns the number of bytes that its argument \fIvalue\fR would require in the variable\-width encoding\&. 61 | .sp 62 | \fBmtbl_varint_length_packed\fR() returns the number of bytes consumed by the variable\-width encoded quantity at its argument \fIdata\fR\&. It will read at most \fIlen_buf\fR bytes from \fIdata\fR\&. The value 0 is returned if a valid varint is not present\&. 63 | -------------------------------------------------------------------------------- /debian/copyright: -------------------------------------------------------------------------------- 1 | Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ 2 | 3 | Files: * 4 | Copyright: 2022-2023 DomainTools LLC 5 | 2012-2021 by Farsight Security, Inc. 6 | License: Apache-2.0 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | . 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | . 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | . 19 | On Debian systems, the full text of the Apache License version 2 can be 20 | found in the file `/usr/share/common-licenses/Apache-2.0'. 21 | 22 | Files: mtbl/block.c mtbl/fixed.c mtbl/varint.c 23 | Copyright: 2011 The LevelDB Authors 24 | License: BSD-3-Clause 25 | Redistribution and use in source and binary forms, with or without 26 | modification, are permitted provided that the following conditions are 27 | met: 28 | . 29 | * Redistributions of source code must retain the above copyright 30 | notice, this list of conditions and the following disclaimer. 31 | . 32 | * Redistributions in binary form must reproduce the above 33 | copyright notice, this list of conditions and the following disclaimer 34 | in the documentation and/or other materials provided with the 35 | distribution. 36 | . 37 | * Neither the name of Google Inc. nor the names of its 38 | contributors may be used to endorse or promote products derived from 39 | this software without specific prior written permission. 40 | . 41 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 42 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 43 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 44 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 45 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 46 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 47 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 48 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 49 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 50 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 51 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 52 | 53 | Files: mtbl/varint.c 54 | Copyright: 2008 Dave Benson 55 | License: Apache-2.0 56 | Licensed under the Apache License, Version 2.0 (the "License"); you 57 | may not use this file except in compliance with the License. You may 58 | obtain a copy of the License at 59 | . 60 | http://www.apache.org/licenses/LICENSE-2.0. 61 | . 62 | Unless required by applicable law or agreed to in writing, software 63 | distributed under the License is distributed on an "AS IS" BASIS, 64 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 65 | implied. See the License for the specific language governing 66 | permissions and limitations under the License. 67 | 68 | Files: debian/* 69 | Copyright: 2012 Robert S. Edmonds 70 | License: permissive 71 | Copying and distribution of this package, with or without modification, 72 | are permitted in any medium without royalty provided the copyright notice 73 | and this notice are preserved. 74 | -------------------------------------------------------------------------------- /t/test-fileset-partition.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include 8 | 9 | #define NAME "test-fileset-partition" 10 | 11 | char * fileset; 12 | 13 | static bool 14 | cb(const char *fn, void *clos) 15 | { 16 | const char *cn = (const char *)clos; 17 | const char *bn; 18 | bn = basename((char*)fn); 19 | return strcmp(bn, cn) == 0; 20 | } 21 | 22 | static void 23 | merge_func(void *clos, 24 | const uint8_t *key, size_t len_key, 25 | const uint8_t *val0, size_t len_val0, 26 | const uint8_t *val1, size_t len_val1, 27 | uint8_t **merged_val, size_t *len_merged_val) 28 | { 29 | } 30 | 31 | static int 32 | test1(void) 33 | { 34 | int ret = 0; 35 | struct mtbl_fileset *fs; 36 | struct mtbl_fileset_options *opt; 37 | struct mtbl_merger *m1, *m2; 38 | const struct mtbl_source *s1, *s2; 39 | struct mtbl_iter *itr; 40 | const uint8_t *key, *val; 41 | size_t len_key, len_val; 42 | 43 | opt = mtbl_fileset_options_init(); 44 | mtbl_fileset_options_set_merge_func(opt, merge_func, NULL); 45 | fs = mtbl_fileset_init(fileset, opt); 46 | 47 | mtbl_fileset_partition(fs, cb, "file1.mtbl", &m1, &m2); 48 | 49 | s1 = mtbl_merger_source(m1); 50 | s2 = mtbl_merger_source(m2); 51 | 52 | itr = mtbl_source_iter(s1); 53 | if (mtbl_iter_next(itr, &key, &len_key, &val, &len_val) != mtbl_res_success) { 54 | fprintf (stderr, NAME ": m1 does not have any keys\n"); 55 | ret++; 56 | } else { 57 | if (strncmp((const char*)key, "file1", len_key) != 0) { 58 | fprintf (stderr, NAME ": '%.*s' != 'file1'\n", (int)len_key, (char*)key); 59 | } 60 | } 61 | if (mtbl_iter_next(itr, &key, &len_key, &val, &len_val) == mtbl_res_success) { 62 | fprintf (stderr, NAME ": m1 has too many keys\n"); 63 | ret++; 64 | } 65 | mtbl_iter_destroy(&itr); 66 | 67 | itr = mtbl_source_iter(s2); 68 | if (mtbl_iter_next(itr, &key, &len_key, &val, &len_val) != mtbl_res_success) { 69 | fprintf (stderr, NAME ": m2 does not have any keys\n"); 70 | ret++; 71 | } else { 72 | if (strncmp((const char*)key, "file2", len_key) != 0) { 73 | fprintf (stderr, NAME ": '%.*s' != 'file2'\n", (int)len_key, (char*)key); 74 | } 75 | } 76 | if (mtbl_iter_next(itr, &key, &len_key, &val, &len_val) != mtbl_res_success) { 77 | fprintf (stderr, NAME ": m2 does not have enough keys\n"); 78 | ret++; 79 | } else { 80 | if (strncmp((const char*)key, "file3", len_key) != 0) { 81 | fprintf (stderr, NAME ": '%.*s' != 'file3'\n", (int)len_key, (char*)key); 82 | } 83 | } 84 | if (mtbl_iter_next(itr, &key, &len_key, &val, &len_val) == mtbl_res_success) { 85 | fprintf (stderr, NAME ": m2 has too many keys\n"); 86 | ret++; 87 | } 88 | mtbl_iter_destroy(&itr); 89 | 90 | mtbl_merger_destroy(&m1); 91 | mtbl_merger_destroy(&m2); 92 | mtbl_fileset_destroy(&fs); 93 | mtbl_fileset_options_destroy(&opt); 94 | 95 | return (ret); 96 | } 97 | 98 | static int 99 | check(int ret, const char *s) 100 | { 101 | if (ret == 0) 102 | fprintf(stderr, NAME ": PASS: %s\n", s); 103 | else 104 | fprintf(stderr, NAME ": FAIL: %s\n", s); 105 | return (ret); 106 | } 107 | 108 | int 109 | main(int argc, char **argv) 110 | { 111 | int ret = 0; 112 | if (argc < 2) { 113 | fprintf(stderr, NAME ": ERROR. Need fileset argument.\n"); 114 | return (EXIT_FAILURE); 115 | } 116 | fileset = argv[1]; 117 | 118 | ret |= check(test1(), "test1"); 119 | 120 | if (ret) 121 | return (EXIT_FAILURE); 122 | return (EXIT_SUCCESS); 123 | } 124 | -------------------------------------------------------------------------------- /man/mtbl_dump.1: -------------------------------------------------------------------------------- 1 | '\" t 2 | .\" Title: mtbl_dump 3 | .\" Author: [FIXME: author] [see http://docbook.sf.net/el/author] 4 | .\" Generator: DocBook XSL Stylesheets v1.79.1 5 | .\" Date: 07/13/2021 6 | .\" Manual: \ \& 7 | .\" Source: \ \& 8 | .\" Language: English 9 | .\" 10 | .TH "MTBL_DUMP" "1" "07/13/2021" "\ \&" "\ \&" 11 | .\" ----------------------------------------------------------------- 12 | .\" * Define some portability stuff 13 | .\" ----------------------------------------------------------------- 14 | .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 15 | .\" http://bugs.debian.org/507673 16 | .\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html 17 | .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 18 | .ie \n(.g .ds Aq \(aq 19 | .el .ds Aq ' 20 | .\" ----------------------------------------------------------------- 21 | .\" * set default formatting 22 | .\" ----------------------------------------------------------------- 23 | .\" disable hyphenation 24 | .nh 25 | .\" disable justification (adjust text to left margin only) 26 | .ad l 27 | .\" ----------------------------------------------------------------- 28 | .\" * MAIN CONTENT STARTS HERE * 29 | .\" ----------------------------------------------------------------- 30 | .SH "NAME" 31 | mtbl_dump \- print key\-value entries from an MTBL file 32 | .SH "SYNOPSIS" 33 | .sp 34 | \fBmtbl_dump\fR [\fI\-s\fR] [\fI\-x\fR] [\fI\-k ABCD\fR] [\fI\-v ABCD\fR] [\fI\-K #\fR] [\fI\-V #\fR] \fIFILE\fR 35 | .SH "DESCRIPTION" 36 | .sp 37 | \fBmtbl_dump\fR(1) prints all key\-value entries from an MTBL file to stdout, in file order\&. 38 | .sp 39 | By default, each entry is printed on its own line, with double quotes surrounding the key and the value, with a single space character separating the two\&. Unprintable characters and embedded double quote characters are escaped using the Python string literal syntax\&. 40 | .sp 41 | With the \fB\-x\fR option, each entry is printed on its own line, formatted as an 8 digit hex length, colon, the hex digit pairs separated by a dash, for each of the key and value\&. Examples: 42 | .sp 43 | .if n \{\ 44 | .RS 4 45 | .\} 46 | .nf 47 | \fB0000006:00\-02\-61\-63\-08\-64 00000003:f2\-e9\-e4\fR 48 | \fB00000004:03\-01\-2d\-00 00000000:\fR 49 | .fi 50 | .if n \{\ 51 | .RE 52 | .\} 53 | .sp 54 | The \fI\-k\fR and \fI\-v\fR options take as arguments a prefix as a hex\-string with no embedded punctuation\&. For example, \fB\-k "00ab"\fR is legal, \fB\-v "00\-ab"\fR is illegal\&. With the \fB\-k\fR option, only entries whose key matches the prefix will be printed\&. With the \fB\-v\fR option, only entries whose value matches the prefix will be printed\&. 55 | .sp 56 | The \fI\-K\fR AND \fI\-V\fR options take as arguments a positive integer which specifies the minimum length of the key or value, respectfully\&. 57 | .sp 58 | \fI\-k\fR, \fI\-v\fR, \fI\-K\fR, and \fI\-V\fR can all be specified, and only entries that match all will be printed\&. 59 | .SH "OPTIONS" 60 | .PP 61 | \fB\-s\fR 62 | .RS 4 63 | Silent mode (don\(cqt output anything)\&. Useful for benchmarking\&. 64 | .RE 65 | .PP 66 | \fB\-x\fR 67 | .RS 4 68 | Use hex mode output, described above\&. 69 | .RE 70 | .PP 71 | \fB\-k\fR 72 | .RS 4 73 | Match on a key prefix, described above\&. 74 | .RE 75 | .PP 76 | \fB\-v\fR 77 | .RS 4 78 | Match on a value prefix, described above\&. 79 | .RE 80 | .PP 81 | \fB\-K\fR 82 | .RS 4 83 | Match on a key prefix by minimal length, described above\&. 84 | .RE 85 | .PP 86 | \fB\-V\fR 87 | .RS 4 88 | Match on a value prefix by minimal length, described above\&. 89 | .RE 90 | -------------------------------------------------------------------------------- /debian/control: -------------------------------------------------------------------------------- 1 | Source: mtbl 2 | Section: libs 3 | Priority: optional 4 | Maintainer: Farsight Security, Inc. 5 | Build-Depends: 6 | debhelper (>= 10~), 7 | dpkg-dev (>= 1.16.0~), 8 | lcov, 9 | liblz4-dev (>= 0.0~r130), 10 | libsnappy-dev, 11 | libzstd-dev, 12 | pkg-config, 13 | zlib1g-dev, 14 | Standards-Version: 4.5.1 15 | 16 | Package: libmtbl-dev 17 | Section: libdevel 18 | Architecture: any 19 | Depends: libmtbl1 (= ${binary:Version}), ${misc:Depends} 20 | Multi-Arch: same 21 | Description: immutable sorted string table library (development files) 22 | mtbl is a C library implementation of the Sorted String Table (SSTable) 23 | data structure, based on the SSTable implementation in the open source 24 | Google LevelDB library. An SSTable is a file containing an immutable 25 | mapping of keys to values. Keys are stored in sorted order, with an 26 | index at the end of the file allowing keys to be located quickly. 27 | . 28 | mtbl is not a database library. It does not provide an updateable 29 | key-value data store, but rather exposes primitives for creating, 30 | searching and merging SSTable files. Unlike databases which use 31 | the SSTable data structure internally as part of their data store, 32 | management of SSTable files -- creation, merging, deletion, combining 33 | of search results from multiple SSTables -- is left to the 34 | discretion of the mtbl library user. 35 | . 36 | This package contains the static library, header file, and documentation 37 | for libmtbl. 38 | 39 | Package: libmtbl1 40 | Architecture: any 41 | Depends: ${misc:Depends}, ${shlibs:Depends} 42 | Pre-Depends: ${misc:Pre-Depends} 43 | Multi-Arch: same 44 | Description: immutable sorted string table library 45 | mtbl is a C library implementation of the Sorted String Table (SSTable) 46 | data structure, based on the SSTable implementation in the open source 47 | Google LevelDB library. An SSTable is a file containing an immutable 48 | mapping of keys to values. Keys are stored in sorted order, with an 49 | index at the end of the file allowing keys to be located quickly. 50 | . 51 | mtbl is not a database library. It does not provide an updateable 52 | key-value data store, but rather exposes primitives for creating, 53 | searching and merging SSTable files. Unlike databases which use 54 | the SSTable data structure internally as part of their data store, 55 | management of SSTable files -- creation, merging, deletion, combining 56 | of search results from multiple SSTables -- is left to the 57 | discretion of the mtbl library user. 58 | . 59 | This package contains the shared library for libmtbl. 60 | 61 | Package: mtbl-bin 62 | Section: utils 63 | Architecture: any 64 | Depends: ${misc:Depends}, ${shlibs:Depends} 65 | Description: immutable sorted string table library (utilities) 66 | mtbl is a C library implementation of the Sorted String Table (SSTable) 67 | data structure, based on the SSTable implementation in the open source 68 | Google LevelDB library. An SSTable is a file containing an immutable 69 | mapping of keys to values. Keys are stored in sorted order, with an 70 | index at the end of the file allowing keys to be located quickly. 71 | . 72 | mtbl is not a database library. It does not provide an updateable 73 | key-value data store, but rather exposes primitives for creating, 74 | searching and merging SSTable files. Unlike databases which use 75 | the SSTable data structure internally as part of their data store, 76 | management of SSTable files -- creation, merging, deletion, combining 77 | of search results from multiple SSTables -- is left to the 78 | discretion of the mtbl library user. 79 | . 80 | This package contains command line utilities for libmtbl. 81 | -------------------------------------------------------------------------------- /libmy/atomic.h: -------------------------------------------------------------------------------- 1 | #ifndef MY_ATOMIC_H 2 | #define MY_ATOMIC_H 3 | 4 | /* public domain, from http://golubenco.org/2007/06/14/atomic-operations/ */ 5 | 6 | /** 7 | * Atomic type. 8 | */ 9 | 10 | typedef struct { 11 | volatile int counter; 12 | } atomic_t; 13 | 14 | #define ATOMIC_INIT(i) { (i) } 15 | 16 | /** 17 | * Read atomic variable 18 | * @param v pointer of type atomic_t 19 | * 20 | * Atomically reads the value of @v. 21 | */ 22 | #define atomic_read(v) ((v)->counter) 23 | 24 | /** 25 | * Set atomic variable 26 | * @param v pointer of type atomic_t 27 | * @param i required value 28 | */ 29 | #define atomic_set(v, i) (((v)->counter) = (i)) 30 | 31 | /** 32 | * Add to the atomic variable 33 | * @param i integer value to add 34 | * @param v pointer of type atomic_t 35 | */ 36 | static inline void 37 | atomic_add(int i, atomic_t *v) { 38 | (void)__sync_add_and_fetch(&v->counter, i); 39 | } 40 | 41 | /** 42 | * Subtract the atomic variable 43 | * @param i integer value to subtract 44 | * @param v pointer of type atomic_t 45 | * 46 | * Atomically subtracts @i from @v. 47 | */ 48 | static inline void 49 | atomic_sub(int i, atomic_t *v) { 50 | (void)__sync_sub_and_fetch(&v->counter, i); 51 | } 52 | 53 | /** 54 | * Read atomic variable and reset to zero. 55 | * 56 | * @param v pointer of type atomic_t 57 | */ 58 | static inline int 59 | atomic_zero(atomic_t *v) { 60 | return (__sync_fetch_and_and(&v->counter, 0)); 61 | } 62 | 63 | /** 64 | * Subtract value from variable and test result 65 | * @param i integer value to subtract 66 | * @param v pointer of type atomic_t 67 | * 68 | * Atomically subtracts @i from @v and returns 69 | * true if the result is zero, or false for all 70 | * other cases. 71 | */ 72 | static inline int 73 | atomic_sub_and_test(int i, atomic_t *v) { 74 | return !(__sync_sub_and_fetch(&v->counter, i)); 75 | } 76 | 77 | /** 78 | * Increment atomic variable 79 | * @param v pointer of type atomic_t 80 | * 81 | * Atomically increments @v by 1. 82 | */ 83 | static inline void 84 | atomic_inc(atomic_t *v) { 85 | (void)__sync_fetch_and_add(&v->counter, 1); 86 | } 87 | 88 | /** 89 | * @brief decrement atomic variable 90 | * @param v: pointer of type atomic_t 91 | * 92 | * Atomically decrements @v by 1. Note that the guaranteed 93 | * useful range of an atomic_t is only 24 bits. 94 | */ 95 | static inline void 96 | atomic_dec(atomic_t *v) { 97 | (void)__sync_fetch_and_sub(&v->counter, 1); 98 | } 99 | 100 | /** 101 | * @brief Decrement and test 102 | * @param v pointer of type atomic_t 103 | * 104 | * Atomically decrements @v by 1 and 105 | * returns true if the result is 0, or false for all other 106 | * cases. 107 | */ 108 | static inline int 109 | atomic_dec_and_test(atomic_t *v) { 110 | return !(__sync_sub_and_fetch(&v->counter, 1)); 111 | } 112 | 113 | /** 114 | * @brief Increment and test 115 | * @param v pointer of type atomic_t 116 | * 117 | * Atomically increments @v by 1 118 | * and returns true if the result is zero, or false for all 119 | * other cases. 120 | */ 121 | static inline int 122 | atomic_inc_and_test(atomic_t *v) { 123 | return !(__sync_add_and_fetch(&v->counter, 1)); 124 | } 125 | 126 | /** 127 | * @brief add and test if negative 128 | * @param v pointer of type atomic_t 129 | * @param i integer value to add 130 | * 131 | * Atomically adds @i to @v and returns true 132 | * if the result is negative, or false when 133 | * result is greater than or equal to zero. 134 | */ 135 | static inline int 136 | atomic_add_negative(int i, atomic_t *v) { 137 | return (__sync_add_and_fetch(&v->counter, i) < 0); 138 | } 139 | 140 | #endif /* MY_ATOMIC_H */ 141 | -------------------------------------------------------------------------------- /libmy/atomic_64.h: -------------------------------------------------------------------------------- 1 | #ifndef MY_ATOMIC_64_H 2 | #define MY_ATOMIC_64_H 3 | 4 | #include 5 | 6 | /* public domain, from http://golubenco.org/2007/06/14/atomic-operations/ */ 7 | 8 | /** 9 | * Atomic 64-bit type. 10 | */ 11 | 12 | typedef struct { 13 | volatile int64_t counter; 14 | } atomic64_t; 15 | 16 | #define ATOMIC64_INIT(i) { (i) } 17 | 18 | /** 19 | * Read atomic variable 20 | * @param v pointer of type atomic64_t 21 | * 22 | * Atomically reads the value of @v. 23 | */ 24 | #define atomic64_read(v) ((v)->counter) 25 | 26 | /** 27 | * Set atomic variable 28 | * @param v pointer of type atomic64_t 29 | * @param i required value 30 | */ 31 | #define atomic64_set(v, i) (((v)->counter) = (i)) 32 | 33 | /** 34 | * Add to the atomic variable 35 | * @param i integer value to add 36 | * @param v pointer of type atomic64_t 37 | */ 38 | static inline void 39 | atomic64_add(int i, atomic64_t *v) { 40 | (void)__sync_add_and_fetch(&v->counter, i); 41 | } 42 | 43 | /** 44 | * Subtract the atomic variable 45 | * @param i integer value to subtract 46 | * @param v pointer of type atomic64_t 47 | * 48 | * Atomically subtracts @i from @v. 49 | */ 50 | static inline void 51 | atomic64_sub(int i, atomic64_t *v) { 52 | (void)__sync_sub_and_fetch(&v->counter, i); 53 | } 54 | 55 | /** 56 | * Read atomic variable and reset to zero. 57 | * 58 | * @param v pointer of type atomic64_t 59 | */ 60 | static inline int 61 | atomic64_zero(atomic64_t *v) { 62 | return (__sync_fetch_and_and(&v->counter, 0)); 63 | } 64 | 65 | /** 66 | * Subtract value from variable and test result 67 | * @param i integer value to subtract 68 | * @param v pointer of type atomic64_t 69 | * 70 | * Atomically subtracts @i from @v and returns 71 | * true if the result is zero, or false for all 72 | * other cases. 73 | */ 74 | static inline int 75 | atomic64_sub_and_test(int i, atomic64_t *v) { 76 | return !(__sync_sub_and_fetch(&v->counter, i)); 77 | } 78 | 79 | /** 80 | * Increment atomic variable 81 | * @param v pointer of type atomic64_t 82 | * 83 | * Atomically increments @v by 1. 84 | */ 85 | static inline void 86 | atomic64_inc(atomic64_t *v) { 87 | (void)__sync_fetch_and_add(&v->counter, 1); 88 | } 89 | 90 | /** 91 | * @brief decrement atomic variable 92 | * @param v: pointer of type atomic64_t 93 | * 94 | * Atomically decrements @v by 1. Note that the guaranteed 95 | * useful range of an atomic64_t is only 24 bits. 96 | */ 97 | static inline void 98 | atomic64_dec(atomic64_t *v) { 99 | (void)__sync_fetch_and_sub(&v->counter, 1); 100 | } 101 | 102 | /** 103 | * @brief Decrement and test 104 | * @param v pointer of type atomic64_t 105 | * 106 | * Atomically decrements @v by 1 and 107 | * returns true if the result is 0, or false for all other 108 | * cases. 109 | */ 110 | static inline int 111 | atomic64_dec_and_test(atomic64_t *v) { 112 | return !(__sync_sub_and_fetch(&v->counter, 1)); 113 | } 114 | 115 | /** 116 | * @brief Increment and test 117 | * @param v pointer of type atomic64_t 118 | * 119 | * Atomically increments @v by 1 120 | * and returns true if the result is zero, or false for all 121 | * other cases. 122 | */ 123 | static inline int 124 | atomic64_inc_and_test(atomic64_t *v) { 125 | return !(__sync_add_and_fetch(&v->counter, 1)); 126 | } 127 | 128 | /** 129 | * @brief add and test if negative 130 | * @param v pointer of type atomic64_t 131 | * @param i integer value to add 132 | * 133 | * Atomically adds @i to @v and returns true 134 | * if the result is negative, or false when 135 | * result is greater than or equal to zero. 136 | */ 137 | static inline int 138 | atomic64_add_negative(int i, atomic64_t *v) { 139 | return (__sync_add_and_fetch(&v->counter, i) < 0); 140 | } 141 | 142 | #endif /* MY_ATOMIC_H */ 143 | -------------------------------------------------------------------------------- /mtbl/libmtbl.sym: -------------------------------------------------------------------------------- 1 | LIBMTBL_1.0.0 { 2 | global: 3 | mtbl_compress; 4 | mtbl_compress_level; 5 | mtbl_compression_type_from_str; 6 | mtbl_compression_type_to_str; 7 | mtbl_crc32c; 8 | mtbl_decompress; 9 | mtbl_fileset_destroy; 10 | mtbl_fileset_init; 11 | mtbl_fileset_options_destroy; 12 | mtbl_fileset_options_init; 13 | mtbl_fileset_options_set_merge_func; 14 | mtbl_fileset_options_set_reload_interval; 15 | mtbl_fileset_reload; 16 | mtbl_fileset_reload_now; 17 | mtbl_fileset_source; 18 | mtbl_fixed_decode32; 19 | mtbl_fixed_decode64; 20 | mtbl_fixed_encode32; 21 | mtbl_fixed_encode64; 22 | mtbl_iter_destroy; 23 | mtbl_iter_init; 24 | mtbl_iter_next; 25 | mtbl_iter_seek; 26 | mtbl_merger_add_source; 27 | mtbl_merger_destroy; 28 | mtbl_merger_init; 29 | mtbl_merger_options_destroy; 30 | mtbl_merger_options_init; 31 | mtbl_merger_options_set_merge_func; 32 | mtbl_merger_source; 33 | mtbl_metadata_bytes_data_blocks; 34 | mtbl_metadata_bytes_index_block; 35 | mtbl_metadata_bytes_keys; 36 | mtbl_metadata_bytes_values; 37 | mtbl_metadata_compression_algorithm; 38 | mtbl_metadata_count_data_blocks; 39 | mtbl_metadata_count_entries; 40 | mtbl_metadata_data_block_size; 41 | mtbl_metadata_file_version; 42 | mtbl_metadata_index_block_offset; 43 | mtbl_reader_destroy; 44 | mtbl_reader_init; 45 | mtbl_reader_init_fd; 46 | mtbl_reader_metadata; 47 | mtbl_reader_options_destroy; 48 | mtbl_reader_options_init; 49 | mtbl_reader_options_set_madvise_random; 50 | mtbl_reader_options_set_verify_checksums; 51 | mtbl_reader_source; 52 | mtbl_sorter_add; 53 | mtbl_sorter_destroy; 54 | mtbl_sorter_init; 55 | mtbl_sorter_iter; 56 | mtbl_sorter_options_destroy; 57 | mtbl_sorter_options_init; 58 | mtbl_sorter_options_set_max_memory; 59 | mtbl_sorter_options_set_merge_func; 60 | mtbl_sorter_options_set_temp_dir; 61 | mtbl_sorter_write; 62 | mtbl_source_destroy; 63 | mtbl_source_get; 64 | mtbl_source_get_prefix; 65 | mtbl_source_get_range; 66 | mtbl_source_init; 67 | mtbl_source_iter; 68 | mtbl_source_write; 69 | mtbl_varint_decode32; 70 | mtbl_varint_decode64; 71 | mtbl_varint_encode32; 72 | mtbl_varint_encode64; 73 | mtbl_varint_length; 74 | mtbl_varint_length_packed; 75 | mtbl_writer_add; 76 | mtbl_writer_destroy; 77 | mtbl_writer_init; 78 | mtbl_writer_init_fd; 79 | mtbl_writer_options_destroy; 80 | mtbl_writer_options_init; 81 | mtbl_writer_options_set_block_restart_interval; 82 | mtbl_writer_options_set_block_size; 83 | mtbl_writer_options_set_compression; 84 | mtbl_writer_options_set_compression_level; 85 | local: 86 | *; 87 | }; 88 | 89 | LIBMTBL_1.1.0 { 90 | global: 91 | mtbl_fileset_partition; 92 | } LIBMTBL_1.0.0; 93 | 94 | LIBMTBL_1.2.0 { 95 | global: 96 | mtbl_merger_options_set_dupsort_func; 97 | mtbl_fileset_options_set_filename_filter_func; 98 | mtbl_fileset_options_set_dupsort_func; 99 | mtbl_fileset_dup; 100 | } LIBMTBL_1.1.0; 101 | 102 | LIBMTBL_1.4.0 { 103 | global: 104 | mtbl_fileset_options_set_reader_filter_func; 105 | } LIBMTBL_1.2.0; 106 | 107 | LIBMTBL_1.7.0 { 108 | global: 109 | mtbl_writer_options_set_threadpool; 110 | mtbl_sorter_options_set_threadpool; 111 | mtbl_threadpool_init; 112 | mtbl_threadpool_destroy; 113 | } LIBMTBL_1.4.0; 114 | -------------------------------------------------------------------------------- /libmy/m4/ax_prog_xsltproc.m4: -------------------------------------------------------------------------------- 1 | # =========================================================================== 2 | # http://www.gnu.org/software/autoconf-archive/ax_prog_xsltproc.html 3 | # =========================================================================== 4 | # 5 | # SYNOPSIS 6 | # 7 | # AX_PROG_XSLTPROC([default-flags]) 8 | # 9 | # DESCRIPTION 10 | # 11 | # Find an xsltproc executable. 12 | # 13 | # Input: 14 | # 15 | # "default-flags" is the default $XSLTPROC_FLAGS, which will be overridden 16 | # if the user specifies --with-xsltproc-flags. 17 | # 18 | # Output: 19 | # 20 | # $XSLTPROC contains the path to xsltproc, or is empty if none was found 21 | # or the user specified --without-xsltproc. $XSLTPROC_FLAGS contains the 22 | # flags to use with xsltproc. 23 | # 24 | # LICENSE 25 | # 26 | # Copyright (c) 2008,2009 Zmanda Inc. 27 | # Copyright (c) 2008,2009 Dustin J. Mitchell 28 | # 29 | # This program is free software; you can redistribute it and/or modify it 30 | # under the terms of the GNU General Public License as published by the 31 | # Free Software Foundation; either version 2 of the License, or (at your 32 | # option) any later version. 33 | # 34 | # This program is distributed in the hope that it will be useful, but 35 | # WITHOUT ANY WARRANTY; without even the implied warranty of 36 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 37 | # Public License for more details. 38 | # 39 | # You should have received a copy of the GNU General Public License along 40 | # with this program. If not, see . 41 | # 42 | # As a special exception, the respective Autoconf Macro's copyright owner 43 | # gives unlimited permission to copy, distribute and modify the configure 44 | # scripts that are the output of Autoconf when processing the Macro. You 45 | # need not follow the terms of the GNU General Public License when using 46 | # or distributing such scripts, even though portions of the text of the 47 | # Macro appear in them. The GNU General Public License (GPL) does govern 48 | # all other use of the material that constitutes the Autoconf Macro. 49 | # 50 | # This special exception to the GPL applies to versions of the Autoconf 51 | # Macro released by the Autoconf Archive. When you make and distribute a 52 | # modified version of the Autoconf Macro, you may extend this special 53 | # exception to the GPL to apply to your modified version as well. 54 | 55 | #serial 5 56 | 57 | AU_ALIAS([AC_PROG_XSLTPROC], [AX_PROG_XSLTPROC]) 58 | AC_DEFUN([AX_PROG_XSLTPROC], 59 | [ 60 | XSLTPROC_FLAGS="$1" 61 | AC_SUBST(XSLTPROC_FLAGS) 62 | 63 | # The (lack of) whitespace and overquoting here are all necessary for 64 | # proper formatting. 65 | AC_ARG_WITH(xsltproc, 66 | AS_HELP_STRING([--with-xsltproc[[[[[=PATH]]]]]], 67 | [Use the xsltproc binary in PATH.]), 68 | [ ac_with_xsltproc=$withval; ], 69 | [ ac_with_xsltproc=maybe; ]) 70 | 71 | AC_ARG_WITH(xsltproc-flags, 72 | AS_HELP_STRING([ --with-xsltproc-flags=FLAGS], 73 | [Flags to pass to xsltproc (default $1)]), 74 | [ if test "x$withval" == "xno"; then 75 | XSLTPROC_FLAGS='' 76 | else 77 | if test "x$withval" != "xyes"; then 78 | XSLTPROC_FLAGS="$withval" 79 | fi 80 | fi 81 | ]) 82 | 83 | # search for xsltproc if it wasn't specified 84 | if test "$ac_with_xsltproc" = "yes" -o "$ac_with_xsltproc" = "maybe"; then 85 | AC_PATH_PROGS(XSLTPROC,xsltproc) 86 | else 87 | if test "$ac_with_xsltproc" != "no"; then 88 | if test -x "$ac_with_xsltproc"; then 89 | XSLTPROC="$ac_with_xsltproc"; 90 | else 91 | AC_MSG_WARN([Specified xsltproc of $ac_with_xsltproc isn't]) 92 | AC_MSG_WARN([executable; searching for an alternative.]) 93 | AC_PATH_PROGS(XSLTPROC,xsltproc) 94 | fi 95 | fi 96 | fi 97 | ]) 98 | -------------------------------------------------------------------------------- /libmy/my_queue_mb.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, 2014 by Farsight Security, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "my_memory_barrier.h" 18 | 19 | #ifdef MY_HAVE_MEMORY_BARRIERS 20 | 21 | #include 22 | #include 23 | #include 24 | 25 | #include "my_alloc.h" 26 | 27 | #include "my_queue.h" 28 | 29 | #define MY_ACCESS_ONCE(x) (*(volatile typeof(x) *)&(x)) 30 | 31 | struct my_queue * 32 | my_queue_mb_init(unsigned, unsigned); 33 | 34 | void 35 | my_queue_mb_destroy(struct my_queue **); 36 | 37 | const char * 38 | my_queue_mb_impl_type(void); 39 | 40 | bool 41 | my_queue_mb_insert(struct my_queue *, void *, unsigned *); 42 | 43 | bool 44 | my_queue_mb_remove(struct my_queue *, void *, unsigned *); 45 | 46 | struct my_queue { 47 | uint8_t *data; 48 | unsigned num_elems; 49 | unsigned sizeof_elem; 50 | unsigned head; 51 | unsigned tail; 52 | }; 53 | 54 | struct my_queue * 55 | my_queue_mb_init(unsigned num_elems, unsigned sizeof_elem) 56 | { 57 | struct my_queue *q; 58 | if (num_elems < 2 || ((num_elems - 1) & num_elems) != 0) 59 | return (NULL); 60 | q = my_calloc(1, sizeof(*q)); 61 | q->num_elems = num_elems; 62 | q->sizeof_elem = sizeof_elem; 63 | q->data = my_calloc(q->num_elems, q->sizeof_elem); 64 | return (q); 65 | } 66 | 67 | void 68 | my_queue_mb_destroy(struct my_queue **q) 69 | { 70 | if (*q) { 71 | free((*q)->data); 72 | free(*q); 73 | *q = NULL; 74 | } 75 | } 76 | 77 | const char * 78 | my_queue_mb_impl_type(void) 79 | { 80 | return ("memory barrier"); 81 | } 82 | 83 | static inline unsigned 84 | q_space(unsigned head, unsigned tail, unsigned size) 85 | { 86 | return ((tail - (head + 1)) & (size - 1)); 87 | } 88 | 89 | static inline unsigned 90 | q_count(unsigned head, unsigned tail, unsigned size) 91 | { 92 | return ((head - tail) & (size - 1)); 93 | } 94 | 95 | bool 96 | my_queue_mb_insert(struct my_queue *q, void *item, unsigned *pspace) 97 | { 98 | bool res = false; 99 | unsigned head = q->head; 100 | unsigned tail = MY_ACCESS_ONCE(q->tail); 101 | unsigned space = q_space(head, tail, q->num_elems); 102 | if (space >= 1) { 103 | memcpy(&q->data[head * q->sizeof_elem], item, q->sizeof_elem); 104 | smp_wmb(); 105 | q->head = (head + 1) & (q->num_elems - 1); 106 | smp_wmb(); 107 | res = true; 108 | space--; 109 | } 110 | if (pspace != NULL) 111 | *pspace = space; 112 | return (res); 113 | } 114 | 115 | bool 116 | my_queue_mb_remove(struct my_queue *q, void *item, unsigned *pcount) 117 | { 118 | bool res = false; 119 | unsigned head = MY_ACCESS_ONCE(q->head); 120 | unsigned tail = q->tail; 121 | unsigned count = q_count(head, tail, q->num_elems); 122 | if (count >= 1) { 123 | memcpy(item, &q->data[tail * q->sizeof_elem], q->sizeof_elem); 124 | smp_mb(); 125 | q->tail = (tail + 1) & (q->num_elems - 1); 126 | res = true; 127 | count--; 128 | } 129 | if (pcount != NULL) 130 | *pcount = count; 131 | return (res); 132 | } 133 | 134 | const struct my_queue_ops my_queue_mb_ops = { 135 | .init = 136 | my_queue_mb_init, 137 | .destroy = 138 | my_queue_mb_destroy, 139 | .impl_type = 140 | my_queue_mb_impl_type, 141 | .insert = 142 | my_queue_mb_insert, 143 | .remove = 144 | my_queue_mb_remove, 145 | }; 146 | 147 | #endif /* MY_HAVE_MEMORY_BARRIERS */ 148 | -------------------------------------------------------------------------------- /mtbl/metadata.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2012-2015, 2017 by Farsight Security, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "mtbl-private.h" 18 | 19 | void 20 | metadata_write(const struct mtbl_metadata *m, uint8_t *buf) 21 | { 22 | size_t padding; 23 | uint8_t *p = buf; 24 | 25 | p += mtbl_fixed_encode64(p, m->index_block_offset); 26 | p += mtbl_fixed_encode64(p, m->data_block_size); 27 | p += mtbl_fixed_encode64(p, m->compression_algorithm); 28 | p += mtbl_fixed_encode64(p, m->count_entries); 29 | p += mtbl_fixed_encode64(p, m->count_data_blocks); 30 | p += mtbl_fixed_encode64(p, m->bytes_data_blocks); 31 | p += mtbl_fixed_encode64(p, m->bytes_index_block); 32 | p += mtbl_fixed_encode64(p, m->bytes_keys); 33 | p += mtbl_fixed_encode64(p, m->bytes_values); 34 | 35 | padding = MTBL_METADATA_SIZE - (p - buf) - sizeof(uint32_t); 36 | while (padding-- != 0) 37 | *(p++) = '\0'; 38 | mtbl_fixed_encode32(buf + MTBL_METADATA_SIZE - sizeof(uint32_t), MTBL_MAGIC); 39 | } 40 | 41 | bool 42 | metadata_read(const uint8_t *buf, struct mtbl_metadata *m) 43 | { 44 | uint32_t magic; 45 | const uint8_t *p = buf; 46 | 47 | magic = mtbl_fixed_decode32(buf + MTBL_METADATA_SIZE - sizeof(uint32_t)); 48 | if (magic == MTBL_MAGIC_V1) 49 | m->file_version = MTBL_FORMAT_V1; 50 | else if (magic == MTBL_MAGIC) 51 | m->file_version = MTBL_FORMAT_V2; 52 | else 53 | return (false); 54 | 55 | m->index_block_offset = mtbl_fixed_decode64(p); p += 8; 56 | m->data_block_size = mtbl_fixed_decode64(p); p += 8; 57 | m->compression_algorithm = mtbl_fixed_decode64(p); p += 8; 58 | m->count_entries = mtbl_fixed_decode64(p); p += 8; 59 | m->count_data_blocks = mtbl_fixed_decode64(p); p += 8; 60 | m->bytes_data_blocks = mtbl_fixed_decode64(p); p += 8; 61 | m->bytes_index_block = mtbl_fixed_decode64(p); p += 8; 62 | m->bytes_keys = mtbl_fixed_decode64(p); p += 8; 63 | m->bytes_values = mtbl_fixed_decode64(p); 64 | 65 | return (true); 66 | 67 | } 68 | 69 | mtbl_file_version 70 | mtbl_metadata_file_version(const struct mtbl_metadata *m) 71 | { 72 | return m->file_version; 73 | } 74 | 75 | uint64_t 76 | mtbl_metadata_index_block_offset(const struct mtbl_metadata *m) 77 | { 78 | return m->index_block_offset; 79 | } 80 | 81 | uint64_t 82 | mtbl_metadata_data_block_size(const struct mtbl_metadata *m) 83 | { 84 | return m->data_block_size; 85 | } 86 | 87 | uint64_t 88 | mtbl_metadata_compression_algorithm(const struct mtbl_metadata *m) 89 | { 90 | return m->compression_algorithm; 91 | } 92 | 93 | uint64_t 94 | mtbl_metadata_count_entries(const struct mtbl_metadata *m) 95 | { 96 | return m->count_entries; 97 | } 98 | 99 | uint64_t 100 | mtbl_metadata_count_data_blocks(const struct mtbl_metadata *m) 101 | { 102 | return m->count_data_blocks; 103 | } 104 | 105 | uint64_t 106 | mtbl_metadata_bytes_data_blocks(const struct mtbl_metadata *m) 107 | { 108 | return m->bytes_data_blocks; 109 | } 110 | 111 | uint64_t 112 | mtbl_metadata_bytes_index_block(const struct mtbl_metadata *m) 113 | { 114 | return m->bytes_index_block; 115 | } 116 | 117 | uint64_t 118 | mtbl_metadata_bytes_keys(const struct mtbl_metadata *m) 119 | { 120 | return m->bytes_keys; 121 | } 122 | 123 | uint64_t 124 | mtbl_metadata_bytes_values(const struct mtbl_metadata *m) 125 | { 126 | return m->bytes_values; 127 | } 128 | -------------------------------------------------------------------------------- /src/mtbl_info.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2012, 2015 by Farsight Security, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | #include 30 | 31 | static void 32 | print_info(const char *fname) 33 | { 34 | int fd, ret; 35 | struct stat ss; 36 | struct mtbl_reader *r; 37 | const struct mtbl_metadata *m; 38 | 39 | fd = open(fname, O_RDONLY); 40 | if (fd < 0) { 41 | fprintf(stderr, "Error: unable to open file %s: %s\n", fname, strerror(errno)); 42 | exit(EXIT_FAILURE); 43 | } 44 | 45 | ret = fstat(fd, &ss); 46 | if (ret < 0) { 47 | perror("Error: fstat"); 48 | exit(EXIT_FAILURE); 49 | } 50 | 51 | r = mtbl_reader_init_fd(fd, NULL); 52 | if (r == NULL) { 53 | fprintf(stderr, "Error: mtbl_reader_init_fd() on %s failed\n", fname); 54 | exit(EXIT_FAILURE); 55 | } 56 | 57 | m = mtbl_reader_metadata(r); 58 | 59 | uint64_t data_block_size = mtbl_metadata_data_block_size(m); 60 | mtbl_compression_type compression_algorithm = 61 | mtbl_metadata_compression_algorithm(m); 62 | uint64_t count_entries = mtbl_metadata_count_entries(m); 63 | uint64_t count_data_blocks = mtbl_metadata_count_data_blocks(m); 64 | uint64_t bytes_data_blocks = mtbl_metadata_bytes_data_blocks(m); 65 | uint64_t bytes_index_block = mtbl_metadata_bytes_index_block(m); 66 | uint64_t bytes_keys = mtbl_metadata_bytes_keys(m); 67 | uint64_t bytes_values = mtbl_metadata_bytes_values(m); 68 | uint64_t index_block_offset = mtbl_metadata_index_block_offset(m); 69 | 70 | double p_data = 100.0 * bytes_data_blocks / ss.st_size; 71 | double p_index = 100.0 * bytes_index_block / ss.st_size; 72 | double compactness = 100.0 * ss.st_size / (bytes_keys + bytes_values); 73 | 74 | printf("file name: %s\n", fname); 75 | printf("file size: %'zd\n", (size_t) ss.st_size); 76 | printf("index block offset: %'" PRIu64 "\n", index_block_offset); 77 | printf("index bytes: %'" PRIu64 " (%'.2f%%)\n", bytes_index_block, p_index); 78 | printf("data block bytes %'" PRIu64 " (%'.2f%%)\n", bytes_data_blocks, p_data); 79 | printf("data block size: %'" PRIu64 "\n", data_block_size); 80 | printf("data block count %'" PRIu64 "\n", count_data_blocks); 81 | printf("entry count: %'" PRIu64 "\n", count_entries); 82 | printf("key bytes: %'" PRIu64 "\n", bytes_keys); 83 | printf("value bytes: %'" PRIu64 "\n", bytes_values); 84 | 85 | printf("compression algorithm: "); 86 | const char *compression = mtbl_compression_type_to_str(compression_algorithm); 87 | if (compression != NULL) 88 | puts(compression); 89 | else 90 | printf("%u\n", compression_algorithm); 91 | 92 | printf("compactness: %'.2f%%\n", compactness); 93 | 94 | putchar('\n'); 95 | 96 | mtbl_reader_destroy(&r); 97 | } 98 | 99 | int 100 | main(int argc, char **argv) 101 | { 102 | setlocale(LC_ALL, ""); 103 | 104 | if (argc < 2) { 105 | fprintf(stderr, "Usage: %s [...]\n", argv[0]); 106 | exit(EXIT_FAILURE); 107 | } 108 | for (int i = 1; i < argc; i++) 109 | print_info(argv[i]); 110 | 111 | return (EXIT_SUCCESS); 112 | } 113 | -------------------------------------------------------------------------------- /libmy/ip_arith.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2012 by Farsight Security, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef MY_IP_ARITH_H 18 | #define MY_IP_ARITH_H 19 | 20 | #ifdef HAVE_ENDIAN_H 21 | # include 22 | #else 23 | # ifdef HAVE_SYS_ENDIAN_H 24 | # include 25 | # endif 26 | #endif 27 | 28 | #include 29 | #include 30 | 31 | /* given big endian IPv4 prefix in 'address'/'prefix_len', 32 | * write lowest IP address ("network address") into 'out' */ 33 | static inline void 34 | ip4_lower(const void *address, unsigned prefix_len, void *out) { 35 | uint32_t addr; 36 | memcpy(&addr, address, 4); 37 | addr &= htobe32(~((1 << (32 - prefix_len)) - 1)); 38 | memcpy(out, &addr, 4); 39 | } 40 | 41 | /* given big endian IPv4 prefix in 'address'/'prefix_len', 42 | * write highest IP address ("broadcast address") into 'out' */ 43 | static inline void 44 | ip4_upper(const void *address, unsigned prefix_len, void *out) { 45 | uint32_t addr; 46 | memcpy(&addr, address, 4); 47 | addr |= htobe32((1 << (32 - prefix_len)) - 1); 48 | memcpy(out, &addr, 4); 49 | } 50 | 51 | /* increment IPv4 address by 1 */ 52 | static inline void 53 | ip4_incr(void *address) { 54 | uint32_t addr; 55 | memcpy(&addr, address, 4); 56 | addr = be32toh(addr) + 1; 57 | addr = htobe32(addr); 58 | memcpy(address, &addr, 4); 59 | } 60 | 61 | /* given big endian IPv6 prefix in 'address'/'prefix_len', 62 | * write lowest IP address ("network address") into 'lower' */ 63 | static inline void 64 | ip6_lower(const void *address, unsigned prefix_len, void *lower) 65 | { 66 | uint64_t addr[2], mask[2]; 67 | if (prefix_len < 64) { 68 | mask[0] = htobe64(((1ULL << prefix_len) - 1ULL) << (64ULL - prefix_len)); 69 | mask[1] = htobe64(0); 70 | } else if (prefix_len < 128) { 71 | prefix_len -= 64; 72 | mask[0] = htobe64(UINT64_MAX); 73 | mask[1] = htobe64(((1ULL << prefix_len) - 1ULL) << (64ULL - prefix_len)); 74 | } else { 75 | memset(mask, 0xFF, sizeof(mask)); 76 | } 77 | memcpy(addr, address, 16); 78 | addr[0] &= mask[0]; 79 | addr[1] &= mask[1]; 80 | memcpy(lower, addr, 16); 81 | } 82 | 83 | /* given big endian IPv6 prefix in 'address'/'prefix_len', 84 | * write highest IP address ("broadcast address") into 'upper' */ 85 | static inline void 86 | ip6_upper(const void *address, unsigned prefix_len, void *upper) 87 | { 88 | uint64_t addr[2], mask[2]; 89 | if (prefix_len < 64) { 90 | mask[0] = htobe64(~(((1ULL << prefix_len) - 1ULL) << (64ULL - prefix_len))); 91 | mask[1] = htobe64(UINT64_MAX); 92 | } else if (prefix_len < 128) { 93 | prefix_len -= 64; 94 | mask[0] = htobe64(0); 95 | mask[1] = htobe64(~(((1ULL << prefix_len) - 1ULL) << (64ULL - prefix_len))); 96 | } else { 97 | memset(&mask, 0, sizeof(mask)); 98 | } 99 | memcpy(addr, address, 16); 100 | addr[0] |= mask[0]; 101 | addr[1] |= mask[1]; 102 | memcpy(upper, addr, 16); 103 | } 104 | 105 | /* increment IPv6 address by 1 */ 106 | static inline void 107 | ip6_incr(void *address) { 108 | uint64_t addr[2]; 109 | memcpy(addr, address, 16); 110 | addr[0] = be64toh(addr[0]); 111 | addr[1] = be64toh(addr[1]); 112 | if (addr[1] == UINT64_MAX) { 113 | addr[0] += 1; 114 | addr[1] = 0; 115 | } else { 116 | addr[1] += 1; 117 | } 118 | addr[0] = htobe64(addr[0]); 119 | addr[1] = htobe64(addr[1]); 120 | memcpy(address, addr, 16); 121 | } 122 | 123 | #endif /* MY_IP_ARITH_H */ 124 | -------------------------------------------------------------------------------- /man/mtbl_source.3: -------------------------------------------------------------------------------- 1 | '\" t 2 | .\" Title: mtbl_source 3 | .\" Author: [FIXME: author] [see http://docbook.sf.net/el/author] 4 | .\" Generator: DocBook XSL Stylesheets v1.78.1 5 | .\" Date: 11/21/2016 6 | .\" Manual: \ \& 7 | .\" Source: \ \& 8 | .\" Language: English 9 | .\" 10 | .TH "MTBL_SOURCE" "3" "11/21/2016" "\ \&" "\ \&" 11 | .\" ----------------------------------------------------------------- 12 | .\" * Define some portability stuff 13 | .\" ----------------------------------------------------------------- 14 | .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 15 | .\" http://bugs.debian.org/507673 16 | .\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html 17 | .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 18 | .ie \n(.g .ds Aq \(aq 19 | .el .ds Aq ' 20 | .\" ----------------------------------------------------------------- 21 | .\" * set default formatting 22 | .\" ----------------------------------------------------------------- 23 | .\" disable hyphenation 24 | .nh 25 | .\" disable justification (adjust text to left margin only) 26 | .ad l 27 | .\" ----------------------------------------------------------------- 28 | .\" * MAIN CONTENT STARTS HERE * 29 | .\" ----------------------------------------------------------------- 30 | .SH "NAME" 31 | mtbl_source \- obtain key\-value entries from a data source 32 | .SH "SYNOPSIS" 33 | .sp 34 | \fB#include \fR 35 | .sp 36 | .nf 37 | \fBstruct mtbl_iter * 38 | mtbl_source_iter(const struct mtbl_source *\fR\fB\fIs\fR\fR\fB);\fR 39 | .fi 40 | .sp 41 | .nf 42 | \fBstruct mtbl_iter * 43 | mtbl_source_get(const struct mtbl_source *\fR\fB\fIs\fR\fR\fB, const uint8_t *\fR\fB\fIkey\fR\fR\fB, size_t \fR\fB\fIlen_key\fR\fR\fB);\fR 44 | .fi 45 | .sp 46 | .nf 47 | \fBstruct mtbl_iter * 48 | mtbl_source_get_prefix( 49 | const struct mtbl_source *\fR\fB\fIs\fR\fR\fB, 50 | const uint8_t *\fR\fB\fIprefix\fR\fR\fB, size_t \fR\fB\fIlen_prefix\fR\fR\fB);\fR 51 | .fi 52 | .sp 53 | .nf 54 | \fBstruct mtbl_iter * 55 | mtbl_source_get_range( 56 | const struct mtbl_source *\fR\fB\fIs\fR\fR\fB, 57 | const uint8_t *\fR\fB\fIkey0\fR\fR\fB, size_t \fR\fB\fIlen_key0\fR\fR\fB, 58 | const uint8_t *\fR\fB\fIkey1\fR\fR\fB, size_t \fR\fB\fIlen_key1\fR\fR\fB);\fR 59 | .fi 60 | .sp 61 | .nf 62 | \fBmtbl_res 63 | mtbl_source_write(const struct mtbl_source *\fR\fB\fIs\fR\fR\fB, struct mtbl_writer *\fR\fB\fIw\fR\fR\fB);\fR 64 | .fi 65 | .sp 66 | .nf 67 | \fBvoid 68 | mtbl_source_destroy(struct mtbl_source **\fR\fB\fIs\fR\fR\fB);\fR 69 | .fi 70 | .SH "DESCRIPTION" 71 | .sp 72 | The \fBmtbl_source\fR interface provides an abstraction for reading key\-value entries from mtbl data sources\&. 73 | .sp 74 | \fBmtbl_source_iter\fR() provides an iterator over all of the entries in the data source\&. 75 | .sp 76 | \fBmtbl_source_get\fR() provides an exact match iterator which returns all entries whose key matches the key provided in the arguments \fIkey\fR and \fIlen_key\fR\&. 77 | .sp 78 | \fBmtbl_source_get_prefix\fR() provides a prefix iterator which returns all entries whose keys start with \fIprefix\fR and are at least \fIlen_prefix\fR bytes long\&. 79 | .sp 80 | \fBmtbl_source_get_range\fR() provides a range iterator which returns all entries whose keys are between \fIkey0\fR and \fIkey1\fR inclusive\&. 81 | .sp 82 | \fBmtbl_source_write\fR() is a convenience function for reading all of the entries from a source and writing them to an \fBmtbl_writer\fR object\&. It is equivalent to calling \fBmtbl_writer_add\fR() on all of the entries returned from \fBmtbl_source_iter\fR()\&. 83 | .SH "RETURN VALUE" 84 | .sp 85 | \fBmtbl_source_iter\fR(), \fBmtbl_source_get\fR(), \fBmtbl_source_get_prefix\fR(), and \fBmtbl_source_get_range\fR() return \fBmtbl_iter\fR objects\&. 86 | .sp 87 | \fBmtbl_source_write\fR() returns \fBmtbl_res_success\fR if all of the entries in the data source were successfully written to the \fBmtbl_writer\fR argument, and \fBmtbl_res_failure\fR otherwise\&. 88 | .SH "SEE ALSO" 89 | .sp 90 | \fBmtbl_iter\fR(3) 91 | -------------------------------------------------------------------------------- /man/mtbl_metadata.3: -------------------------------------------------------------------------------- 1 | '\" t 2 | .\" Title: mtbl_metadata 3 | .\" Author: [FIXME: author] [see http://docbook.sf.net/el/author] 4 | .\" Generator: DocBook XSL Stylesheets v1.79.1 5 | .\" Date: 02/10/2017 6 | .\" Manual: \ \& 7 | .\" Source: \ \& 8 | .\" Language: English 9 | .\" 10 | .TH "MTBL_METADATA" "3" "02/10/2017" "\ \&" "\ \&" 11 | .\" ----------------------------------------------------------------- 12 | .\" * Define some portability stuff 13 | .\" ----------------------------------------------------------------- 14 | .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 15 | .\" http://bugs.debian.org/507673 16 | .\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html 17 | .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 18 | .ie \n(.g .ds Aq \(aq 19 | .el .ds Aq ' 20 | .\" ----------------------------------------------------------------- 21 | .\" * set default formatting 22 | .\" ----------------------------------------------------------------- 23 | .\" disable hyphenation 24 | .nh 25 | .\" disable justification (adjust text to left margin only) 26 | .ad l 27 | .\" ----------------------------------------------------------------- 28 | .\" * MAIN CONTENT STARTS HERE * 29 | .\" ----------------------------------------------------------------- 30 | .SH "NAME" 31 | mtbl_metadata \- get MTBL file metadata 32 | .SH "SYNOPSIS" 33 | .sp 34 | \fB#include \fR 35 | .sp 36 | .nf 37 | \fBmtbl_file_version 38 | mtbl_metadata_file_version(const struct mtbl_metadata *\fR\fB\fIm\fR\fR\fB);\fR 39 | .fi 40 | .sp 41 | .nf 42 | \fBuint64_t 43 | mtbl_metadata_index_block_offset(const struct mtbl_metadata *\fR\fB\fIm\fR\fR\fB);\fR 44 | .fi 45 | .sp 46 | .nf 47 | \fBuint64_t 48 | mtbl_metadata_data_block_size(const struct mtbl_metadata *\fR\fB\fIm\fR\fR\fB);\fR 49 | .fi 50 | .sp 51 | .nf 52 | \fBmtbl_compression_type 53 | mtbl_metadata_compression_algorithm(const struct mtbl_metadata *\fR\fB\fIm\fR\fR\fB);\fR 54 | .fi 55 | .sp 56 | .nf 57 | \fBuint64_t 58 | mtbl_metadata_count_entries(const struct mtbl_metadata *\fR\fB\fIm\fR\fR\fB);\fR 59 | .fi 60 | .sp 61 | .nf 62 | \fBuint64_t 63 | mtbl_metadata_count_data_blocks(const struct mtbl_metadata *\fR\fB\fIm\fR\fR\fB);\fR 64 | .fi 65 | .sp 66 | .nf 67 | \fBuint64_t 68 | mtbl_metadata_bytes_data_blocks(const struct mtbl_metadata *\fR\fB\fIm\fR\fR\fB);\fR 69 | .fi 70 | .sp 71 | .nf 72 | \fBuint64_t 73 | mtbl_metadata_bytes_index_block(const struct mtbl_metadata *\fR\fB\fIm\fR\fR\fB);\fR 74 | .fi 75 | .sp 76 | .nf 77 | \fBuint64_t 78 | mtbl_metadata_bytes_keys(const struct mtbl_metadata *\fR\fB\fIm\fR\fR\fB);\fR 79 | .fi 80 | .sp 81 | .nf 82 | \fBuint64_t 83 | mtbl_metadata_bytes_values(const struct mtbl_metadata *\fR\fB\fIm\fR\fR\fB);\fR 84 | .fi 85 | .SH "DESCRIPTION" 86 | .sp 87 | An \fBmtbl_metadata\fR object may be obtained from an \fBmtbl_reader\fR(3)\&. Its accessors export attributes and file statistics that are recorded in the metadata block\&. 88 | .SH "RETURN VALUE" 89 | .SS "mtbl_metadata_file_version()" 90 | .sp 91 | File format version of the MTBL file\&. Either MTBL_FORMAT_V1 or MTBL_FORMAT_V2\&. 92 | .SS "mtbl_metadata_index_block_offset()" 93 | .sp 94 | Byte offset in the MTBL file where the index begins\&. 95 | .SS "mtbl_metadata_data_block_size()" 96 | .sp 97 | Maximum size of an uncompressed data block, see \fBmtbl_writer\fR(3)\&. 98 | .SS "mtbl_metadata_compression_algorithm()" 99 | .sp 100 | One of the \fBcompression\fR values allowed by \fBmtbl_writer\fR(3)\&. 101 | .SS "mtbl_metadata_count_entries()" 102 | .sp 103 | Total number of key\-value entries\&. 104 | .SS "mtbl_metadata_count_data_blocks()" 105 | .sp 106 | Total number of data blocks\&. 107 | .SS "mtbl_metadata_bytes_data_blocks()" 108 | .sp 109 | Total number of bytes consumed by data blocks\&. 110 | .SS "mtbl_metadata_bytes_index_block()" 111 | .sp 112 | Total number of bytes consumed by the index\&. 113 | .SS "mtbl_metadata_bytes_keys()" 114 | .sp 115 | Total number of bytes that all keys would occupy if stored end\-to\-end in a byte array with no delimiters\&. 116 | .SS "mtbl_metadata_bytes_values()" 117 | .sp 118 | Total number of bytes that all values in the file would occupy if stored end\-to\-end in a byte array with no delimiters\&. 119 | -------------------------------------------------------------------------------- /configure.ac: -------------------------------------------------------------------------------- 1 | AC_PREREQ(2.64) 2 | AC_INIT([mtbl], 3 | [1.7.1], 4 | [https://github.com/farsightsec/mtbl/issues], 5 | [mtbl], 6 | [https://github.com/farsightsec/mtbl]) 7 | AC_CONFIG_SRCDIR([mtbl/mtbl.h]) 8 | AC_CONFIG_AUX_DIR([build-aux]) 9 | AM_INIT_AUTOMAKE([foreign 1.11 -Wall -Wno-portability silent-rules subdir-objects color-tests]) 10 | AC_PROG_CC_STDC 11 | AC_USE_SYSTEM_EXTENSIONS 12 | AC_SYS_LARGEFILE 13 | AC_CONFIG_MACRO_DIR([m4]) 14 | AM_SILENT_RULES([yes]) 15 | LT_INIT 16 | 17 | my_CFLAGS="-Wall \ 18 | -Wmissing-declarations -Wmissing-prototypes \ 19 | -Wnested-externs -Wpointer-arith \ 20 | -Wpointer-arith -Wsign-compare -Wchar-subscripts \ 21 | -Wstrict-prototypes -Wshadow \ 22 | -Wformat-security" 23 | AC_SUBST([my_CFLAGS]) 24 | 25 | AC_CONFIG_HEADERS(config.h) 26 | AC_CONFIG_FILES([Makefile mtbl/libmtbl.pc]) 27 | 28 | PKG_PROG_PKG_CONFIG 29 | if test -n "$PKG_CONFIG"; then 30 | # Horrible hack for systems where the pkg-config install directory is simply wrong! 31 | if $PKG_CONFIG --variable=pc_path pkg-config 2>/dev/null | grep -q /libdata/; then 32 | PKG_INSTALLDIR(['${prefix}/libdata/pkgconfig']) 33 | else 34 | PKG_INSTALLDIR 35 | fi 36 | else 37 | AC_MSG_ERROR([pkg-config is required!]) 38 | fi 39 | 40 | PKG_CHECK_MODULES([liblz4], [liblz4]) 41 | 42 | # space in third argument so success does nothing (does not set LIBS) 43 | AC_CHECK_LIB(lz4, LZ4_compress_HC, [ ], 44 | [AC_MSG_ERROR([liblz4 >= r130 required])], [${liblz4_LIBS}]) 45 | 46 | PKG_CHECK_MODULES([libzstd], [libzstd >= 0.8.0]) 47 | 48 | AC_C_BIGENDIAN 49 | 50 | AC_CHECK_FUNC([mkstemp], [], [ 51 | AC_MSG_ERROR([required system function not found]) 52 | ]) 53 | 54 | AC_CHECK_FUNCS([posix_madvise madvise]) 55 | 56 | AC_CHECK_HEADERS([sys/endian.h endian.h]) 57 | 58 | AC_CHECK_HEADER([snappy-c.h], [], [ 59 | AC_MSG_ERROR([required header file not found]) 60 | ]) 61 | AC_CHECK_LIB([snappy], [snappy_compress], [], [ 62 | AC_MSG_ERROR([required library not found]) 63 | ]) 64 | 65 | AC_CHECK_HEADER([zlib.h], [], [ 66 | AC_MSG_ERROR([required header file not found]) 67 | ]) 68 | AC_CHECK_LIB([z], [deflate], [], [ 69 | AC_MSG_ERROR([required library not found]) 70 | ]) 71 | 72 | AC_SEARCH_LIBS([dlopen], [dl]) 73 | 74 | AC_SEARCH_LIBS([clock_gettime], [rt]) 75 | AC_CHECK_FUNCS([clock_gettime]) 76 | 77 | AX_PTHREAD([ 78 | LIBS="$PTHREAD_LIBS $LIBS" 79 | CFLAGS="$CFLAGS $PTHREAD_CFLAGS" 80 | CC="$PTHREAD_CC" 81 | ]) 82 | 83 | AC_PATH_PROG([ASCIIDOC], [a2x]) 84 | AM_CONDITIONAL([BUILD_MAN], [test -n "$ASCIIDOC"]) 85 | if test -n "$ASCIIDOC"; then 86 | DOC_MAN_MSG="yes (asciidoc available)" 87 | else 88 | DOC_MAN_MSG="no (asciidoc not available)" 89 | fi 90 | 91 | gl_LD_VERSION_SCRIPT 92 | 93 | AC_ARG_WITH(coverage, 94 | [ --with-coverage[=PROGRAM] enable gtest and coverage target using the specified lcov], lcov="$withval", lcov="no") 95 | 96 | USE_LCOV="no" 97 | if test "$lcov" != "no"; then 98 | if test "$lcov" != "yes"; then 99 | LCOV=$lcov 100 | else 101 | AC_PATH_PROG([LCOV], [lcov]) 102 | fi 103 | if test -x "${LCOV}"; then 104 | USE_LCOV="yes" 105 | else 106 | AC_MSG_ERROR([Cannot find lcov.]) 107 | fi 108 | # is genhtml always in the same directory? 109 | GENHTML=`echo "$LCOV" | ${SED} s/lcov$/genhtml/` 110 | if test ! -x $GENHTML; then 111 | AC_MSG_ERROR([genhtml not found, needed for lcov]) 112 | fi 113 | CFLAGS="$CFLAGS --coverage" 114 | LIBS=" $LIBS -lgcov" 115 | AC_SUBST(CPPFLAGS) 116 | AC_SUBST(LIBS) 117 | AC_SUBST(LCOV) 118 | AC_SUBST(GENHTML) 119 | fi 120 | AC_SUBST(USE_LCOV) 121 | 122 | AC_OUTPUT 123 | AC_MSG_RESULT([ 124 | $PACKAGE $VERSION 125 | 126 | compiler: ${CC} 127 | cflags: ${CFLAGS} 128 | ldflags: ${LDFLAGS} 129 | libs: ${LIBS} 130 | 131 | prefix: ${prefix} 132 | sysconfdir: ${sysconfdir} 133 | libdir: ${libdir} 134 | includedir: ${includedir} 135 | pkgconfigdir: ${pkgconfigdir} 136 | 137 | bigendian: ${ac_cv_c_bigendian} 138 | 139 | building manpage docs: ${DOC_MAN_MSG} 140 | 141 | code coverage enabled: ${USE_LCOV} 142 | ]) 143 | -------------------------------------------------------------------------------- /libmy/my_queue_mutex.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, 2014 by Farsight Security, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include "my_alloc.h" 24 | 25 | #include "my_queue.h" 26 | 27 | #if defined(__GNUC__) 28 | # define _aligned __attribute__((aligned(64))) 29 | #else 30 | # define _aligned 31 | #endif 32 | 33 | struct my_queue { 34 | uint8_t *data; 35 | unsigned num_elems; 36 | unsigned sizeof_elem; 37 | unsigned head; 38 | unsigned tail; 39 | pthread_mutex_t lock _aligned; 40 | }; 41 | 42 | struct my_queue * 43 | my_queue_mutex_init(unsigned, unsigned); 44 | 45 | void 46 | my_queue_mutex_destroy(struct my_queue **); 47 | 48 | const char * 49 | my_queue_mutex_impl_type(void); 50 | 51 | bool 52 | my_queue_mutex_insert(struct my_queue *, void *, unsigned *); 53 | 54 | bool 55 | my_queue_mutex_remove(struct my_queue *, void *, unsigned *); 56 | 57 | struct my_queue * 58 | my_queue_mutex_init(unsigned num_elems, unsigned sizeof_elem) 59 | { 60 | struct my_queue *q; 61 | if (num_elems < 2 || ((num_elems - 1) & num_elems) != 0) 62 | return (NULL); 63 | q = my_calloc(1, sizeof(*q)); 64 | q->num_elems = num_elems; 65 | q->sizeof_elem = sizeof_elem; 66 | q->data = my_calloc(q->num_elems, q->sizeof_elem); 67 | int rc = pthread_mutex_init(&q->lock, NULL); 68 | assert(rc == 0); 69 | return (q); 70 | } 71 | 72 | void 73 | my_queue_mutex_destroy(struct my_queue **q) 74 | { 75 | if (*q) { 76 | pthread_mutex_destroy(&(*q)->lock); 77 | free((*q)->data); 78 | free(*q); 79 | *q = NULL; 80 | } 81 | } 82 | 83 | const char * 84 | my_queue_mutex_impl_type(void) 85 | { 86 | return ("pthread mutex"); 87 | } 88 | 89 | static inline void 90 | q_lock(struct my_queue *q) 91 | { 92 | int rc = pthread_mutex_lock(&q->lock); 93 | assert(rc == 0); 94 | } 95 | 96 | static inline void 97 | q_unlock(struct my_queue *q) 98 | { 99 | int rc = pthread_mutex_unlock(&q->lock); 100 | assert(rc == 0); 101 | } 102 | 103 | static inline unsigned 104 | q_space(unsigned head, unsigned tail, unsigned size) 105 | { 106 | return ((tail - (head + 1)) & (size - 1)); 107 | } 108 | 109 | static inline unsigned 110 | q_count(unsigned head, unsigned tail, unsigned size) 111 | { 112 | return ((head - tail) & (size - 1)); 113 | } 114 | 115 | bool 116 | my_queue_mutex_insert(struct my_queue *q, void *item, unsigned *pspace) 117 | { 118 | q_lock(q); 119 | bool res = false; 120 | unsigned head = q->head; 121 | unsigned tail = q->tail; 122 | unsigned space = q_space(head, tail, q->num_elems); 123 | if (space >= 1) { 124 | memcpy(&q->data[head * q->sizeof_elem], item, q->sizeof_elem); 125 | q->head = (head + 1) & (q->num_elems - 1); 126 | res = true; 127 | space--; 128 | } 129 | q_unlock(q); 130 | if (pspace) 131 | *pspace = space; 132 | return (res); 133 | } 134 | 135 | bool 136 | my_queue_mutex_remove(struct my_queue *q, void *item, unsigned *pcount) 137 | { 138 | q_lock(q); 139 | bool res = false; 140 | unsigned head = q->head; 141 | unsigned tail = q->tail; 142 | unsigned count = q_count(head, tail, q->num_elems); 143 | if (count >= 1) { 144 | memcpy(item, &q->data[tail * q->sizeof_elem], q->sizeof_elem); 145 | q->tail = (tail + 1) & (q->num_elems - 1); 146 | res = true; 147 | count--; 148 | } 149 | q_unlock(q); 150 | if (pcount) 151 | *pcount = count; 152 | return (res); 153 | } 154 | 155 | const struct my_queue_ops my_queue_mutex_ops = { 156 | .init = 157 | my_queue_mutex_init, 158 | .destroy = 159 | my_queue_mutex_destroy, 160 | .impl_type = 161 | my_queue_mutex_impl_type, 162 | .insert = 163 | my_queue_mutex_insert, 164 | .remove = 165 | my_queue_mutex_remove, 166 | }; 167 | -------------------------------------------------------------------------------- /libmy/zonefile.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include 8 | 9 | #include "my_alloc.h" 10 | #include "ubuf.h" 11 | #include "zonefile.h" 12 | 13 | struct zonefile { 14 | FILE *fp; 15 | bool eof; 16 | bool is_pipe; 17 | bool valid; 18 | ldns_rdf *domain; 19 | ldns_rdf *origin; 20 | ldns_rdf *prev; 21 | ldns_rr *rr_soa; 22 | uint32_t ttl; 23 | size_t count; 24 | }; 25 | 26 | static ldns_status 27 | read_soa(struct zonefile *z) 28 | { 29 | ldns_rr *rr; 30 | ldns_status status; 31 | 32 | for (;;) { 33 | status = ldns_rr_new_frm_fp_l(&rr, z->fp, &z->ttl, &z->origin, &z->prev, NULL); 34 | switch (status) { 35 | case LDNS_STATUS_OK: 36 | goto out; 37 | case LDNS_STATUS_SYNTAX_EMPTY: 38 | case LDNS_STATUS_SYNTAX_TTL: 39 | case LDNS_STATUS_SYNTAX_ORIGIN: 40 | status = LDNS_STATUS_OK; 41 | break; 42 | default: 43 | goto out; 44 | } 45 | } 46 | out: 47 | if (status != LDNS_STATUS_OK) { 48 | z->valid = false; 49 | return (LDNS_STATUS_ERR); 50 | } 51 | 52 | if (ldns_rr_get_type(rr) != LDNS_RR_TYPE_SOA) { 53 | ldns_rr_free(rr); 54 | z->valid = false; 55 | return (LDNS_STATUS_ERR); 56 | } 57 | 58 | z->count = 1; 59 | z->domain = ldns_rdf_clone(ldns_rr_owner(rr)); 60 | z->origin = ldns_rdf_clone(ldns_rr_owner(rr)); 61 | z->rr_soa = rr; 62 | return (LDNS_STATUS_OK); 63 | } 64 | 65 | struct zonefile * 66 | zonefile_init_fname(const char *fname) 67 | { 68 | struct zonefile *z = my_calloc(1, sizeof(struct zonefile)); 69 | 70 | size_t len_fname = strlen(fname); 71 | if (len_fname >= 3 && 72 | fname[len_fname - 3] == '.' && 73 | fname[len_fname - 2] == 'g' && 74 | fname[len_fname - 1] == 'z') 75 | { 76 | ubuf *u = ubuf_new(); 77 | ubuf_add_cstr(u, "zcat "); 78 | ubuf_add_cstr(u, fname); 79 | z->fp = popen(ubuf_cstr(u), "r"); 80 | z->is_pipe = true; 81 | ubuf_destroy(&u); 82 | } else { 83 | z->fp = fopen(fname, "r"); 84 | } 85 | 86 | if (z->fp == NULL) 87 | return (NULL); 88 | 89 | z->valid = true; 90 | if (read_soa(z) != LDNS_STATUS_OK) 91 | zonefile_destroy(&z); 92 | 93 | return (z); 94 | } 95 | 96 | void 97 | zonefile_destroy(struct zonefile **z) 98 | { 99 | if (*z) { 100 | if ((*z)->fp) { 101 | if ((*z)->is_pipe) 102 | pclose((*z)->fp); 103 | else 104 | fclose((*z)->fp); 105 | } 106 | if ((*z)->origin) 107 | ldns_rdf_deep_free((*z)->origin); 108 | if ((*z)->prev) 109 | ldns_rdf_deep_free((*z)->prev); 110 | if ((*z)->domain) 111 | ldns_rdf_deep_free((*z)->domain); 112 | if ((*z)->rr_soa) 113 | ldns_rr_free((*z)->rr_soa); 114 | free(*z); 115 | *z = NULL; 116 | } 117 | } 118 | 119 | const ldns_rdf * 120 | zonefile_get_domain(struct zonefile *z) 121 | { 122 | return (z->domain); 123 | } 124 | 125 | size_t 126 | zonefile_get_count(struct zonefile *z) 127 | { 128 | return (z->count); 129 | } 130 | 131 | uint32_t 132 | zonefile_get_serial(struct zonefile *z) 133 | { 134 | ldns_rdf *rdf = ldns_rr_rdf(z->rr_soa, 2); 135 | assert(rdf != NULL); 136 | return (ldns_rdf2native_int32(rdf)); 137 | } 138 | 139 | ldns_status 140 | zonefile_read(struct zonefile *z, ldns_rr **out) 141 | { 142 | ldns_rr *rr; 143 | ldns_status status = LDNS_STATUS_OK; 144 | 145 | if (z->eof) { 146 | *out = NULL; 147 | return (LDNS_STATUS_OK); 148 | } 149 | 150 | if (!z->valid) 151 | return (LDNS_STATUS_ERR); 152 | 153 | if (z->count == 1 && z->rr_soa != NULL) { 154 | *out = z->rr_soa; 155 | z->rr_soa = NULL; 156 | return (LDNS_STATUS_OK); 157 | } 158 | for (;;) { 159 | if (feof(z->fp)) { 160 | *out = NULL; 161 | z->eof = true; 162 | return (LDNS_STATUS_OK); 163 | } 164 | status = ldns_rr_new_frm_fp_l(&rr, z->fp, &z->ttl, &z->origin, &z->prev, NULL); 165 | switch (status) { 166 | case LDNS_STATUS_OK: 167 | if (ldns_rr_get_type(rr) == LDNS_RR_TYPE_SOA) { 168 | ldns_rr_free(rr); 169 | *out = NULL; 170 | return (LDNS_STATUS_OK); 171 | } 172 | z->count++; 173 | goto out; 174 | case LDNS_STATUS_SYNTAX_EMPTY: 175 | case LDNS_STATUS_SYNTAX_TTL: 176 | case LDNS_STATUS_SYNTAX_ORIGIN: 177 | status = LDNS_STATUS_OK; 178 | break; 179 | default: 180 | goto out; 181 | } 182 | } 183 | out: 184 | if (status != LDNS_STATUS_OK) 185 | return (status); 186 | *out = rr; 187 | return (status); 188 | } 189 | -------------------------------------------------------------------------------- /libmy/heap.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023 DomainTools LLC 3 | * Copyright (c) 2012, 2019 by Farsight Security, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include 19 | 20 | #include "my_alloc.h" 21 | #include "heap.h" 22 | #include "vector.h" 23 | 24 | VECTOR_GENERATE(ptrvec, void *); 25 | 26 | struct heap { 27 | ptrvec *vec; 28 | heap_compare_func cmp; 29 | void *clos; 30 | }; 31 | 32 | struct heap * 33 | heap_init(heap_compare_func cmp, void *clos) 34 | { 35 | struct heap *h = my_calloc(1, sizeof(*h)); 36 | h->cmp = cmp; 37 | h->clos = clos; 38 | h->vec = ptrvec_init(1); 39 | return (h); 40 | } 41 | 42 | void 43 | heap_destroy(struct heap **h) 44 | { 45 | if (*h != NULL) { 46 | ptrvec_destroy(&(*h)->vec); 47 | free(*h); 48 | *h = NULL; 49 | } 50 | } 51 | 52 | void heap_reset(struct heap *h) 53 | { 54 | ptrvec_reset(h->vec); 55 | } 56 | 57 | void heap_clip(struct heap *h, size_t n_elems) 58 | { 59 | ptrvec_clip(h->vec, n_elems); 60 | } 61 | 62 | static void 63 | siftup(struct heap *h) 64 | { 65 | size_t pos = ptrvec_size(h->vec)-1; 66 | void *newitem = ptrvec_value(h->vec, pos); 67 | while (pos > 0) { 68 | size_t parentpos = (pos - 1) >> 1; 69 | void *parent = ptrvec_value(h->vec, parentpos); 70 | if (h->cmp(parent, newitem, h->clos) <= 0) 71 | break; 72 | ptrvec_data(h->vec)[pos] = parent; 73 | pos = parentpos; 74 | } 75 | ptrvec_data(h->vec)[pos] = newitem; 76 | } 77 | 78 | static void 79 | siftdown(struct heap *h, size_t pos) 80 | { 81 | assert(pos < ptrvec_size(h->vec)); 82 | void *newitem = ptrvec_value(h->vec, pos); 83 | size_t endpos = ptrvec_size(h->vec); 84 | size_t childpos = 2 * pos + 1; 85 | 86 | while (childpos < endpos) { 87 | size_t rightpos = childpos + 1; 88 | void *childval = ptrvec_value(h->vec, childpos); 89 | if (rightpos < endpos) { 90 | void *rightval = ptrvec_value(h->vec, rightpos); 91 | if (h->cmp(rightval, childval, h->clos) <= 0) { 92 | childpos = rightpos; 93 | childval = rightval; 94 | } 95 | } 96 | if (h->cmp(newitem, childval, h->clos) <= 0) 97 | break; 98 | ptrvec_data(h->vec)[pos] = childval; 99 | pos = childpos; 100 | childpos = 2 * pos + 1; 101 | } 102 | ptrvec_data(h->vec)[pos] = newitem; 103 | } 104 | 105 | void 106 | heap_heapify(struct heap *h) 107 | { 108 | for (size_t i = ptrvec_size(h->vec)/2; i-- > 0; ) { 109 | siftdown(h, i); 110 | } 111 | } 112 | 113 | void 114 | heap_add(struct heap *h, void *item) 115 | { 116 | ptrvec_add(h->vec, item); 117 | } 118 | 119 | void 120 | heap_push(struct heap *h, void *item) 121 | { 122 | ptrvec_add(h->vec, item); 123 | siftup(h); 124 | } 125 | 126 | void * 127 | heap_pop(struct heap *h) 128 | { 129 | if (ptrvec_size(h->vec) < 1) 130 | return (NULL); 131 | void *returnitem; 132 | void *lastelt = ptrvec_value(h->vec, ptrvec_size(h->vec) - 1); 133 | ptrvec_clip(h->vec, ptrvec_size(h->vec) - 1); 134 | if (ptrvec_size(h->vec) > 0) { 135 | returnitem = ptrvec_value(h->vec, 0); 136 | ptrvec_data(h->vec)[0] = lastelt; 137 | siftdown(h, 0); 138 | } else { 139 | returnitem = lastelt; 140 | } 141 | return (returnitem); 142 | } 143 | 144 | void * 145 | heap_replace(struct heap *h, void *item) 146 | { 147 | if (ptrvec_size(h->vec) < 1) 148 | return (NULL); 149 | void *returnitem = ptrvec_value(h->vec, 0); 150 | ptrvec_data(h->vec)[0] = item; 151 | siftdown(h, 0); 152 | return (returnitem); 153 | } 154 | 155 | void * 156 | heap_peek(struct heap *h) 157 | { 158 | if (ptrvec_size(h->vec) < 1) 159 | return (NULL); 160 | return ptrvec_data(h->vec)[0]; 161 | } 162 | 163 | void * 164 | heap_get(struct heap *h, size_t i) 165 | { 166 | if (i > ptrvec_size(h->vec) - 1) 167 | return (NULL); 168 | return (ptrvec_value(h->vec, i)); 169 | } 170 | 171 | size_t 172 | heap_size(struct heap *h) 173 | { 174 | return (ptrvec_size(h->vec)); 175 | } 176 | -------------------------------------------------------------------------------- /t/test-vector.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | 11 | #include "mtbl-private.h" 12 | #include "libmy/ubuf.h" 13 | 14 | #define NAME "test-vector" 15 | 16 | VECTOR_GENERATE(uint32_vec, uint32_t); 17 | 18 | const uint32_t u32_array[] = { 23, 17, 1, 2, 3, 4, 5, 4, 3, 2, 1, 17, 23 }; 19 | #define NUM_U32_ARRAY 13 20 | 21 | const char str_array[] = "`1234567890[]',.PYFGCRL/=AOEUIDHTNS-;QJKXBMWVZ"; 22 | 23 | static int 24 | test6(void) 25 | { 26 | int ret = 0; 27 | ubuf *v = ubuf_init(16); 28 | ubuf_append(v, (uint8_t *) str_array, sizeof(str_array)); 29 | ubuf_clip(v, 17); 30 | if (ubuf_size(v) != 17) 31 | ret |= 1; 32 | ubuf_destroy(&v); 33 | return (ret); 34 | } 35 | 36 | static int 37 | test5(void) 38 | { 39 | int ret = 0; 40 | uint32_vec *v; 41 | 42 | v = uint32_vec_init(0); 43 | for (unsigned i = 0; i < 16; i++) 44 | uint32_vec_append(v, u32_array, sizeof(u32_array) / sizeof(uint32_t)); 45 | for (unsigned i = 0; i < uint32_vec_size(v); i++) { 46 | //fprintf(stderr, "v->v[%d] = %u\n", i, uint32_vec_value(v, i)); 47 | if (uint32_vec_value(v, i) != u32_array[i % NUM_U32_ARRAY]) 48 | ret |= 1; 49 | } 50 | uint32_vec_reset(v); 51 | 52 | for (uint32_t i = 0; i < 1024; i++) 53 | uint32_vec_add(v, i); 54 | for (uint32_t i = 0; i < 1024; i++) { 55 | //fprintf(stderr, "v->v[%d] = %u\n", i, v->v[i]); 56 | if (uint32_vec_value(v, i) != i) 57 | ret |= 1; 58 | } 59 | 60 | uint32_vec_destroy(&v); 61 | 62 | return (ret); 63 | } 64 | 65 | static int 66 | test4(void) 67 | { 68 | int ret = 0; 69 | ubuf *v; 70 | 71 | v = ubuf_init(0); 72 | for (unsigned i = 0; i < 1024; i++) 73 | ubuf_append(v, (uint8_t *) str_array, sizeof(str_array)); 74 | 75 | for (unsigned i = 0; i < ubuf_size(v); i++) { 76 | unsigned j = i % sizeof(str_array); 77 | /* 78 | fprintf(stderr, "v->v[%d] = '%c', str_array[%d] = '%c'\n", 79 | i, v->v[i], j, str_array[j]); 80 | */ 81 | if (ubuf_value(v, i) != str_array[j]) 82 | ret |= 1; 83 | } 84 | 85 | ubuf_destroy(&v); 86 | 87 | return (ret); 88 | } 89 | 90 | static int 91 | test3(void) 92 | { 93 | int ret = 0; 94 | ubuf *v; 95 | 96 | v = ubuf_init(16); 97 | ubuf_append(v, (uint8_t *) str_array, sizeof(str_array)); 98 | for (unsigned i = 0; i < ubuf_size(v); i++) { 99 | //fprintf(stderr, "v->v[%d] = '%c'\n", i, v->v[i]); 100 | if (ubuf_value(v, i) != str_array[i]) 101 | ret |= 1; 102 | } 103 | ubuf_destroy(&v); 104 | 105 | return (ret); 106 | } 107 | 108 | static int 109 | test2(void) 110 | { 111 | int ret = 0; 112 | uint32_vec *v; 113 | 114 | v = uint32_vec_init(0); 115 | uint32_vec_append(v, u32_array, sizeof(u32_array) / sizeof(uint32_t)); 116 | for (uint32_t i = 0; i < uint32_vec_size(v); i++) { 117 | if (uint32_vec_value(v, i) != u32_array[i]) 118 | ret |= 1; 119 | //fprintf(stderr, "v->v[%d] = %u\n", i, v->v[i]); 120 | } 121 | if (uint32_vec_size(v) != NUM_U32_ARRAY) 122 | ret |= 1; 123 | uint32_vec_destroy(&v); 124 | 125 | return (ret); 126 | } 127 | 128 | static int 129 | test1(void) 130 | { 131 | int ret = 0; 132 | uint32_vec *v; 133 | 134 | v = uint32_vec_init(0); 135 | for (uint32_t i = 0; i < 1024; i++) 136 | uint32_vec_add(v, i); 137 | if (uint32_vec_bytes(v) != uint32_vec_size(v) * sizeof(uint32_t)) { 138 | fprintf(stderr, "uint32_vec_bytes (%zd) != v->n * sizeof(uint32_t) (%zd)\n", 139 | uint32_vec_bytes(v), uint32_vec_size(v) * sizeof(uint32_t)); 140 | ret |= 1; 141 | } 142 | for (uint32_t i = 0; i < 1024; i++) { 143 | //fprintf(stderr, "v->v[%d] = %u\n", i, v->v[i]); 144 | if (uint32_vec_value(v, i) != i) 145 | ret |= 1; 146 | } 147 | uint32_vec_destroy(&v); 148 | 149 | return (ret); 150 | } 151 | 152 | static int 153 | check(int ret, const char *s) 154 | { 155 | if (ret == 0) 156 | fprintf(stderr, NAME ": PASS: %s\n", s); 157 | else 158 | fprintf(stderr, NAME ": FAIL: %s\n", s); 159 | return (ret); 160 | } 161 | 162 | int 163 | main(int argc, char **argv) 164 | { 165 | int ret = 0; 166 | 167 | ret |= check(test1(), "test1"); 168 | ret |= check(test2(), "test2"); 169 | ret |= check(test3(), "test3"); 170 | ret |= check(test4(), "test4"); 171 | ret |= check(test5(), "test5"); 172 | ret |= check(test6(), "test6"); 173 | 174 | if (ret) 175 | return (EXIT_FAILURE); 176 | return (EXIT_SUCCESS); 177 | } 178 | -------------------------------------------------------------------------------- /libmy/b32_encode.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 by Internet Systems Consortium, Inc. ("ISC") 3 | * 4 | * Permission to use, copy, modify, and/or distribute this software for any 5 | * purpose with or without fee is hereby granted, provided that the above 6 | * copyright notice and this permission notice appear in all copies. 7 | * 8 | * THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES 9 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 10 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR 11 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 12 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 13 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT 14 | * OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 15 | */ 16 | 17 | /* 18 | * Copyright (c) 2006 Christian Biere 19 | * All rights reserved. 20 | * 21 | * Redistribution and use in source and binary forms, with or without 22 | * modification, are permitted provided that the following conditions 23 | * are met: 24 | * 25 | * 1. Redistributions of source code must retain the above copyright 26 | * notice, this list of conditions and the following disclaimer. 27 | * 2. Redistributions in binary form must reproduce the above copyright 28 | * notice, this list of conditions and the following disclaimer in the 29 | * documentation and/or other materials provided with the distribution. 30 | * 3. Neither the name of the authors nor the names of its contributors 31 | * may be used to endorse or promote products derived from this software 32 | * without specific prior written permission. 33 | * 34 | * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 35 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 36 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 37 | * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 38 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 39 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 40 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 41 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 42 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 43 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 44 | * SUCH DAMAGE. 45 | */ 46 | 47 | /* 48 | * See RFC 4648 for details about Base 32 hex encoding: 49 | * http://tools.ietf.org/html/rfc4648 50 | */ 51 | 52 | #include 53 | #include 54 | 55 | #include "b32_encode.h" 56 | 57 | static const char base32_alphabet[32] = { 58 | '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 59 | 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 60 | 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 61 | 'U', 'V' 62 | }; 63 | 64 | /** 65 | * Encode in base32 `len' bytes of `data' into the buffer `dst'. 66 | * 67 | * @param dst destination buffer 68 | * @param size length of destination 69 | * @param data start of data to encode 70 | * @param len amount of bytes to encode 71 | * 72 | * @return the amount of bytes generated into the destination. 73 | */ 74 | size_t 75 | base32_encode(char *dst, size_t size, const void *data, size_t len) 76 | { 77 | size_t i = 0; 78 | const uint8_t *p = data; 79 | const char *end = &dst[size]; 80 | char *q = dst; 81 | 82 | do { 83 | size_t j, k; 84 | uint8_t x[5]; 85 | char s[8]; 86 | 87 | switch (len - i) { 88 | case 4: 89 | k = 7; 90 | break; 91 | case 3: 92 | k = 5; 93 | break; 94 | case 2: 95 | k = 3; 96 | break; 97 | case 1: 98 | k = 2; 99 | break; 100 | default: 101 | k = 8; 102 | } 103 | 104 | for (j = 0; j < 5; j++) 105 | x[j] = i < len ? p[i++] : 0; 106 | 107 | s[0] = (x[0] >> 3); 108 | s[1] = ((x[0] & 0x07) << 2) | (x[1] >> 6); 109 | s[2] = (x[1] >> 1) & 0x1f; 110 | s[3] = ((x[1] & 0x01) << 4) | (x[2] >> 4); 111 | s[4] = ((x[2] & 0x0f) << 1) | (x[3] >> 7); 112 | s[5] = (x[3] >> 2) & 0x1f; 113 | s[6] = ((x[3] & 0x03) << 3) | (x[4] >> 5); 114 | s[7] = x[4] & 0x1f; 115 | 116 | for (j = 0; j < k && q != end; j++) { 117 | *q++ = base32_alphabet[(uint8_t) s[j]]; 118 | } 119 | 120 | if (end == q) { 121 | break; 122 | } 123 | 124 | } while (i < len); 125 | 126 | return q - dst; 127 | } 128 | -------------------------------------------------------------------------------- /t/test-block_builder.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | 11 | #include "block_builder.c" 12 | 13 | #define NAME "test-block_builder" 14 | 15 | #include 16 | 17 | static void 18 | print_string(const uint8_t *data, size_t len, FILE *out) { 19 | unsigned c; 20 | 21 | fputc('\'', out); 22 | while (len-- != 0) { 23 | c = *(data++); 24 | if (isprint(c)) 25 | fputc(c, out); 26 | else 27 | fprintf(out, "\\x%02x", c); 28 | } 29 | fputc('\'', out); 30 | fputc('\n', out); 31 | } 32 | 33 | static int 34 | test1(void) 35 | { 36 | int ret = 0; 37 | struct block_builder *b; 38 | uint8_t *buf; 39 | size_t bufsz; 40 | 41 | b = block_builder_init(16); 42 | assert(b != NULL); 43 | 44 | fprintf(stderr, "block_builder_current_size_estimate(): %zd\n", 45 | block_builder_current_size_estimate(b)); 46 | 47 | block_builder_add(b, (uint8_t *) "foobar1", 7, (uint8_t *) "42", 2); 48 | block_builder_add(b, (uint8_t *) "foobar2", 7, (uint8_t *) "43", 2); 49 | block_builder_add(b, (uint8_t *) "foobar2ZZ", 9, (uint8_t *) "44", 2); 50 | block_builder_add(b, (uint8_t *) "foobar3", 7, (uint8_t *) "45", 2); 51 | block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "46", 2); 52 | block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "47", 2); 53 | block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "48", 2); 54 | block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "49", 2); 55 | block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "50", 2); 56 | block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "51", 2); 57 | block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "52", 2); 58 | block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "53", 2); 59 | block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "54", 2); 60 | block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "55", 2); 61 | block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "56", 2); 62 | block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "57", 2); 63 | block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "58", 2); 64 | block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "59", 2); 65 | block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "60", 2); 66 | block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "61", 2); 67 | block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "62", 2); 68 | block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "63", 2); 69 | block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "64", 2); 70 | block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "ZA", 2); 71 | block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "ZB", 2); 72 | block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "ZC", 2); 73 | block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "ZD", 2); 74 | block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "ZE", 2); 75 | block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "ZF", 2); 76 | block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "ZG", 2); 77 | block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "ZH", 2); 78 | block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "ZI", 2); 79 | block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "ZJ", 2); 80 | block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "ZK", 2); 81 | block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "ZM", 2); 82 | 83 | fprintf(stderr, "block_builder_current_size_estimate(): %zd\n", 84 | block_builder_current_size_estimate(b)); 85 | 86 | block_builder_finish(b, &buf, &bufsz); 87 | fprintf(stderr, "buf= %p bufsz= %zd\n", buf, bufsz); 88 | 89 | print_string(buf, bufsz, stderr); 90 | 91 | block_builder_reset(b); 92 | block_builder_destroy(&b); 93 | 94 | return (ret); 95 | } 96 | 97 | static int 98 | check(int ret, const char *s) 99 | { 100 | if (ret == 0) 101 | fprintf(stderr, NAME ": PASS: %s\n", s); 102 | else 103 | fprintf(stderr, NAME ": FAIL: %s\n", s); 104 | return (ret); 105 | } 106 | 107 | int 108 | main(int argc, char **argv) 109 | { 110 | int ret = 0; 111 | 112 | ret |= check(test1(), "test1"); 113 | 114 | if (ret) 115 | return (EXIT_FAILURE); 116 | return (EXIT_SUCCESS); 117 | } 118 | --------------------------------------------------------------------------------